Merge pull request #2 from slash3b/improv

fix: better support for reading tiny payloads

Author: slash3b

utfbom.go

@@ -7,22 +7,23 @@ package utfbom
 
 import (
 	"bufio"
+	"bytes"
 	"errors"
 	"io"
 	"sync"
 )
 
 var (
 	_          io.Reader = (*Reader)(nil)
-	utf8BOM              = [3]byte{0xef, 0xbb, 0xbf}
-	utf16BEBOM           = [2]byte{0xfe, 0xff}
-	utf16LEBOM           = [2]byte{0xff, 0xfe}
-	utf32BEBOM           = [4]byte{0x00, 0x00, 0xfe, 0xff}
-	utf32LEBOM           = [4]byte{0xff, 0xfe, 0x00, 0x00}
+	utf8BOM              = []byte{0xef, 0xbb, 0xbf}
+	utf16BEBOM           = []byte{0xfe, 0xff}
+	utf16LEBOM           = []byte{0xff, 0xfe}
+	utf32BEBOM           = []byte{0x00, 0x00, 0xfe, 0xff}
+	utf32LEBOM           = []byte{0xff, 0xfe, 0x00, 0x00}
 )
 
 // ErrRead helps to trace error origin.
-var ErrRead = errors.New("utfbom library unable to detect BOM")
+var ErrRead = errors.New("utfbom: I/O error during BOM processing")
 
 // Encoding is a character encoding standard.
 type Encoding int
@@ -64,39 +65,31 @@ const (
 //   - UTF-32 Big Endian (BOM: 0x00 0x00 0xfe 0xff)
 //   - UTF-32 Little Endian (BOM: 0xff 0xfe 0x00 0x00)
 func DetectEncoding[T string | []byte](input T) Encoding {
-	bytes := []byte(input)
+	ibs := []byte(input)
 
-	if len(bytes) < 2 {
+	if len(ibs) < 2 {
 		return Unknown
 	}
 
-	if len(bytes) >= 4 {
-		if utf32BEBOM[0] == bytes[0] &&
-			utf32BEBOM[1] == bytes[1] &&
-			utf32BEBOM[2] == bytes[2] &&
-			utf32BEBOM[3] == bytes[3] {
+	if len(ibs) >= 4 {
+		if bytes.HasPrefix(ibs, utf32BEBOM) {
 			return UTF32BigEndian
 		}
 
-		if utf32LEBOM[0] == bytes[0] &&
-			utf32LEBOM[1] == bytes[1] &&
-			utf32LEBOM[2] == bytes[2] &&
-			utf32LEBOM[3] == bytes[3] {
+		if bytes.HasPrefix(ibs, utf32LEBOM) {
 			return UTF32LittleEndian
 		}
 	}
 
-	if len(bytes) >= 3 {
-		if utf8BOM[0] == bytes[0] && utf8BOM[1] == bytes[1] && utf8BOM[2] == bytes[2] {
-			return UTF8
-		}
+	if len(ibs) >= 3 && bytes.HasPrefix(ibs, utf8BOM) {
+		return UTF8
 	}
 
-	if utf16BEBOM[0] == bytes[0] && utf16BEBOM[1] == bytes[1] {
+	if bytes.HasPrefix(ibs, utf16BEBOM) {
 		return UTF16BigEndian
 	}
 
-	if utf16LEBOM[0] == bytes[0] && utf16LEBOM[1] == bytes[1] {
+	if bytes.HasPrefix(ibs, utf16LEBOM) {
 		return UTF16LittleEndian
 	}
 
@@ -194,7 +187,9 @@ func (r *Reader) Read(buf []byte) (int, error) {
 
 	r.once.Do(func() {
 		bytes, err := r.rd.Peek(maxBOMLen)
-		if err != nil {
+		// do not error out in case underlying payload is too small
+		// still attempt to read fewer than n bytes.
+		if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
 			bomErr = errors.Join(ErrRead, err)
 
 			return

utfbom_test.go

@@ -310,21 +310,44 @@ func TestReader_EmptyBuffer(t *testing.T) {
 	}
 }
 
-func TestReader_WrappeeReaderIsTooSmall(t *testing.T) {
+// TestReader_WrappeeReaderHasTinyPayload tests that bufio.Reader is able to
+// read on the first Read without failing.
+func TestReader_WrappeeReaderHasTinyPayload_EnoughBuffer(t *testing.T) {
 	t.Parallel()
 
-	wrappee := strings.NewReader("a")
+	wrappee := bytes.NewReader([]byte{0xff, 0xfe, 0x01, 0x02, 0x03})
 	wrapped := utfbom.NewReader(wrappee)
 
 	buf := make([]byte, 100)
 	n, err := wrapped.Read(buf)
-	be.Equal(t, 0, n)
-	be.Err(t, err, io.EOF)
-	be.Err(t, err, utfbom.ErrRead)
+	be.Equal(t, 3, n)
+	t.Logf("have read %q", string(buf[:n]))
+	t.Logf("detected enc: %s", wrapped.Enc)
+	t.Logf("err: %v", err)
+	be.Equal(t, buf[:n], []byte{0x01, 0x02, 0x03})
+	be.Err(t, err, nil)
 
-	// you might proceed reading if you want
 	n, err = wrapped.Read(buf)
-	be.Err(t, err, nil)
-	be.Equal(t, 1, n)
-	be.Equal(t, string(buf[:n]), "a")
+	t.Logf("second read returns err: %v", err)
+	be.Err(t, err, io.EOF)
+	be.Equal(t, 0, n)
+}
+
+func TestReader_WrappeeReaderHasTinyPayload_OneByteBuffer(t *testing.T) {
+	t.Parallel()
+
+	wrappee := bytes.NewReader([]byte{0xff, 0xfe, 0x01, 0x02, 0x03})
+	rd := iotest.OneByteReader(utfbom.NewReader(wrappee))
+
+	buf := make([]byte, 1)
+	for i := range 3 {
+		n, err := rd.Read(buf)
+		be.Err(t, err, nil)
+		be.Equal(t, 1, n)
+		be.Equal(t, []byte{0x01 + byte(i)}, buf[:n])
+	}
+
+	n, err := rd.Read(buf)
+	be.Err(t, err, io.EOF)
+	be.Equal(t, 0, n)
 }