diff --git a/Dockerfile b/Dockerfile index 8492e9fd..cf42db04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # and "Missing User Instruction" since 2ms container is stopped after scan # Builder image -FROM checkmarx/go:1.27.3-r0-94581adffb0633@sha256:94581adffb0633c4cdde4b15edc8834c192204e9af0a5247f85e91294ceb2e82 AS builder +FROM checkmarx/go:1.27.4-r0-949d8d960bf5ce@sha256:949d8d960bf5ce70198fe2ddc9c00e6f09c2badcc799a8f6d46308eacda1ad63 AS builder WORKDIR /app @@ -20,7 +20,7 @@ COPY . . RUN GOOS=linux GOARCH=amd64 go build -buildvcs=false -ldflags="-s -w" -a -o /app/2ms . # Runtime image -FROM checkmarx/git:2.56.0-r3-f9cad2bd619390@sha256:f9cad2bd61939058ef50a139cb2d8fbd3cf01b435a0992029e3584e9c6f10f61 +FROM checkmarx/git:2.56.1-r1-bf7f4c34ff4dd6@sha256:bf7f4c34ff4dd67bbd402dd5aaade8f5ace24ea2be6eb46c263e0ea982acf6e3 WORKDIR /app diff --git a/engine/chunk/chunk.go b/engine/chunk/chunk.go index 4011d7d3..57a7b16a 100644 --- a/engine/chunk/chunk.go +++ b/engine/chunk/chunk.go @@ -7,6 +7,7 @@ import ( "bytes" "errors" "fmt" + "io" "sync" "unicode" @@ -47,7 +48,6 @@ func WithSmallFileThreshold(smallFileThreshold int64) Option { // Chunk holds two pools and sizing parameters needed for reading chunks of data with look-ahead type Chunk struct { bufPool *sync.Pool // *bytes.Buffer with cap Size + MaxPeekSize - peekedBufPool *sync.Pool // *[]byte slices of length Size + MaxPeekSize size int // base chunk size maxPeekSize int // max size of look-ahead bytes smallFileThreshold int64 // files smaller than this skip chunking @@ -77,13 +77,6 @@ func New(opts ...Option) *Chunk { return bytes.NewBuffer(make([]byte, 0, c.size+c.maxPeekSize)) }, } - c.peekedBufPool = &sync.Pool{ - New: func() interface{} { - // pre-allocate fixed-size block for loading chunks - b := make([]byte, c.size+c.maxPeekSize) - return &b - }, - } return c } @@ -103,23 +96,6 @@ func (c *Chunk) PutBuf(window *bytes.Buffer) { c.bufPool.Put(window) } -// GetPeekedBuf returns a fixed-size []byte from the pool -func (c *Chunk) GetPeekedBuf() (*[]byte, bool) { - b, ok := c.peekedBufPool.Get().(*[]byte) - if !ok { - return nil, false - } - // reslice to its full capacity so Read can fill it - *b = (*b)[:cap(*b)] - return b, ok -} - -// PutPeekedBuf returns the fixed-size []byte to the pool -func (c *Chunk) PutPeekedBuf(b *[]byte) { - *b = (*b)[:0] // reset the slice to zero length - c.peekedBufPool.Put(b) -} - func (c *Chunk) GetSize() int { return c.size } @@ -133,29 +109,36 @@ func (c *Chunk) GetFileThreshold() int64 { } // ReadChunk reads the next chunk of data from file +// Only the bytes that are part of the returned chunk are consumed from the reader func (c *Chunk) ReadChunk(reader *bufio.Reader, totalLines int) (string, error) { - // borrow a []bytes from the pool and seed it with raw data from file (up to chunk size + peek size) - rawData, ok := c.GetPeekedBuf() - if !ok { - return "", fmt.Errorf("expected *bytes.Buffer, got %T", rawData) - } - defer c.PutPeekedBuf(rawData) - n, err := reader.Read(*rawData) + // look at the raw data from file (up to chunk size + peek size) without consuming it + rawData, err := reader.Peek(c.size + c.maxPeekSize) - var chunkStr string // "Callers should always process the n > 0 bytes returned before considering the error err." // https://pkg.go.dev/io#Reader - if n > 0 { - // only check the filetype at the start of file - if totalLines == 0 && ShouldSkipFile((*rawData)[:n]) { - return "", fmt.Errorf("skipping file: %w", ErrUnsupportedFileType) - } + // Peek returns fewer bytes together with io.EOF when the file ends before the full window, that is not a failure + if len(rawData) == 0 { + return "", err + } + if err != nil && !errors.Is(err, io.EOF) { + return "", err + } - chunkStr, err = c.generateChunk((*rawData)[:n]) + // only check the filetype at the start of file + if totalLines == 0 && ShouldSkipFile(rawData) { + return "", fmt.Errorf("skipping file: %w", ErrUnsupportedFileType) } + + // generateChunk copies the data, so rawData is not used after the Discard below + chunkStr, err := c.generateChunk(rawData) if err != nil { return "", err } + + // consume only the bytes that went into the chunk + if _, err = reader.Discard(len(chunkStr)); err != nil { + return "", fmt.Errorf("failed to discard chunk bytes: %w", err) + } return chunkStr, nil } diff --git a/engine/chunk/chunk_test.go b/engine/chunk/chunk_test.go index aaa17e5a..372939c6 100644 --- a/engine/chunk/chunk_test.go +++ b/engine/chunk/chunk_test.go @@ -3,10 +3,12 @@ package chunk import ( "bufio" "bytes" - "github.com/stretchr/testify/require" + "errors" "io" "strings" "testing" + + "github.com/stretchr/testify/require" ) const ( @@ -26,15 +28,6 @@ func TestGetAndPutBuf(t *testing.T) { require.Equal(t, string(data), buf.String()) } -func TestGetAndPutPeekedBuf(t *testing.T) { - c := New() - window, ok := c.GetPeekedBuf() - defer c.PutPeekedBuf(window) - - require.True(t, ok) - require.Equal(t, defaultSize+defaultMaxPeekSize, len(*window)) -} - func TestGetSize(t *testing.T) { c := New() require.Equal(t, defaultSize, c.GetSize()) @@ -101,6 +94,60 @@ func TestReadChunk(t *testing.T) { } } +func TestReadChunkDoesNotDropBytes(t *testing.T) { + // Arrange + testCases := []struct { + name string + input string + }{ + { + name: "safe boundary found inside the peek window", + input: "abcdefghi\n\njklmnopqrstuvwxyz\n", + }, + { + name: "CRLF safe boundary inside the peek window", + input: "abcdefg\r\n\r\nhijklmnopqrstuvwxyz\r\n", + }, + { + name: "several safe boundaries", + input: "abcdefghi\n\njklmnopqr\n\nstuvwxyz\n\nabc", + }, + { + name: "no safe boundary", + input: "abcdefghijklmnopqrstuvwxyz0123456789", + }, + { + name: "file ends inside the peek window", + input: "abcdefghijkl", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + c := New(WithSize(chunkSize), WithMaxPeekSize(maxPeekSize), WithSmallFileThreshold(smallFileThreshold)) + reader := bufio.NewReaderSize(strings.NewReader(tc.input), chunkSize+maxPeekSize) + + // Act + var rebuilt strings.Builder + totalLines := 0 + for { + chunkStr, err := c.ReadChunk(reader, totalLines) + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + require.NotEmpty(t, chunkStr) + rebuilt.WriteString(chunkStr) + totalLines += strings.Count(chunkStr, "\n") + } + + // Assert: every byte of the input is scanned exactly once and the line count matches the file + require.Equal(t, tc.input, rebuilt.String()) + require.Equal(t, strings.Count(tc.input, "\n"), totalLines) + }) + } +} + func TestGenerateChunk(t *testing.T) { // Arrange testCases := []struct {