Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# and "Missing User Instruction" since 2ms container is stopped after scan

# Builder image
FROM checkmarx/go:1.27.3-r0-94581adffb0633@sha256:94581adffb0633c4cdde4b15edc8834c192204e9af0a5247f85e91294ceb2e82 AS builder
FROM checkmarx/go:1.27.4-r0-949d8d960bf5ce@sha256:949d8d960bf5ce70198fe2ddc9c00e6f09c2badcc799a8f6d46308eacda1ad63 AS builder

WORKDIR /app

Expand All @@ -20,7 +20,7 @@ COPY . .
RUN GOOS=linux GOARCH=amd64 go build -buildvcs=false -ldflags="-s -w" -a -o /app/2ms .

# Runtime image
FROM checkmarx/git:2.56.0-r3-f9cad2bd619390@sha256:f9cad2bd61939058ef50a139cb2d8fbd3cf01b435a0992029e3584e9c6f10f61
FROM checkmarx/git:2.56.1-r1-bf7f4c34ff4dd6@sha256:bf7f4c34ff4dd67bbd402dd5aaade8f5ace24ea2be6eb46c263e0ea982acf6e3

WORKDIR /app

Expand Down
61 changes: 22 additions & 39 deletions engine/chunk/chunk.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"bytes"
"errors"
"fmt"
"io"
"sync"
"unicode"

Expand Down Expand Up @@ -47,7 +48,6 @@ func WithSmallFileThreshold(smallFileThreshold int64) Option {
// Chunk holds two pools and sizing parameters needed for reading chunks of data with look-ahead
type Chunk struct {
bufPool *sync.Pool // *bytes.Buffer with cap Size + MaxPeekSize
peekedBufPool *sync.Pool // *[]byte slices of length Size + MaxPeekSize
size int // base chunk size
maxPeekSize int // max size of look-ahead bytes
smallFileThreshold int64 // files smaller than this skip chunking
Expand Down Expand Up @@ -77,13 +77,6 @@ func New(opts ...Option) *Chunk {
return bytes.NewBuffer(make([]byte, 0, c.size+c.maxPeekSize))
},
}
c.peekedBufPool = &sync.Pool{
New: func() interface{} {
// pre-allocate fixed-size block for loading chunks
b := make([]byte, c.size+c.maxPeekSize)
return &b
},
}
return c
}

Expand All @@ -103,23 +96,6 @@ func (c *Chunk) PutBuf(window *bytes.Buffer) {
c.bufPool.Put(window)
}

// GetPeekedBuf returns a fixed-size []byte from the pool
func (c *Chunk) GetPeekedBuf() (*[]byte, bool) {
b, ok := c.peekedBufPool.Get().(*[]byte)
if !ok {
return nil, false
}
// reslice to its full capacity so Read can fill it
*b = (*b)[:cap(*b)]
return b, ok
}

// PutPeekedBuf returns the fixed-size []byte to the pool
func (c *Chunk) PutPeekedBuf(b *[]byte) {
*b = (*b)[:0] // reset the slice to zero length
c.peekedBufPool.Put(b)
}

func (c *Chunk) GetSize() int {
return c.size
}
Expand All @@ -133,29 +109,36 @@ func (c *Chunk) GetFileThreshold() int64 {
}

// ReadChunk reads the next chunk of data from file
// Only the bytes that are part of the returned chunk are consumed from the reader
func (c *Chunk) ReadChunk(reader *bufio.Reader, totalLines int) (string, error) {
// borrow a []bytes from the pool and seed it with raw data from file (up to chunk size + peek size)
rawData, ok := c.GetPeekedBuf()
if !ok {
return "", fmt.Errorf("expected *bytes.Buffer, got %T", rawData)
}
defer c.PutPeekedBuf(rawData)
Comment thread
cx-jose-fernandes marked this conversation as resolved.
Resolved
n, err := reader.Read(*rawData)
// look at the raw data from file (up to chunk size + peek size) without consuming it
rawData, err := reader.Peek(c.size + c.maxPeekSize)

var chunkStr string
// "Callers should always process the n > 0 bytes returned before considering the error err."
// https://pkg.go.dev/io#Reader
if n > 0 {
// only check the filetype at the start of file
if totalLines == 0 && ShouldSkipFile((*rawData)[:n]) {
return "", fmt.Errorf("skipping file: %w", ErrUnsupportedFileType)
}
// Peek returns fewer bytes together with io.EOF when the file ends before the full window, that is not a failure
if len(rawData) == 0 {
return "", err
}
if err != nil && !errors.Is(err, io.EOF) {
return "", err
}

chunkStr, err = c.generateChunk((*rawData)[:n])
// only check the filetype at the start of file
if totalLines == 0 && ShouldSkipFile(rawData) {
return "", fmt.Errorf("skipping file: %w", ErrUnsupportedFileType)
}

// generateChunk copies the data, so rawData is not used after the Discard below
chunkStr, err := c.generateChunk(rawData)
if err != nil {
return "", err
}

// consume only the bytes that went into the chunk
if _, err = reader.Discard(len(chunkStr)); err != nil {
return "", fmt.Errorf("failed to discard chunk bytes: %w", err)
}
return chunkStr, nil
}

Expand Down
67 changes: 57 additions & 10 deletions engine/chunk/chunk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@ package chunk
import (
"bufio"
"bytes"
"github.com/stretchr/testify/require"
"errors"
"io"
"strings"
"testing"

"github.com/stretchr/testify/require"
)

const (
Expand All @@ -26,15 +28,6 @@ func TestGetAndPutBuf(t *testing.T) {
require.Equal(t, string(data), buf.String())
}

func TestGetAndPutPeekedBuf(t *testing.T) {
c := New()
window, ok := c.GetPeekedBuf()
defer c.PutPeekedBuf(window)

require.True(t, ok)
require.Equal(t, defaultSize+defaultMaxPeekSize, len(*window))
}

func TestGetSize(t *testing.T) {
c := New()
require.Equal(t, defaultSize, c.GetSize())
Expand Down Expand Up @@ -101,6 +94,60 @@ func TestReadChunk(t *testing.T) {
}
}

func TestReadChunkDoesNotDropBytes(t *testing.T) {
// Arrange
testCases := []struct {
name string
input string
}{
{
name: "safe boundary found inside the peek window",
input: "abcdefghi\n\njklmnopqrstuvwxyz\n",
},
{
name: "CRLF safe boundary inside the peek window",
input: "abcdefg\r\n\r\nhijklmnopqrstuvwxyz\r\n",
},
{
name: "several safe boundaries",
input: "abcdefghi\n\njklmnopqr\n\nstuvwxyz\n\nabc",
},
{
name: "no safe boundary",
input: "abcdefghijklmnopqrstuvwxyz0123456789",
},
{
name: "file ends inside the peek window",
input: "abcdefghijkl",
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
c := New(WithSize(chunkSize), WithMaxPeekSize(maxPeekSize), WithSmallFileThreshold(smallFileThreshold))
reader := bufio.NewReaderSize(strings.NewReader(tc.input), chunkSize+maxPeekSize)

// Act
var rebuilt strings.Builder
totalLines := 0
for {
chunkStr, err := c.ReadChunk(reader, totalLines)
if errors.Is(err, io.EOF) {
break
}
require.NoError(t, err)
require.NotEmpty(t, chunkStr)
rebuilt.WriteString(chunkStr)
totalLines += strings.Count(chunkStr, "\n")
}

// Assert: every byte of the input is scanned exactly once and the line count matches the file
require.Equal(t, tc.input, rebuilt.String())
require.Equal(t, strings.Count(tc.input, "\n"), totalLines)
})
}
}

func TestGenerateChunk(t *testing.T) {
// Arrange
testCases := []struct {
Expand Down
Loading