diff --git a/src/go.mod b/src/go.mod index 17c1a601..c7d8cbdf 100644 --- a/src/go.mod +++ b/src/go.mod @@ -13,7 +13,7 @@ require ( github.com/jackc/pgx/v5 v5.10.0 github.com/joho/godotenv v1.5.1 github.com/looplab/fsm v1.0.3 - github.com/minio/minio-go/v7 v7.2.1 + github.com/minio/minio-go/v7 v7.3.0 github.com/pquerna/otp v1.5.0 github.com/pressly/goose/v3 v3.27.3 github.com/redis/go-redis/v9 v9.21.0 @@ -21,10 +21,10 @@ require ( github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef github.com/valyala/fastjson v1.6.10 - golang.org/x/crypto v0.54.0 + golang.org/x/crypto v0.55.0 golang.org/x/image v0.44.0 golang.org/x/sync v0.22.0 - golang.org/x/text v0.40.0 + golang.org/x/text v0.41.0 ) require ( @@ -43,7 +43,7 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/klauspost/compress v1.19.1 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/kr/text v0.2.0 // indirect @@ -63,8 +63,8 @@ require ( github.com/zeebo/xxh3 v1.1.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.57.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.58.0 // indirect golang.org/x/sys v0.47.0 // indirect gopkg.in/ini.v1 v1.67.3 // indirect ) diff --git a/src/go.sum b/src/go.sum index e463bac9..a112aa2b 100644 --- a/src/go.sum +++ b/src/go.sum @@ -69,6 +69,8 @@ github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2o github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= @@ -94,6 +96,8 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.2.1 h1:PfBfwvKB/MmqyN8Vb1G9voWisaM9OrLv+WwOvMwS9Dw= github.com/minio/minio-go/v7 v7.2.1/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0= +github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU= +github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= @@ -164,16 +168,22 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I= golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= @@ -182,6 +192,8 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/src/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/src/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index 2d6ef64b..7035d656 100644 --- a/src/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/src/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -1,222 +1,49 @@ //go:build amd64 && !appengine && !noasm && gc -// This file contains the specialisation of Decoder.Decompress4X -// and Decoder.Decompress1X that use an asm implementation of thir main loops. +// amd64 stubs and dispatch for the asm loops used by decompress_asm.go. package huff0 import ( - "errors" - "fmt" - "github.com/klauspost/compress/internal/cpuinfo" ) -// decompress4x_main_loop_x86 is an x86 assembler implementation +// decompress4x_main_loop_amd64 is an x86 assembler implementation // of Decompress4X when tablelog > 8. // //go:noescape func decompress4x_main_loop_amd64(ctx *decompress4xContext) -// decompress4x_8b_loop_x86 is an x86 assembler implementation +// decompress4x_8b_main_loop_amd64 is an x86 assembler implementation // of Decompress4X when tablelog <= 8 which decodes 4 entries // per loop. // //go:noescape func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) -// fallback8BitSize is the size where using Go version is faster. -const fallback8BitSize = 800 - -type decompress4xContext struct { - pbr *[4]bitReaderShifted - peekBits uint8 - out *byte - dstEvery int - tbl *dEntrySingle - decoded int - limit *byte -} - -// Decompress4X will decompress a 4X encoded stream. -// The length of the supplied input must match the end of a block exactly. -// The *capacity* of the dst slice must match the destination size of -// the uncompressed data exactly. -func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - if len(src) < 6+(4*1) { - return nil, errors.New("input too small") - } - - use8BitTables := d.actualTableLog <= 8 - if cap(dst) < fallback8BitSize && use8BitTables { - return d.decompress4X8bit(dst, src) - } - - var br [4]bitReaderShifted - // Decode "jump table" - start := 6 - for i := range 3 { - length := int(src[i*2]) | (int(src[i*2+1]) << 8) - if start+length >= len(src) { - return nil, errors.New("truncated input (or invalid offset)") - } - err := br[i].init(src[start : start+length]) - if err != nil { - return nil, err - } - start += length - } - err := br[3].init(src[start:]) - if err != nil { - return nil, err - } - - // destination, offset to match first output - dstSize := cap(dst) - dst = dst[:dstSize] - out := dst - dstEvery := (dstSize + 3) / 4 - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - single := d.dt.single[:tlSize] - - var decoded int - - if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { - ctx := decompress4xContext{ - pbr: &br, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - out: &out[0], - dstEvery: dstEvery, - tbl: &single[0], - limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. - } - if use8BitTables { - decompress4x_8b_main_loop_amd64(&ctx) - } else { - decompress4x_main_loop_amd64(&ctx) - } - - decoded = ctx.decoded - out = out[decoded/4:] - } - - // Decode remaining. - remainBytes := dstEvery - (decoded / 4) - for i := range br { - offset := dstEvery * i - endsAt := min(offset+remainBytes, len(out)) - br := &br[i] - bitsLeft := br.remaining() - for bitsLeft > 0 { - br.fill() - if offset >= endsAt { - return nil, errors.New("corruption detected: stream overrun 4") - } - - // Read value and increment offset. - val := br.peekBitsFast(d.actualTableLog) - v := single[val&tlMask].entry - nBits := uint8(v) - br.advance(nBits) - bitsLeft -= uint(nBits) - out[offset] = uint8(v >> 8) - offset++ - } - if offset != endsAt { - return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) - } - decoded += offset - dstEvery*i - err = br.close() - if err != nil { - return nil, err - } - } - if dstSize != decoded { - return nil, errors.New("corruption detected: short output block") - } - return dst, nil -} - -// decompress4x_main_loop_x86 is an x86 assembler implementation +// decompress1x_main_loop_amd64 is an x86 assembler implementation // of Decompress1X when tablelog > 8. // //go:noescape func decompress1x_main_loop_amd64(ctx *decompress1xContext) -// decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation +// decompress1x_main_loop_bmi2 is an x86 with BMI2 assembler implementation // of Decompress1X when tablelog > 8. // //go:noescape func decompress1x_main_loop_bmi2(ctx *decompress1xContext) -type decompress1xContext struct { - pbr *bitReaderShifted - peekBits uint8 - out *byte - outCap int - tbl *dEntrySingle - decoded int +func decompress4x_main_loop_asm(ctx *decompress4xContext) { + decompress4x_main_loop_amd64(ctx) } -// Error reported by asm implementations -const error_max_decoded_size_exeeded = -1 - -// Decompress1X will decompress a 1X encoded stream. -// The cap of the output buffer will be the maximum decompressed size. -// The length of the supplied input must match the end of a block exactly. -func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - var br bitReaderShifted - err := br.init(src) - if err != nil { - return dst, err - } - maxDecodedSize := cap(dst) - dst = dst[:maxDecodedSize] - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - - if maxDecodedSize >= 4 { - ctx := decompress1xContext{ - pbr: &br, - out: &dst[0], - outCap: maxDecodedSize, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - tbl: &d.dt.single[0], - } - - if cpuinfo.HasBMI2() { - decompress1x_main_loop_bmi2(&ctx) - } else { - decompress1x_main_loop_amd64(&ctx) - } - if ctx.decoded == error_max_decoded_size_exeeded { - return nil, ErrMaxDecodedSizeExceeded - } - - dst = dst[:ctx.decoded] - } +func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { + decompress4x_8b_main_loop_amd64(ctx) +} - // br < 8, so uint8 is fine - bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead - for bitsLeft > 0 { - br.fill() - if len(dst) >= maxDecodedSize { - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] - nBits := uint8(v.entry) - br.advance(nBits) - bitsLeft -= nBits - dst = append(dst, uint8(v.entry>>8)) +func decompress1x_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_main_loop_bmi2(ctx) + } else { + decompress1x_main_loop_amd64(ctx) } - return dst, br.close() } diff --git a/src/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/src/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s index c4c7ab2d..c5d4a710 100644 --- a/src/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/src/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -1,6 +1,6 @@ -// Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. -//go:build amd64 && !appengine && !noasm && gc +//go:build !appengine && !noasm && gc // func decompress4x_main_loop_amd64(ctx *decompress4xContext) TEXT ·decompress4x_main_loop_amd64(SB), $0-8 diff --git a/src/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go b/src/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go new file mode 100644 index 00000000..8ba3c810 --- /dev/null +++ b/src/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go @@ -0,0 +1,37 @@ +//go:build arm64 && !appengine && !noasm && gc + +// arm64 stubs and dispatch for the asm loops used by decompress_asm.go. +// The asm (decompress_arm64.s) is generated by the avo arm64 lowering +// printer from the same source as the amd64 asm; see _generate/gen.go. +package huff0 + +// decompress4x_main_loop_arm64 is an arm64 assembler implementation +// of Decompress4X when tablelog > 8. +// +//go:noescape +func decompress4x_main_loop_arm64(ctx *decompress4xContext) + +// decompress4x_8b_main_loop_arm64 is an arm64 assembler implementation +// of Decompress4X when tablelog <= 8 which decodes 4 entries +// per loop. +// +//go:noescape +func decompress4x_8b_main_loop_arm64(ctx *decompress4xContext) + +// decompress1x_main_loop_arm64 is an arm64 assembler implementation +// of Decompress1X when tablelog > 8. +// +//go:noescape +func decompress1x_main_loop_arm64(ctx *decompress1xContext) + +func decompress4x_main_loop_asm(ctx *decompress4xContext) { + decompress4x_main_loop_arm64(ctx) +} + +func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { + decompress4x_8b_main_loop_arm64(ctx) +} + +func decompress1x_main_loop_asm(ctx *decompress1xContext) { + decompress1x_main_loop_arm64(ctx) +} diff --git a/src/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s b/src/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s new file mode 100644 index 00000000..e1601272 --- /dev/null +++ b/src/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s @@ -0,0 +1,851 @@ +// Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. +// EXPERIMENTAL arm64 output lowered from an amd64 avo program. + +//go:build arm64 && !appengine && !noasm && gc + +// func decompress4x_main_loop_amd64(ctx *decompress4xContext) +TEXT ·decompress4x_main_loop_arm64(SB), $0-8 + // Preload values + MOVD ctx+0(FP), R0 + MOVBU 8(R0), R6 + MOVD 16(R0), R3 + MOVD 48(R0), R5 + MOVD 24(R0), R7 + MOVD 32(R0), R8 + MOVD (R0), R9 + + // Main loop +main_loop: + MOVD $0, R2 + CMP R5, R3 + CSET GE, R16 + BFI $0, R16, $8, R2 + + // br0.fillFast32() + MOVD 32(R9), R10 + MOVBU 40(R9), R11 + CMP $0x20, R11 + BLS skip_fill0 + MOVD 24(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD (R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 24(R9) + ORR R12, R10, R10 + + // exhausted += (br0.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill0: + // val0 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br0.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + MOVH R0, (R3) + + // update the bitreader structure + MOVD R10, 32(R9) + MOVB R11, 40(R9) + + // br1.fillFast32() + MOVD 80(R9), R10 + MOVBU 88(R9), R11 + CMP $0x20, R11 + BLS skip_fill1 + MOVD 72(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD 48(R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 72(R9) + ORR R12, R10, R10 + + // exhausted += (br1.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill1: + // val0 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br1.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + ADD R7, R3, R15 + MOVH R0, (R15) + + // update the bitreader structure + MOVD R10, 80(R9) + MOVB R11, 88(R9) + + // br2.fillFast32() + MOVD 128(R9), R10 + MOVBU 136(R9), R11 + CMP $0x20, R11 + BLS skip_fill2 + MOVD 120(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD 96(R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 120(R9) + ORR R12, R10, R10 + + // exhausted += (br2.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill2: + // val0 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br2.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + ADD R7<<1, R3, R15 + MOVH R0, (R15) + + // update the bitreader structure + MOVD R10, 128(R9) + MOVB R11, 136(R9) + + // br3.fillFast32() + MOVD 176(R9), R10 + MOVBU 184(R9), R11 + CMP $0x20, R11 + BLS skip_fill3 + MOVD 168(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD 144(R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 168(R9) + ORR R12, R10, R10 + + // exhausted += (br3.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill3: + // val0 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br3.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + ADD R7<<1, R7, R1 + ADD R1, R3, R15 + MOVH R0, (R15) + + // update the bitreader structure + MOVD R10, 176(R9) + MOVB R11, 184(R9) + ADD $0x02, R3, R3 + AND $0xff, R2, R15 + AND $0xff, R2, R16 + TST R16, R15 + BEQ main_loop + MOVD ctx+0(FP), R0 + MOVD 16(R0), R16 + SUB R16, R3, R3 + LSL $0x02, R3, R3 + MOVD R3, 40(R0) + RET + +// func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) +TEXT ·decompress4x_8b_main_loop_arm64(SB), $0-8 + // Preload values + MOVD ctx+0(FP), R1 + MOVBU 8(R1), R6 + MOVD 16(R1), R3 + MOVD 48(R1), R5 + MOVD 24(R1), R7 + MOVD 32(R1), R8 + MOVD (R1), R9 + + // Main loop +main_loop: + MOVD $0, R2 + CMP R5, R3 + CSET GE, R16 + BFI $0, R16, $8, R2 + + // br0.fillFast32() + MOVD 32(R9), R10 + MOVBU 40(R9), R11 + CMP $0x20, R11 + BLS skip_fill0 + MOVD 24(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD (R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 24(R9) + ORR R13, R10, R10 + + // exhausted += (br0.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill0: + // val0 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + MOVW R0, (R3) + + // update the bitreader structure + MOVD R10, 32(R9) + MOVB R11, 40(R9) + + // br1.fillFast32() + MOVD 80(R9), R10 + MOVBU 88(R9), R11 + CMP $0x20, R11 + BLS skip_fill1 + MOVD 72(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD 48(R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 72(R9) + ORR R13, R10, R10 + + // exhausted += (br1.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill1: + // val0 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + ADD R7, R3, R15 + MOVW R0, (R15) + + // update the bitreader structure + MOVD R10, 80(R9) + MOVB R11, 88(R9) + + // br2.fillFast32() + MOVD 128(R9), R10 + MOVBU 136(R9), R11 + CMP $0x20, R11 + BLS skip_fill2 + MOVD 120(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD 96(R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 120(R9) + ORR R13, R10, R10 + + // exhausted += (br2.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill2: + // val0 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + ADD R7<<1, R3, R15 + MOVW R0, (R15) + + // update the bitreader structure + MOVD R10, 128(R9) + MOVB R11, 136(R9) + + // br3.fillFast32() + MOVD 176(R9), R10 + MOVBU 184(R9), R11 + CMP $0x20, R11 + BLS skip_fill3 + MOVD 168(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD 144(R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 168(R9) + ORR R13, R10, R10 + + // exhausted += (br3.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill3: + // val0 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + ADD R7<<1, R7, R1 + ADD R1, R3, R15 + MOVW R0, (R15) + + // update the bitreader structure + MOVD R10, 176(R9) + MOVB R11, 184(R9) + ADD $0x04, R3, R3 + AND $0xff, R2, R15 + AND $0xff, R2, R16 + TST R16, R15 + BEQ main_loop + MOVD ctx+0(FP), R0 + MOVD 16(R0), R16 + SUB R16, R3, R3 + LSL $0x02, R3, R3 + MOVD R3, 40(R0) + RET + +// func decompress1x_main_loop_amd64(ctx *decompress1xContext) +TEXT ·decompress1x_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R1 + MOVD 16(R1), R2 + MOVD 24(R1), R3 + CMP $0x04, R3 + BLO error_max_decoded_size_exceeded + ADD R3, R2, R3 + MOVD (R1), R5 + MOVD (R5), R7 + MOVD 24(R5), R8 + MOVD 32(R5), R9 + MOVBU 40(R5), R10 + MOVD 32(R1), R5 + MOVBU 8(R1), R6 + JMP loop_condition + +main_loop: + // Check if we have room for 4 bytes in the output buffer + ADD $4, R2, R1 + CMP R3, R1 + BGE error_max_decoded_size_exceeded + + // Decode 4 values + CMP $0x20, R10 + BLT bitReader_fillFast_1_end + SUB $0x20, R10, R10 + SUB $0x04, R8, R8 + ADD R8, R7, R15 + MOVWU (R15), R11 + MOVD R10, R1 + LSL R1, R11, R11 + ORR R11, R9, R9 + +bitReader_fillFast_1_end: + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + REVW R0, R0 + CMP $0x20, R10 + BLT bitReader_fillFast_2_end + SUB $0x20, R10, R10 + SUB $0x04, R8, R8 + ADD R8, R7, R15 + MOVWU (R15), R11 + MOVD R10, R1 + LSL R1, R11, R11 + ORR R11, R9, R9 + +bitReader_fillFast_2_end: + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + REVW R0, R0 + + // Store the decoded values + MOVW R0, (R2) + ADD $0x04, R2, R2 + +loop_condition: + CMP $0x08, R8 + BGE main_loop + + // Update ctx structure + MOVD ctx+0(FP), R0 + MOVD 16(R0), R16 + SUB R16, R2, R2 + MOVD R2, 40(R0) + MOVD (R0), R0 + MOVD R8, 24(R0) + MOVD R9, 32(R0) + MOVB R10, 40(R0) + RET + + // Report error +error_max_decoded_size_exceeded: + MOVD ctx+0(FP), R0 + MOVD $-1, R1 + MOVD R1, 40(R0) + RET + +// skipped decompress1x_main_loop_bmi2 (generic twin preferred on arm64) diff --git a/src/vendor/github.com/klauspost/compress/huff0/decompress_asm.go b/src/vendor/github.com/klauspost/compress/huff0/decompress_asm.go new file mode 100644 index 00000000..4854dd43 --- /dev/null +++ b/src/vendor/github.com/klauspost/compress/huff0/decompress_asm.go @@ -0,0 +1,193 @@ +//go:build (amd64 || arm64) && !appengine && !noasm && gc + +// This file contains the specialisation of Decoder.Decompress4X +// and Decoder.Decompress1X that use an asm implementation of their main loops. +// The asm function stubs and any per-arch dispatch live in decompress_amd64.go +// and decompress_arm64.go. +package huff0 + +import ( + "errors" + "fmt" +) + +// fallback8BitSize is the size where using Go version is faster. +const fallback8BitSize = 800 + +type decompress4xContext struct { + pbr *[4]bitReaderShifted + peekBits uint8 + out *byte + dstEvery int + tbl *dEntrySingle + decoded int + limit *byte +} + +// Decompress4X will decompress a 4X encoded stream. +// The length of the supplied input must match the end of a block exactly. +// The *capacity* of the dst slice must match the destination size of +// the uncompressed data exactly. +func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + if len(src) < 6+(4*1) { + return nil, errors.New("input too small") + } + + use8BitTables := d.actualTableLog <= 8 + if cap(dst) < fallback8BitSize && use8BitTables { + return d.decompress4X8bit(dst, src) + } + + var br [4]bitReaderShifted + // Decode "jump table" + start := 6 + for i := range 3 { + length := int(src[i*2]) | (int(src[i*2+1]) << 8) + if start+length >= len(src) { + return nil, errors.New("truncated input (or invalid offset)") + } + err := br[i].init(src[start : start+length]) + if err != nil { + return nil, err + } + start += length + } + err := br[3].init(src[start:]) + if err != nil { + return nil, err + } + + // destination, offset to match first output + dstSize := cap(dst) + dst = dst[:dstSize] + out := dst + dstEvery := (dstSize + 3) / 4 + + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + single := d.dt.single[:tlSize] + + var decoded int + + if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { + ctx := decompress4xContext{ + pbr: &br, + peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() + out: &out[0], + dstEvery: dstEvery, + tbl: &single[0], + limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. + } + if use8BitTables { + decompress4x_8b_main_loop_asm(&ctx) + } else { + decompress4x_main_loop_asm(&ctx) + } + + decoded = ctx.decoded + out = out[decoded/4:] + } + + // Decode remaining. + remainBytes := dstEvery - (decoded / 4) + for i := range br { + offset := dstEvery * i + endsAt := min(offset+remainBytes, len(out)) + br := &br[i] + bitsLeft := br.remaining() + for bitsLeft > 0 { + br.fill() + if offset >= endsAt { + return nil, errors.New("corruption detected: stream overrun 4") + } + + // Read value and increment offset. + val := br.peekBitsFast(d.actualTableLog) + v := single[val&tlMask].entry + nBits := uint8(v) + br.advance(nBits) + bitsLeft -= uint(nBits) + out[offset] = uint8(v >> 8) + offset++ + } + if offset != endsAt { + return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) + } + decoded += offset - dstEvery*i + err = br.close() + if err != nil { + return nil, err + } + } + if dstSize != decoded { + return nil, errors.New("corruption detected: short output block") + } + return dst, nil +} + +type decompress1xContext struct { + pbr *bitReaderShifted + peekBits uint8 + out *byte + outCap int + tbl *dEntrySingle + decoded int +} + +// Error reported by asm implementations +const error_max_decoded_size_exeeded = -1 + +// Decompress1X will decompress a 1X encoded stream. +// The cap of the output buffer will be the maximum decompressed size. +// The length of the supplied input must match the end of a block exactly. +func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + var br bitReaderShifted + err := br.init(src) + if err != nil { + return dst, err + } + maxDecodedSize := cap(dst) + dst = dst[:maxDecodedSize] + + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + + if maxDecodedSize >= 4 { + ctx := decompress1xContext{ + pbr: &br, + out: &dst[0], + outCap: maxDecodedSize, + peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() + tbl: &d.dt.single[0], + } + + decompress1x_main_loop_asm(&ctx) + if ctx.decoded == error_max_decoded_size_exeeded { + return nil, ErrMaxDecodedSizeExceeded + } + + dst = dst[:ctx.decoded] + } + + // br < 8, so uint8 is fine + bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead + for bitsLeft > 0 { + br.fill() + if len(dst) >= maxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] + nBits := uint8(v.entry) + br.advance(nBits) + bitsLeft -= nBits + dst = append(dst, uint8(v.entry>>8)) + } + return dst, br.close() +} diff --git a/src/vendor/github.com/klauspost/compress/huff0/decompress_generic.go b/src/vendor/github.com/klauspost/compress/huff0/decompress_generic.go index 61039232..e1bca5c6 100644 --- a/src/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ b/src/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm // This file contains a generic implementation of Decoder.Decompress4X. package huff0 diff --git a/src/vendor/github.com/klauspost/compress/zstd/blockdec.go b/src/vendor/github.com/klauspost/compress/zstd/blockdec.go index 2329e996..51f9da03 100644 --- a/src/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/src/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -400,8 +400,9 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err } } var err error - // Use our out buffer. - huff.MaxDecodedSize = litRegenSize + // Decoder.Decompress* uses cap(dst) for the size limit. Do not write + // MaxDecodedSize on hist.huffTree: with a trained dictionary that + // pointer aliases the shared dict.litEnc and concurrent DecodeAll races. if fourStreams { literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) } else { @@ -673,10 +674,6 @@ func (b *blockDec) executeSequences(hist *history) error { hbytes := hist.b if len(hbytes) > hist.windowSize { hbytes = hbytes[len(hbytes)-hist.windowSize:] - // We do not need history anymore. - if hist.dict != nil { - hist.dict.content = nil - } } hist.decoders.windowSize = hist.windowSize hist.decoders.out = b.dst[:0] diff --git a/src/vendor/github.com/klauspost/compress/zstd/dict.go b/src/vendor/github.com/klauspost/compress/zstd/dict.go index 4f1c4938..67f40e79 100644 --- a/src/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/src/vendor/github.com/klauspost/compress/zstd/dict.go @@ -296,40 +296,81 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if offset > 3 { newOffsets[offset-3]++ } else { - newOffsets[uint32(o.Offsets[offset-1])]++ + // Repeat codes reference the training Offsets. Skip unset + // (zero) entries so they are not ranked as real offsets. + prev := o.Offsets[offset-1] + if prev > 0 { + newOffsets[uint32(prev)]++ + } } } } // Find most used offsets. var sortedOffsets []uint32 for k := range newOffsets { + if k == 0 { + continue + } sortedOffsets = append(sortedOffsets, k) } sort.Slice(sortedOffsets, func(i, j int) bool { a, b := sortedOffsets[i], sortedOffsets[j] - if a == b { + ca, cb := newOffsets[a], newOffsets[b] + if ca == cb { // Prefer the longer offset - return sortedOffsets[i] > sortedOffsets[j] + return a > b } - return newOffsets[sortedOffsets[i]] > newOffsets[sortedOffsets[j]] + return ca > cb }) - if len(sortedOffsets) > 3 { - if debug { - print("Offsets:") - for i, v := range sortedOffsets { - if i > 20 { - break - } - printf("[%d: %d],", v, newOffsets[v]) + if debug { + print("Offsets:") + for i, v := range sortedOffsets { + if i > 20 { + break } - println("") + printf("[%d: %d],", v, newOffsets[v]) + } + println("") + } + // Dictionary recent-offsets must be three positive values within the + // history. Ranked matches may be fewer (or empty when only unset + // repeat codes were seen), so fill remaining slots with defaults. + used := make(map[int]bool, 3) + var finalOffsets [3]int + nOff := 0 + for _, v := range sortedOffsets { + iv := int(v) + if iv <= 0 || iv > len(hist) || used[iv] { + continue + } + finalOffsets[nOff] = iv + used[iv] = true + nOff++ + if nOff == 3 { + break } - - sortedOffsets = sortedOffsets[:3] } - for i, v := range sortedOffsets { - o.Offsets[i] = int(v) + for _, def := range []int{1, 4, 8} { + if nOff == 3 { + break + } + if def <= len(hist) && !used[def] { + finalOffsets[nOff] = def + used[def] = true + nOff++ + } + } + for def := 1; nOff < 3 && def <= len(hist); def++ { + if !used[def] { + finalOffsets[nOff] = def + used[def] = true + nOff++ + } + } + if nOff < 3 { + return nil, fmt.Errorf("could not determine 3 valid dictionary offsets (history size %d)", len(hist)) } + o.Offsets = finalOffsets if debug { println("New repeat offsets", o.Offsets) } @@ -337,6 +378,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if nUsed == 0 || seqs == 0 { return nil, fmt.Errorf("%d blocks, %d sequences found", nUsed, seqs) } + if litTotal == 0 { + return nil, errors.New("0 literals found") + } if debug { println("Sequences:", seqs, "Blocks:", nUsed, "Literals:", litTotal) } @@ -517,11 +561,10 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[1]))) out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[2]))) out.Write(hist) + if _, err := loadDict(out.Bytes()); err != nil { + return nil, fmt.Errorf("built dictionary failed validation: %w", err) + } if debug { - _, err := loadDict(out.Bytes()) - if err != nil { - panic(err) - } i, err := InspectDictionary(out.Bytes()) if err != nil { panic(err) diff --git a/src/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/src/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 3fc381c7..4d3188ff 100644 --- a/src/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/src/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -2081,7 +2081,7 @@ sequenceDecs_decodeSync_amd64_match_len_ofs_ok: MOVQ 16(SP), R13 // Check if we have enough space in s.out - LEAQ (AX)(R13*1), R14 + LEAQ 16(AX)(R13*1), R14 ADDQ R10, R14 CMPQ R14, 32(SP) JA error_not_enough_space @@ -2601,7 +2601,7 @@ sequenceDecs_decodeSync_bmi2_match_len_ofs_ok: MOVQ 16(SP), R13 // Check if we have enough space in s.out - LEAQ (CX)(R13*1), R14 + LEAQ 16(CX)(R13*1), R14 ADDQ R9, R14 CMPQ R14, 32(SP) JA error_not_enough_space diff --git a/src/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s b/src/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s index a468e5fc..6f54ef21 100644 --- a/src/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s +++ b/src/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s @@ -12,7 +12,7 @@ TEXT ·sequenceDecs_decode_arm64(SB), $8-32 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 @@ -24,7 +24,7 @@ TEXT ·sequenceDecs_decode_arm64(SB), $8-32 MOVD 160(R0), R12 sequenceDecs_decode_amd64_main_loop: - MOVD (RSP), R13 + MOVD 8(RSP), R13 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 @@ -60,7 +60,8 @@ sequenceDecs_decode_amd64_fill_end: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_amd64_of_update_zero @@ -81,7 +82,8 @@ sequenceDecs_decode_amd64_of_update_zero: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_amd64_ml_update_zero @@ -131,7 +133,8 @@ sequenceDecs_decode_amd64_fill_2_end: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_amd64_ll_update_zero @@ -148,7 +151,7 @@ sequenceDecs_decode_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates - MOVD R13, (RSP) + MOVD R13, 8(RSP) MOVD R8, R0 LSR $0x08, R0, R0 MOVBU R0, R0 @@ -166,7 +169,7 @@ sequenceDecs_decode_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -187,7 +190,7 @@ sequenceDecs_decode_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -208,7 +211,7 @@ sequenceDecs_decode_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -360,7 +363,7 @@ TEXT ·sequenceDecs_decode_56_arm64(SB), $8-32 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 @@ -372,7 +375,7 @@ TEXT ·sequenceDecs_decode_56_arm64(SB), $8-32 MOVD 160(R0), R12 sequenceDecs_decode_56_amd64_main_loop: - MOVD (RSP), R13 + MOVD 8(RSP), R13 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 @@ -408,7 +411,8 @@ sequenceDecs_decode_56_amd64_fill_end: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_56_amd64_of_update_zero @@ -429,7 +433,8 @@ sequenceDecs_decode_56_amd64_of_update_zero: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_56_amd64_ml_update_zero @@ -450,7 +455,8 @@ sequenceDecs_decode_56_amd64_ml_update_zero: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_56_amd64_ll_update_zero @@ -467,7 +473,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates - MOVD R13, (RSP) + MOVD R13, 8(RSP) MOVD R8, R0 LSR $0x08, R0, R0 MOVBU R0, R0 @@ -485,7 +491,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -506,7 +512,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -527,7 +533,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -766,13 +772,14 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R10 - MOVB 2(R13), R11 - MOVH R10, (R3) - MOVB R11, 2(R3) - ADD R12, R13, R13 - ADD R12, R3, R3 - JMP copy_4_end + MOVHU (R13), R10 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R11 + MOVH R10, (R3) + MOVB R11, 2(R3) + ADD R12, R13, R13 + ADD R12, R3, R3 + JMP copy_4_end copy_4_move_4through7: MOVWU (R13), R10 @@ -834,24 +841,27 @@ copy_5_small: JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R10, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R3) - ADD R10, R3, R15 - MOVB R4, -1(R15) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R10, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R3) + ADD R10, R3, R15 + MOVB R4, -1(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R3) - MOVB R4, 2(R3) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R3) + MOVB R4, 2(R3) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 @@ -906,12 +916,13 @@ copy_overlapping_match: ADD R12, R6, R6 copy_slow_3: - MOVB (R10), R11 - MOVB R11, (R3) - ADD $1, R10, R10 - ADD $1, R3, R3 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R10), R16 + BFI $0, R16, $8, R11 + MOVB R11, (R3) + ADD $1, R10, R10 + ADD $1, R3, R3 + SUBS $1, R12, R12 + BNE copy_slow_3 handle_loop: ADD $0x18, R0, R0 @@ -1016,24 +1027,27 @@ copy_1_small: JMP copy_1_move_8through16 copy_1_move_1or2: - MOVB (R5), R13 - ADD R10, R5, R15 - MOVB -1(R15), R14 - MOVB R13, (R3) - ADD R10, R3, R15 - MOVB R14, -1(R15) - ADD R10, R5, R5 - ADD R10, R3, R3 - JMP copy_1_end + MOVBU (R5), R16 + BFI $0, R16, $8, R13 + ADD R10, R5, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R14 + MOVB R13, (R3) + ADD R10, R3, R15 + MOVB R14, -1(R15) + ADD R10, R5, R5 + ADD R10, R3, R3 + JMP copy_1_end copy_1_move_3: - MOVH (R5), R13 - MOVB 2(R5), R14 - MOVH R13, (R3) - MOVB R14, 2(R3) - ADD R10, R5, R5 - ADD R10, R3, R3 - JMP copy_1_end + MOVHU (R5), R13 + MOVBU 2(R5), R16 + BFI $0, R16, $8, R14 + MOVH R13, (R3) + MOVB R14, 2(R3) + ADD R10, R5, R5 + ADD R10, R3, R3 + JMP copy_1_end copy_1_move_4through7: MOVWU (R5), R13 @@ -1104,13 +1118,14 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R10 - MOVB 2(R13), R11 - MOVH R10, (R3) - MOVB R11, 2(R3) - ADD R12, R13, R13 - ADD R12, R3, R3 - JMP copy_4_end + MOVHU (R13), R10 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R11 + MOVH R10, (R3) + MOVB R11, 2(R3) + ADD R12, R13, R13 + ADD R12, R3, R3 + JMP copy_4_end copy_4_move_4through7: MOVWU (R13), R10 @@ -1172,24 +1187,27 @@ copy_5_small: JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R10, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R3) - ADD R10, R3, R15 - MOVB R4, -1(R15) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R10, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R3) + ADD R10, R3, R15 + MOVB R4, -1(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R3) - MOVB R4, 2(R3) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R3) + MOVB R4, 2(R3) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 @@ -1257,24 +1275,27 @@ copy_2_small: JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (R10), R11 - ADD R12, R10, R15 - MOVB -1(R15), R13 - MOVB R11, (R3) - ADD R12, R3, R15 - MOVB R13, -1(R15) - ADD R12, R10, R10 - ADD R12, R3, R3 - JMP copy_2_end + MOVBU (R10), R16 + BFI $0, R16, $8, R11 + ADD R12, R10, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R13 + MOVB R11, (R3) + ADD R12, R3, R15 + MOVB R13, -1(R15) + ADD R12, R10, R10 + ADD R12, R3, R3 + JMP copy_2_end copy_2_move_3: - MOVH (R10), R11 - MOVB 2(R10), R13 - MOVH R11, (R3) - MOVB R13, 2(R3) - ADD R12, R10, R10 - ADD R12, R3, R3 - JMP copy_2_end + MOVHU (R10), R11 + MOVBU 2(R10), R16 + BFI $0, R16, $8, R13 + MOVH R11, (R3) + MOVB R13, 2(R3) + ADD R12, R10, R10 + ADD R12, R3, R3 + JMP copy_2_end copy_2_move_4through7: MOVWU (R10), R11 @@ -1305,12 +1326,13 @@ copy_overlapping_match: ADD R12, R6, R6 copy_slow_3: - MOVB (R10), R11 - MOVB R11, (R3) - ADD $1, R10, R10 - ADD $1, R3, R3 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R10), R16 + BFI $0, R16, $8, R11 + MOVB R11, (R3) + ADD $1, R10, R10 + ADD $1, R3, R3 + SUBS $1, R12, R12 + BNE copy_slow_3 handle_loop: ADD $0x18, R0, R0 @@ -1361,41 +1383,41 @@ TEXT ·sequenceDecs_decodeSync_arm64(SB), $64-32 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 MOVD $0, R1 - MOVD R1, 8(RSP) MOVD R1, 16(RSP) MOVD R1, 24(RSP) + MOVD R1, 32(RSP) MOVD 112(R0), R9 MOVD 128(R0), R1 - MOVD R1, 32(RSP) + MOVD R1, 40(RSP) MOVD 144(R0), R10 MOVD 136(R0), R11 MOVD 200(R0), R1 - MOVD R1, 56(RSP) + MOVD R1, 64(RSP) MOVD 176(R0), R1 - MOVD R1, 48(RSP) + MOVD R1, 56(RSP) MOVD 184(R0), R0 - MOVD R0, 40(RSP) - MOVD 40(RSP), R0 - MOVD 48(RSP), R16 + MOVD R0, 48(RSP) + MOVD 48(RSP), R0 + MOVD 56(RSP), R16 ADD R0, R16, R16 - MOVD R16, 48(RSP) + MOVD R16, 56(RSP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - MOVD 32(RSP), R16 + MOVD 40(RSP), R16 ADD R9, R16, R16 - MOVD R16, 32(RSP) + MOVD R16, 40(RSP) // outBase += outPosition ADD R11, R9, R9 sequenceDecs_decodeSync_amd64_main_loop: - MOVD (RSP), R12 + MOVD 8(RSP), R12 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 @@ -1431,7 +1453,8 @@ sequenceDecs_decodeSync_amd64_fill_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_amd64_of_update_zero @@ -1445,14 +1468,15 @@ sequenceDecs_decodeSync_amd64_fill_end: ADD R13, R0, R0 sequenceDecs_decodeSync_amd64_of_update_zero: - MOVD R0, 8(RSP) + MOVD R0, 16(RSP) // Update match length MOVD R7, R0 MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_amd64_ml_update_zero @@ -1466,7 +1490,7 @@ sequenceDecs_decodeSync_amd64_of_update_zero: ADD R13, R0, R0 sequenceDecs_decodeSync_amd64_ml_update_zero: - MOVD R0, 16(RSP) + MOVD R0, 24(RSP) // Fill bitreader to have enough for the remaining CMP $0x08, R5 @@ -1502,7 +1526,8 @@ sequenceDecs_decodeSync_amd64_fill_2_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_amd64_ll_update_zero @@ -1516,10 +1541,10 @@ sequenceDecs_decodeSync_amd64_fill_2_end: ADD R13, R0, R0 sequenceDecs_decodeSync_amd64_ll_update_zero: - MOVD R0, 24(RSP) + MOVD R0, 32(RSP) // Fill bitreader for state updates - MOVD R12, (RSP) + MOVD R12, 8(RSP) MOVD R8, R0 LSR $0x08, R0, R0 MOVBU R0, R0 @@ -1537,7 +1562,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -1558,7 +1583,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -1579,7 +1604,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -1594,7 +1619,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: sequenceDecs_decodeSync_amd64_skip_update: // Adjust offset MOVD s+0(FP), R1 - MOVD 8(RSP), R12 + MOVD 16(RSP), R12 CMP $0x01, R0 BLS sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 ADD $144, R1, R15 @@ -1605,7 +1630,7 @@ sequenceDecs_decodeSync_amd64_skip_update: JMP sequenceDecs_decodeSync_amd64_after_adjust sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: - MOVD 24(RSP), R16 + MOVD 32(RSP), R16 CMP $0x00000000, R16 BNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero ADD $1, R12, R12 @@ -1643,11 +1668,11 @@ sequenceDecs_decodeSync_amd64_adjust_skip: MOVD R13, R12 sequenceDecs_decodeSync_amd64_after_adjust: - MOVD R12, 8(RSP) + MOVD R12, 16(RSP) // Check values - MOVD 16(RSP), R0 - MOVD 24(RSP), R1 + MOVD 24(RSP), R0 + MOVD 32(RSP), R1 ADD R1, R0, R13 MOVD s+0(FP), R14 MOVD 256(R14), R16 @@ -1666,14 +1691,15 @@ sequenceDecs_decodeSync_amd64_after_adjust: BNE sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_amd64_match_len_ofs_ok: - MOVD 24(RSP), R0 - MOVD 8(RSP), R1 - MOVD 16(RSP), R12 + MOVD 32(RSP), R0 + MOVD 16(RSP), R1 + MOVD 24(RSP), R12 // Check if we have enough space in s.out ADD R12, R0, R13 + ADD $16, R13, R13 ADD R9, R13, R13 - MOVD 32(RSP), R16 + MOVD 40(RSP), R16 CMP R16, R13 BHI error_not_enough_space @@ -1697,11 +1723,11 @@ copy_1: // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: MOVD R11, R0 - MOVD 40(RSP), R16 + MOVD 48(RSP), R16 ADD R16, R0, R0 CMP R0, R1 BGT error_match_off_too_big - MOVD 56(RSP), R16 + MOVD 64(RSP), R16 CMP R16, R1 BGT error_match_off_too_big @@ -1709,7 +1735,7 @@ check_offset: MOVD R1, R0 SUBS R11, R0, R0 BLS copy_match - MOVD 48(RSP), R13 + MOVD 56(RSP), R13 SUB R0, R13, R13 CMP R0, R12 BGT copy_all_from_history @@ -1742,13 +1768,14 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R0 - MOVB 2(R13), R1 - MOVH R0, (R9) - MOVB R1, 2(R9) - ADD R12, R13, R13 - ADD R12, R9, R9 - JMP copy_4_end + MOVHU (R13), R0 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R1 + MOVH R0, (R9) + MOVB R1, 2(R9) + ADD R12, R13, R13 + ADD R12, R9, R9 + JMP copy_4_end copy_4_move_4through7: MOVWU (R13), R0 @@ -1807,24 +1834,27 @@ copy_5_small: JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R0, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R9) - ADD R0, R9, R15 - MOVB R4, -1(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R0, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R9) + ADD R0, R9, R15 + MOVB R4, -1(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R9) - MOVB R4, 2(R9) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R9) + MOVB R4, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 @@ -1879,12 +1909,13 @@ copy_overlapping_match: ADD R12, R11, R11 copy_slow_3: - MOVB (R0), R1 - MOVB R1, (R9) - ADD $1, R0, R0 - ADD $1, R9, R9 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R0), R16 + BFI $0, R16, $8, R1 + MOVB R1, (R9) + ADD $1, R0, R0 + ADD $1, R9, R9 + SUBS $1, R12, R12 + BNE copy_slow_3 handle_loop: MOVD ctx+16(FP), R0 @@ -1913,7 +1944,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: - MOVD 16(RSP), R0 + MOVD 24(RSP), R0 MOVD ctx+16(FP), R1 MOVD R0, 216(R1) MOVD $0x00000001, R16 @@ -1923,7 +1954,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_amd64_error_match_len_too_big: MOVD ctx+16(FP), R0 - MOVD 16(RSP), R1 + MOVD 24(RSP), R1 MOVD R1, 216(R0) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) @@ -1932,7 +1963,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVD ctx+16(FP), R0 - MOVD 8(RSP), R1 + MOVD 16(RSP), R1 MOVD R1, 224(R0) MOVD R11, 136(R0) MOVD $0x00000003, R16 @@ -1942,7 +1973,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 32(RSP), R1 MOVD R1, 208(R0) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) @@ -1957,9 +1988,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 32(RSP), R1 MOVD R1, 208(R0) - MOVD 16(RSP), R1 + MOVD 24(RSP), R1 MOVD R1, 216(R0) MOVD R11, 136(R0) MOVD $0x00000005, R16 @@ -1977,41 +2008,41 @@ TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $64-32 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 MOVD $0, R1 - MOVD R1, 8(RSP) MOVD R1, 16(RSP) MOVD R1, 24(RSP) + MOVD R1, 32(RSP) MOVD 112(R0), R9 MOVD 128(R0), R1 - MOVD R1, 32(RSP) + MOVD R1, 40(RSP) MOVD 144(R0), R10 MOVD 136(R0), R11 MOVD 200(R0), R1 - MOVD R1, 56(RSP) + MOVD R1, 64(RSP) MOVD 176(R0), R1 - MOVD R1, 48(RSP) + MOVD R1, 56(RSP) MOVD 184(R0), R0 - MOVD R0, 40(RSP) - MOVD 40(RSP), R0 - MOVD 48(RSP), R16 + MOVD R0, 48(RSP) + MOVD 48(RSP), R0 + MOVD 56(RSP), R16 ADD R0, R16, R16 - MOVD R16, 48(RSP) + MOVD R16, 56(RSP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - MOVD 32(RSP), R16 + MOVD 40(RSP), R16 ADD R9, R16, R16 - MOVD R16, 32(RSP) + MOVD R16, 40(RSP) // outBase += outPosition ADD R11, R9, R9 sequenceDecs_decodeSync_safe_amd64_main_loop: - MOVD (RSP), R12 + MOVD 8(RSP), R12 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 @@ -2047,7 +2078,8 @@ sequenceDecs_decodeSync_safe_amd64_fill_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_safe_amd64_of_update_zero @@ -2061,14 +2093,15 @@ sequenceDecs_decodeSync_safe_amd64_fill_end: ADD R13, R0, R0 sequenceDecs_decodeSync_safe_amd64_of_update_zero: - MOVD R0, 8(RSP) + MOVD R0, 16(RSP) // Update match length MOVD R7, R0 MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_safe_amd64_ml_update_zero @@ -2082,7 +2115,7 @@ sequenceDecs_decodeSync_safe_amd64_of_update_zero: ADD R13, R0, R0 sequenceDecs_decodeSync_safe_amd64_ml_update_zero: - MOVD R0, 16(RSP) + MOVD R0, 24(RSP) // Fill bitreader to have enough for the remaining CMP $0x08, R5 @@ -2118,7 +2151,8 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_safe_amd64_ll_update_zero @@ -2132,10 +2166,10 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_end: ADD R13, R0, R0 sequenceDecs_decodeSync_safe_amd64_ll_update_zero: - MOVD R0, 24(RSP) + MOVD R0, 32(RSP) // Fill bitreader for state updates - MOVD R12, (RSP) + MOVD R12, 8(RSP) MOVD R8, R0 LSR $0x08, R0, R0 MOVBU R0, R0 @@ -2153,7 +2187,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -2174,7 +2208,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -2195,7 +2229,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -2210,7 +2244,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: sequenceDecs_decodeSync_safe_amd64_skip_update: // Adjust offset MOVD s+0(FP), R1 - MOVD 8(RSP), R12 + MOVD 16(RSP), R12 CMP $0x01, R0 BLS sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 ADD $144, R1, R15 @@ -2221,7 +2255,7 @@ sequenceDecs_decodeSync_safe_amd64_skip_update: JMP sequenceDecs_decodeSync_safe_amd64_after_adjust sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: - MOVD 24(RSP), R16 + MOVD 32(RSP), R16 CMP $0x00000000, R16 BNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero ADD $1, R12, R12 @@ -2259,11 +2293,11 @@ sequenceDecs_decodeSync_safe_amd64_adjust_skip: MOVD R13, R12 sequenceDecs_decodeSync_safe_amd64_after_adjust: - MOVD R12, 8(RSP) + MOVD R12, 16(RSP) // Check values - MOVD 16(RSP), R0 - MOVD 24(RSP), R1 + MOVD 24(RSP), R0 + MOVD 32(RSP), R1 ADD R1, R0, R13 MOVD s+0(FP), R14 MOVD 256(R14), R16 @@ -2282,14 +2316,14 @@ sequenceDecs_decodeSync_safe_amd64_after_adjust: BNE sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: - MOVD 24(RSP), R0 - MOVD 8(RSP), R1 - MOVD 16(RSP), R12 + MOVD 32(RSP), R0 + MOVD 16(RSP), R1 + MOVD 24(RSP), R12 // Check if we have enough space in s.out ADD R12, R0, R13 ADD R9, R13, R13 - MOVD 32(RSP), R16 + MOVD 40(RSP), R16 CMP R16, R13 BHI error_not_enough_space @@ -2326,24 +2360,27 @@ copy_1_small: JMP copy_1_move_8through16 copy_1_move_1or2: - MOVB (R10), R13 - ADD R0, R10, R15 - MOVB -1(R15), R14 - MOVB R13, (R9) - ADD R0, R9, R15 - MOVB R14, -1(R15) - ADD R0, R10, R10 - ADD R0, R9, R9 - JMP copy_1_end + MOVBU (R10), R16 + BFI $0, R16, $8, R13 + ADD R0, R10, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R14 + MOVB R13, (R9) + ADD R0, R9, R15 + MOVB R14, -1(R15) + ADD R0, R10, R10 + ADD R0, R9, R9 + JMP copy_1_end copy_1_move_3: - MOVH (R10), R13 - MOVB 2(R10), R14 - MOVH R13, (R9) - MOVB R14, 2(R9) - ADD R0, R10, R10 - ADD R0, R9, R9 - JMP copy_1_end + MOVHU (R10), R13 + MOVBU 2(R10), R16 + BFI $0, R16, $8, R14 + MOVH R13, (R9) + MOVB R14, 2(R9) + ADD R0, R10, R10 + ADD R0, R9, R9 + JMP copy_1_end copy_1_move_4through7: MOVWU (R10), R13 @@ -2372,11 +2409,11 @@ copy_1_end: // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: MOVD R11, R0 - MOVD 40(RSP), R16 + MOVD 48(RSP), R16 ADD R16, R0, R0 CMP R0, R1 BGT error_match_off_too_big - MOVD 56(RSP), R16 + MOVD 64(RSP), R16 CMP R16, R1 BGT error_match_off_too_big @@ -2384,7 +2421,7 @@ check_offset: MOVD R1, R0 SUBS R11, R0, R0 BLS copy_match - MOVD 48(RSP), R13 + MOVD 56(RSP), R13 SUB R0, R13, R13 CMP R0, R12 BGT copy_all_from_history @@ -2417,13 +2454,14 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R0 - MOVB 2(R13), R1 - MOVH R0, (R9) - MOVB R1, 2(R9) - ADD R12, R13, R13 - ADD R12, R9, R9 - JMP copy_4_end + MOVHU (R13), R0 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R1 + MOVH R0, (R9) + MOVB R1, 2(R9) + ADD R12, R13, R13 + ADD R12, R9, R9 + JMP copy_4_end copy_4_move_4through7: MOVWU (R13), R0 @@ -2482,24 +2520,27 @@ copy_5_small: JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R0, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R9) - ADD R0, R9, R15 - MOVB R4, -1(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R0, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R9) + ADD R0, R9, R15 + MOVB R4, -1(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R9) - MOVB R4, 2(R9) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R9) + MOVB R4, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 @@ -2567,24 +2608,27 @@ copy_2_small: JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (R0), R1 - ADD R12, R0, R15 - MOVB -1(R15), R13 - MOVB R1, (R9) - ADD R12, R9, R15 - MOVB R13, -1(R15) - ADD R12, R0, R0 - ADD R12, R9, R9 - JMP copy_2_end + MOVBU (R0), R16 + BFI $0, R16, $8, R1 + ADD R12, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R13 + MOVB R1, (R9) + ADD R12, R9, R15 + MOVB R13, -1(R15) + ADD R12, R0, R0 + ADD R12, R9, R9 + JMP copy_2_end copy_2_move_3: - MOVH (R0), R1 - MOVB 2(R0), R13 - MOVH R1, (R9) - MOVB R13, 2(R9) - ADD R12, R0, R0 - ADD R12, R9, R9 - JMP copy_2_end + MOVHU (R0), R1 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R13 + MOVH R1, (R9) + MOVB R13, 2(R9) + ADD R12, R0, R0 + ADD R12, R9, R9 + JMP copy_2_end copy_2_move_4through7: MOVWU (R0), R1 @@ -2615,12 +2659,13 @@ copy_overlapping_match: ADD R12, R11, R11 copy_slow_3: - MOVB (R0), R1 - MOVB R1, (R9) - ADD $1, R0, R0 - ADD $1, R9, R9 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R0), R16 + BFI $0, R16, $8, R1 + MOVB R1, (R9) + ADD $1, R0, R0 + ADD $1, R9, R9 + SUBS $1, R12, R12 + BNE copy_slow_3 handle_loop: MOVD ctx+16(FP), R0 @@ -2649,7 +2694,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: - MOVD 16(RSP), R0 + MOVD 24(RSP), R0 MOVD ctx+16(FP), R1 MOVD R0, 216(R1) MOVD $0x00000001, R16 @@ -2659,7 +2704,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: MOVD ctx+16(FP), R0 - MOVD 16(RSP), R1 + MOVD 24(RSP), R1 MOVD R1, 216(R0) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) @@ -2668,7 +2713,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVD ctx+16(FP), R0 - MOVD 8(RSP), R1 + MOVD 16(RSP), R1 MOVD R1, 224(R0) MOVD R11, 136(R0) MOVD $0x00000003, R16 @@ -2678,7 +2723,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 32(RSP), R1 MOVD R1, 208(R0) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) @@ -2693,9 +2738,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 32(RSP), R1 MOVD R1, 208(R0) - MOVD 16(RSP), R1 + MOVD 24(RSP), R1 MOVD R1, 216(R0) MOVD R11, 136(R0) MOVD $0x00000005, R16 diff --git a/src/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go b/src/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go index 55405f39..42ce5de7 100644 --- a/src/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go +++ b/src/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go @@ -77,6 +77,35 @@ const errorNotEnoughSpace = 5 // error reported when bits are overread. const errorOverread = 6 +// useSafeDecodeSync reports whether decodeSyncSimple must use the bounds-exact +// ("safe") copy variants for the current buffer geometry. +// +// When the output and literal buffers have compressedBlockOverAlloc (16) +// bytes of slack past their logical use, the assembly may use extended +// memory copies that read and write in 16-byte blocks, overrunning the end +// of a literal run or match by up to 15 bytes. Otherwise it must use the +// bounds-exact ("safe") copies. This mirrors the analogous, always-dynamic +// selection in executeSimple below. +// +// The unsafe copies were disabled in #644 (2022) as a mitigation for a +// crash, but that crash's root cause — an unguarded bitReader overread that +// produced out-of-range match offsets/lengths — was fixed three days later +// in #645, which also added the fuzz corpus that has guarded this path since. +// See #1168. An asan-instrumented fuzz job (see .github/workflows/go.yml) +// covers the extended-copy path, which -race and plain fuzzing cannot. +func (s *sequenceDecs) useSafeDecodeSync() bool { + if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc { + return true + } + if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) { + return true + } + if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { + return true + } + return false +} + // decode sequences from the stream with the provided history but without a dictionary. func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { if len(s.dict) > 0 { @@ -86,9 +115,7 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { return false, nil } - // FIXME: Using unsafe memory copies leads to rare, random crashes - // with fuzz testing. It is therefore disabled for now. - const useSafe = true + useSafe := s.useSafeDecodeSync() br := s.br diff --git a/src/vendor/github.com/minio/minio-go/v7/.gitignore b/src/vendor/github.com/minio/minio-go/v7/.gitignore index 5bb12e74..39f4bee7 100644 --- a/src/vendor/github.com/minio/minio-go/v7/.gitignore +++ b/src/vendor/github.com/minio/minio-go/v7/.gitignore @@ -5,3 +5,4 @@ golangci-lint functional_tests .idea vendor/ +.branch-validate/ diff --git a/src/vendor/github.com/minio/minio-go/v7/api-bucket-policy.go b/src/vendor/github.com/minio/minio-go/v7/api-bucket-policy.go index 0e561bdf..c4b768ce 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api-bucket-policy.go +++ b/src/vendor/github.com/minio/minio-go/v7/api-bucket-policy.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2020 MinIO, Inc. + * Copyright 2026 MinIO, Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -18,14 +18,44 @@ package minio import ( "context" + "encoding/json" + "fmt" "io" "net/http" "net/url" "strings" + "github.com/minio/minio-go/v7/pkg/policy" "github.com/minio/minio-go/v7/pkg/s3utils" ) +// SetCannedBucketPolicy sets a canned (predefined) access policy on a bucket. +// This is a convenience method that translates predefined access levels +// (readonly, writeonly, readwrite) into the corresponding bucket policy. +// If access is empty or results in no policy statements, the existing bucket policy will be removed. +// +// Parameters: +// - ctx: Context for request cancellation and timeout +// - bucketName: Name of the bucket +// - access: Canned access level (policy.BucketPolicyNone, policy.BucketPolicyReadOnly, policy.BucketPolicyWriteOnly, policy.BucketPolicyReadWrite) +// +// Returns an error if the operation fails. +func (c *Client) SetCannedBucketPolicy(ctx context.Context, bucketName string, bucketPolicy policy.BucketPolicy) error { + if !bucketPolicy.IsValidBucketPolicy() { + return fmt.Errorf("invalid bucket policy %q", bucketPolicy) + } + p := policy.BucketAccessPolicy{Version: "2012-10-17"} + p.Statements = policy.SetPolicy(p.Statements, bucketPolicy, bucketName, "") + if len(p.Statements) == 0 { + return c.SetBucketPolicy(ctx, bucketName, "") + } + policyB, e := json.Marshal(p) + if e != nil { + return e + } + return c.SetBucketPolicy(ctx, bucketName, string(policyB)) +} + // SetBucketPolicy sets the access permissions policy on an existing bucket. // The policy should be a valid JSON string that conforms to the IAM policy format. // If policy is an empty string, the existing bucket policy will be removed. diff --git a/src/vendor/github.com/minio/minio-go/v7/api-compose-object.go b/src/vendor/github.com/minio/minio-go/v7/api-compose-object.go index 6d91bed4..2ea833b7 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api-compose-object.go +++ b/src/vendor/github.com/minio/minio-go/v7/api-compose-object.go @@ -386,6 +386,7 @@ func (c *Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, des return p, err } p.PartNumber, p.ETag = partID, cpObjRes.ETag + cpObjRes.setChecksums(&p) return p, nil } @@ -424,6 +425,7 @@ func (c *Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID st return p, err } p.PartNumber, p.ETag = partNumber, cpObjRes.ETag + cpObjRes.setChecksums(&p) return p, nil } @@ -531,6 +533,19 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs .. userTags = srcObjectInfos[0].UserTags } + // Set the requested checksum algorithm on the multipart upload. + if dst.ChecksumType.IsSet() { + meta := make(map[string]string, len(userMeta)+2) + for k, v := range userMeta { + meta[k] = v + } + meta[amzChecksumAlgo] = dst.ChecksumType.String() + if dst.ChecksumType.FullObjectRequested() { + meta[amzChecksumMode] = ChecksumFullObjectMode.String() + } + userMeta = meta + } + uploadID, err := c.newUploadID(ctx, dst.Bucket, dst.Object, PutObjectOptions{ ServerSideEncryption: dst.Encryption, UserMetadata: userMeta, diff --git a/src/vendor/github.com/minio/minio-go/v7/api-datatypes.go b/src/vendor/github.com/minio/minio-go/v7/api-datatypes.go index f359f29a..2391c6e5 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api-datatypes.go +++ b/src/vendor/github.com/minio/minio-go/v7/api-datatypes.go @@ -184,10 +184,30 @@ type ObjectInfo struct { // eg: x-amz-meta-*, content-encoding etc. Metadata http.Header `json:"metadata" xml:"-"` - // x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. + // Headers is the unfiltered set of response headers from the + // HEAD or GET request that produced this ObjectInfo, e.g. + // Content-Range for ranged requests. RFC 2047-encoded + // x-amz-meta-*/x-minio-meta-* values appear MIME-decoded, matching + // Metadata. Set only when the object info is parsed from an HTTP + // object response (StatObject, GetObject); nil elsewhere, e.g. in + // ListObjects results, on error paths, or for RDMA-backed GetObject. + // It aliases the response headers and must be treated as read-only. + Headers http.Header `json:"rawHeaders,omitempty" xml:"-"` + + // UserMetadata contains x-amz-meta-* user metadata. + // StatObject and GetObject return it with the "X-Amz-Meta-" prefix + // stripped; list results with WithMetadata keep the exact values stored + // on the object (prefixed keys plus system entries such as content-type). // Only returned by MinIO servers. UserMetadata StringMap `json:"userMetadata,omitempty"` + // UserMetadataStripped is the user metadata from list results in the + // keyed form StatObject and GetObject return in UserMetadata: + // x-amz-meta-* entries with the "X-Amz-Meta-" prefix stripped and + // values passed through verbatim. + // Only populated by MinIO servers when listing with WithMetadata. + UserMetadataStripped StringMap `json:"userMetadataStripped,omitempty" xml:"-"` + // x-amz-tagging values in their k/v values. // Only returned by MinIO servers. UserTags URLMap `json:"userTags,omitempty" xml:"UserTags"` diff --git a/src/vendor/github.com/minio/minio-go/v7/api-get-object-file.go b/src/vendor/github.com/minio/minio-go/v7/api-get-object-file.go index 6ef9c933..dd62b3f7 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api-get-object-file.go +++ b/src/vendor/github.com/minio/minio-go/v7/api-get-object-file.go @@ -72,7 +72,7 @@ func (c *Client) FGetObject(ctx context.Context, bucketName, objectName, filePat filePartPath := filepath.Join(filepath.Dir(filePath), sum256Hex([]byte(filepath.Base(filePath)+objectStat.ETag))+".part.minio") // If exists, open in append mode. If not create it as a part file. - filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0o600) if err != nil { return err } @@ -98,19 +98,41 @@ func (c *Client) FGetObject(ctx context.Context, bucketName, objectName, filePat // appropriate range offsets to read from. if st.Size() > 0 { opts.SetRange(st.Size(), 0) + if opts.Checksum && objectStat.ChecksumMode == ChecksumFullObjectMode.String() { + if hasherReader := c.newChecksumVerifyingReader(objectStat); hasherReader != nil { + // Read existing file data into hash. + if _, err = io.CopyN(hasherReader.Hash, filePart, st.Size()); err != nil { + _ = hasherReader.Close() + return err + } + opts.checkSumReader = hasherReader + } + } } // Seek to current position for incoming reader. objectReader, objectStat, _, err := c.getObject(ctx, bucketName, objectName, opts) if err != nil { + if opts.checkSumReader != nil { + _ = opts.checkSumReader.Close() + } return err } + defer objectReader.Close() + // Write to the part file. if _, err = io.CopyN(filePart, objectReader, objectStat.Size); err != nil { return err } + // Verify the checksum of the downloaded object before committing the file. + if cr, ok := objectReader.(*checksumVerifyingReader); ok { + if err = cr.VerifyChecksum(); err != nil { + return err + } + } + // Close the file before rename, this is specifically needed for Windows users. closeAndRemove = false if err = filePart.Close(); err != nil { diff --git a/src/vendor/github.com/minio/minio-go/v7/api-get-object.go b/src/vendor/github.com/minio/minio-go/v7/api-get-object.go index ab783551..d41d516a 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api-get-object.go +++ b/src/vendor/github.com/minio/minio-go/v7/api-get-object.go @@ -81,6 +81,8 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o reqCh := make(chan getRequest) // Create response channel. resCh := make(chan getResponse) + // record original range header for stat operation. + originalRangeHeader := opts.Header().Get("Range") // This routine feeds partial object data as and when the caller reads. go func() { @@ -115,6 +117,12 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o httpReader, objectInfo, _, err = c.getObject(gctx, bucketName, objectName, opts) if err != nil { resCh <- getResponse{Error: err} + // An unsatisfiable range is recoverable: the caller + // may Seek or ReadAt within bounds next, so keep + // serving requests instead of ending the stream. + if ToErrorResponse(err).Code == InvalidRange { + continue + } return } etag = objectInfo.ETag @@ -141,19 +149,35 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o // it to io.EOF - return unexpected EOF. err = io.ErrUnexpectedEOF } + // when doing a readAt, we can't reuse the httpReader for next read action. + if req.isReadAt { + httpReader.Close() + httpReader = nil + } + + objectSize := int64(0) + // case for ReadFull with range request + if !req.isReadAt && req.Offset == 0 { + objectSize = objectInfo.Size + } // Send back the first response. resCh <- getResponse{ + ObjectSize: objectSize, objectInfo: objectInfo, Size: size, Error: err, didRead: true, } } else { - // First request is a Stat or Seek call. - // Only need to run a StatObject until an actual Read or ReadAt request comes through. + if originalRangeHeader == "" { + // First request is a Seek call. + // Only need to run a StatObject until an actual Read request comes through. - // Remove range header if already set, for stat Operations to get original file size. - delete(opts.headers, "Range") + // Remove range header if already set, for stat Operations to get original file size. + delete(opts.headers, "Range") + } else { + opts.headers["Range"] = originalRangeHeader + } objectInfo, err = c.StatObject(gctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { resCh <- getResponse{ @@ -165,12 +189,17 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o etag = objectInfo.ETag // Send back the first response. resCh <- getResponse{ + ObjectSize: objectInfo.Size, objectInfo: objectInfo, } } } else if req.settingObjectInfo { // Request is just to get objectInfo. // Remove range header if already set, for stat Operations to get original file size. - delete(opts.headers, "Range") + if originalRangeHeader == "" { + delete(opts.headers, "Range") + } else { + opts.headers["Range"] = originalRangeHeader + } // Check whether this is snowball // if yes do not use If-Match feature // it doesn't work. @@ -187,16 +216,22 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o } // Send back the objectInfo. resCh <- getResponse{ + ObjectSize: objectInfo.Size, objectInfo: objectInfo, } } else { + objectSize := int64(0) + renewReader := false // Offset changes fetch the new object at an Offset. // Because the httpReader may not be set by the first // request if it was a stat or seek it must be checked // if the object has been read or not to only initialize // new ones when they haven't been already. // All readAt requests are new requests. - if req.DidOffsetChange || !req.beenRead { + // A nil httpReader means the previous fetch failed and + // left no stream, so a new one must be established + // regardless of the offset bookkeeping. + if req.DidOffsetChange || !req.beenRead || httpReader == nil { // Check whether this is snowball // if yes do not use If-Match feature // it doesn't work. @@ -214,16 +249,30 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o } else if req.Offset > 0 { // Range is set with respect to the offset. opts.SetRange(req.Offset, 0) } else { - // Remove range header if already set - delete(opts.headers, "Range") + if originalRangeHeader == "" { + delete(opts.headers, "Range") + } else { + opts.headers["Range"] = originalRangeHeader + } } httpReader, objectInfo, _, err = c.getObject(gctx, bucketName, objectName, opts) if err != nil { resCh <- getResponse{ Error: err, } + // An unsatisfiable range is recoverable: the caller + // may Seek or ReadAt within bounds next, so keep + // serving requests instead of ending the stream. + if ToErrorResponse(err).Code == InvalidRange { + continue + } return } + // case for ReadFull with range request + if !req.isReadAt && req.Offset == 0 { + objectSize = objectInfo.Size + } + renewReader = true totalRead = 0 } @@ -250,9 +299,13 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o // it to io.EOF - return unexpected EOF. err = io.ErrUnexpectedEOF } - + if renewReader && req.isReadAt { + httpReader.Close() + httpReader = nil + } // Reply back how much was read. resCh <- getResponse{ + ObjectSize: objectSize, Size: size, Error: err, didRead: true, @@ -281,6 +334,7 @@ type getRequest struct { // get response message container to reply back for the request. type getResponse struct { + ObjectSize int64 Size int Error error didRead bool // Lets subsequent calls know whether or not httpReader has been initiated. @@ -299,6 +353,7 @@ type Object struct { ctx context.Context cancel context.CancelFunc currOffset int64 + totalSize int64 objectInfo ObjectInfo // Ask lower level to initiate data fetching based on currOffset @@ -356,6 +411,10 @@ func (o *Object) doGetRequest(request getRequest) (getResponse, error) { // Data are ready on the wire, no need to reinitiate connection in lower level o.seekData = false + if response.ObjectSize != 0 { + o.totalSize = response.ObjectSize + } + return response, response.Error } @@ -365,7 +424,7 @@ func (o *Object) setOffset(bytesRead int64) error { // Update the currentOffset. o.currOffset += bytesRead - if o.objectInfo.Size > -1 && o.currOffset >= o.objectInfo.Size { + if o.totalSize > -1 && o.currOffset >= o.totalSize { return io.EOF } return nil @@ -387,6 +446,13 @@ func (o *Object) Read(b []byte) (n int, err error) { if o.prevErr != nil || o.isClosed { return 0, o.prevErr } + // If the current offset is at or beyond the known object size, there is + // nothing more to read. Return io.EOF directly instead of issuing a range + // request from EOF (which the server rejects as an unsatisfiable range). + if o.objectInfoSet && o.objectInfo.Size > -1 && o.currOffset >= o.objectInfo.Size { + o.prevErr = io.EOF + return 0, io.EOF + } // Create a new request. readReq := getRequest{ @@ -407,6 +473,14 @@ func (o *Object) Read(b []byte) (n int, err error) { // Send and receive from the first request. response, err := o.doGetRequest(readReq) if err != nil && err != io.EOF { + // An InvalidRange response to a range generated from a non-zero + // read offset means the position is at or past EOF for the object + // as it currently exists. Offset zero sends only a caller-supplied + // range, whose InvalidRange must surface untranslated. + if o.currOffset > 0 && ToErrorResponse(err).Code == InvalidRange { + o.prevErr = io.EOF + return 0, io.EOF + } // Save the error for future calls. o.prevErr = err return response.Size, err @@ -428,6 +502,8 @@ func (o *Object) Read(b []byte) (n int, err error) { } // Stat returns the ObjectInfo structure describing Object. +// When requesting a partial object or reading has started, +// the size returned will reflect the remaining size. func (o *Object) Stat() (ObjectInfo, error) { if o == nil { return ObjectInfo{}, errInvalidArgument("Object is nil") @@ -436,7 +512,24 @@ func (o *Object) Stat() (ObjectInfo, error) { o.mutex.Lock() defer o.mutex.Unlock() - if o.prevErr != nil && o.prevErr != io.EOF || o.isClosed { + if o.prevErr != nil && o.prevErr != io.EOF { + return ObjectInfo{}, o.prevErr + } + + // When the object info is already known (e.g. the RDMA GET path, whose + // payload is delivered out-of-band so the object is created closed, or a + // previously completed request) report it, even for a closed object. + if o.objectInfoSet { + if o.currOffset > o.totalSize { + return ObjectInfo{}, io.EOF + } + if o.currOffset <= o.totalSize { + o.objectInfo.Size = o.totalSize - o.currOffset + } + return o.objectInfo, nil + } + + if o.isClosed { return ObjectInfo{}, o.prevErr } @@ -452,7 +545,12 @@ func (o *Object) Stat() (ObjectInfo, error) { return ObjectInfo{}, err } } - + if o.currOffset > o.totalSize { + return ObjectInfo{}, io.EOF + } + if o.currOffset <= o.totalSize { + o.objectInfo.Size = o.totalSize - o.currOffset + } return o.objectInfo, nil } @@ -488,7 +586,7 @@ func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { if o.objectInfoSet { // If offset is negative than we return io.EOF. // If offset is greater than or equal to object size we return io.EOF. - if (o.objectInfo.Size > -1 && offset >= o.objectInfo.Size) || offset < 0 { + if (o.totalSize > -1 && offset >= o.totalSize) || offset < 0 { return 0, io.EOF } } @@ -511,6 +609,12 @@ func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { // Send and receive from the first request. response, err := o.doGetRequest(readAtReq) if err != nil && err != io.EOF { + // Reading at an offset at or beyond the object size yields + // InvalidRange from the server: report io.EOF, matching the + // io.ReaderAt contract for reads past the end. + if ToErrorResponse(err).Code == InvalidRange { + return 0, io.EOF + } // Save the error. o.prevErr = err return response.Size, err @@ -541,9 +645,10 @@ func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { // and 2 means relative to the end. // Seek returns the new offset and an error, if any. // -// Seeking to a negative offset is an error. Seeking to any positive -// offset is legal, subsequent io operations succeed until the -// underlying object is not closed. +// Seeking to a position before the start of the object is an error. +// Seeking to the end of the object is legal; the subsequent Read reports +// io.EOF. When the object size is known, seeking past the end returns +// io.EOF from Seek itself and leaves the offset unchanged. func (o *Object) Seek(offset int64, whence int) (n int64, err error) { if o == nil { return 0, errInvalidArgument("Object is nil") @@ -558,9 +663,11 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { return 0, o.prevErr } - // Negative offset is valid for whence of '2'. - if offset < 0 && whence != 2 { - return 0, errInvalidArgument(fmt.Sprintf("Negative position not allowed for %d", whence)) + // Negative absolute offsets are invalid for SeekStart. Negative offsets + // for SeekCurrent/SeekEnd are valid as long as the computed position is + // not before the start of the object (checked after applying whence). + if offset < 0 && whence == 0 { + return 0, errInvalidArgument("Negative position not allowed for 0") } // This is the first request. So before anything else @@ -588,31 +695,26 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { default: return 0, errInvalidArgument(fmt.Sprintf("Invalid whence %d", whence)) case 0: - if o.objectInfo.Size > -1 && offset > o.objectInfo.Size { - return 0, io.EOF - } newOffset = offset case 1: - if o.objectInfo.Size > -1 && o.currOffset+offset > o.objectInfo.Size { - return 0, io.EOF - } newOffset += offset case 2: // If we don't know the object size return an error for io.SeekEnd - if o.objectInfo.Size < 0 { + if o.totalSize < 0 { return 0, errInvalidArgument("Whence END is not supported when the object size is unknown") } - // Seeking to positive offset is valid for whence '2', but - // since we are backing a Reader we have reached 'EOF' if - // offset is positive. - if offset > 0 { - return 0, io.EOF - } - // Seeking to negative position not allowed for whence. - if o.objectInfo.Size+offset < 0 { - return 0, errInvalidArgument(fmt.Sprintf("Seeking at negative offset not allowed for %d", whence)) - } - newOffset = o.objectInfo.Size + offset + newOffset = o.totalSize + offset + } + // Seeking to a position before the start of the object is not allowed for + // any whence. + if newOffset < 0 { + return 0, errInvalidArgument(fmt.Sprintf("Seeking at negative offset not allowed for %d", whence)) + } + // Seeking past the end of the object is rejected with io.EOF, preserving + // long-standing behavior. Seeking to exactly the end is legal and the + // subsequent Read reports io.EOF. + if o.objectInfo.Size > -1 && newOffset > o.objectInfo.Size { + return 0, io.EOF } // Reset the saved error since we successfully seeked, let the Read // and ReadAt decide. @@ -717,6 +819,19 @@ func (c *Client) getObject(ctx context.Context, bucketName, objectName string, o return nil, ObjectInfo{}, nil, err } + body := resp.Body + if opts.Checksum { + if opts.checkSumReader != nil { + opts.checkSumReader.SetReader(resp.Body) + body = opts.checkSumReader + } else if objectStat.ChecksumMode == ChecksumFullObjectMode.String() && opts.headers["Range"] == "" { + if hasherReader := c.newChecksumVerifyingReader(objectStat); hasherReader != nil { + hasherReader.SetReader(resp.Body) + body = hasherReader + } + } + } + // do not close body here, caller will close - return resp.Body, objectStat, resp.Header, nil + return body, objectStat, resp.Header, nil } diff --git a/src/vendor/github.com/minio/minio-go/v7/api-get-options.go b/src/vendor/github.com/minio/minio-go/v7/api-get-options.go index dd82f97a..e78235b1 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api-get-options.go +++ b/src/vendor/github.com/minio/minio-go/v7/api-get-options.go @@ -44,10 +44,15 @@ type GetObjectOptions struct { VersionID string PartNumber int - // Include any checksums, if object was uploaded with checksum. // For multipart objects this is a checksum of part checksums. // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + // When the response advertises a full-object checksum, un-ranged + // downloads are verified against it: reads through *Object fail with a + // checksum mismatch error at EOF, and FGetObject fails before renaming + // the file into place. Reads that stop before EOF are not verified. Checksum bool + // If not nil, continue checksum hash verification on the existing data. + checkSumReader *checksumVerifyingReader // RDMABuffer, when non-nil and Options.EnableRDMA=true, downloads directly // into a contiguous buffer via libminiocpp.so. The returned *Object's diff --git a/src/vendor/github.com/minio/minio-go/v7/api-list.go b/src/vendor/github.com/minio/minio-go/v7/api-list.go index 5e4a3817..3e13d36a 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api-list.go +++ b/src/vendor/github.com/minio/minio-go/v7/api-list.go @@ -127,6 +127,9 @@ func (c *Client) listObjectsV2(ctx context.Context, bucketName string, opts List // Return object owner information by default fetchOwner := true + if opts.FetchOwner != nil { + fetchOwner = *opts.FetchOwner + } return func(yield func(ObjectInfo) bool) { if contextCanceled(ctx) { @@ -296,6 +299,7 @@ func (c *Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefi return listBucketResult, err } listBucketResult.Contents[i].LastModified = listBucketResult.Contents[i].LastModified.Truncate(time.Millisecond) + listBucketResult.Contents[i].UserMetadataStripped = stripUserMetadata(obj.UserMetadata) } for i, obj := range listBucketResult.CommonPrefixes { @@ -419,31 +423,32 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts } for _, version := range vers { info := ObjectInfo{ - ETag: trimEtag(version.ETag), - Key: version.Key, - LastModified: version.LastModified.Truncate(time.Millisecond), - Size: version.Size, - Owner: version.Owner, - StorageClass: version.StorageClass, - IsLatest: version.IsLatest, - VersionID: version.VersionID, - IsDeleteMarker: version.isDeleteMarker, - UserTags: version.UserTags, - UserMetadata: version.UserMetadata, - Internal: version.Internal, - NumVersions: numVersions, - ChecksumAlgorithm: version.ChecksumAlgorithm, - ChecksumMode: version.ChecksumType, - ChecksumCRC32: version.ChecksumCRC32, - ChecksumCRC32C: version.ChecksumCRC32C, - ChecksumSHA1: version.ChecksumSHA1, - ChecksumSHA256: version.ChecksumSHA256, - ChecksumCRC64NVME: version.ChecksumCRC64NVME, - ChecksumMD5: version.ChecksumMD5, - ChecksumSHA512: version.ChecksumSHA512, - ChecksumXXHash64: version.ChecksumXXHash64, - ChecksumXXHash3: version.ChecksumXXHash3, - ChecksumXXHash128: version.ChecksumXXHash128, + ETag: trimEtag(version.ETag), + Key: version.Key, + LastModified: version.LastModified.Truncate(time.Millisecond), + Size: version.Size, + Owner: version.Owner, + StorageClass: version.StorageClass, + IsLatest: version.IsLatest, + VersionID: version.VersionID, + IsDeleteMarker: version.isDeleteMarker, + UserTags: version.UserTags, + UserMetadata: version.UserMetadata, + UserMetadataStripped: version.UserMetadataStripped, + Internal: version.Internal, + NumVersions: numVersions, + ChecksumAlgorithm: version.ChecksumAlgorithm, + ChecksumMode: version.ChecksumType, + ChecksumCRC32: version.ChecksumCRC32, + ChecksumCRC32C: version.ChecksumCRC32C, + ChecksumSHA1: version.ChecksumSHA1, + ChecksumSHA256: version.ChecksumSHA256, + ChecksumCRC64NVME: version.ChecksumCRC64NVME, + ChecksumMD5: version.ChecksumMD5, + ChecksumSHA512: version.ChecksumSHA512, + ChecksumXXHash64: version.ChecksumXXHash64, + ChecksumXXHash3: version.ChecksumXXHash3, + ChecksumXXHash128: version.ChecksumXXHash128, } if !yield(info) { return false @@ -606,6 +611,7 @@ func (c *Client) listObjectVersionsQuery(ctx context.Context, bucketName string, if err != nil { return listObjectVersionsOutput, err } + listObjectVersionsOutput.Versions[i].UserMetadataStripped = stripUserMetadata(obj.UserMetadata) } for i, obj := range listObjectVersionsOutput.CommonPrefixes { @@ -738,6 +744,10 @@ type ListObjectsOptions struct { // Use the deprecated list objects V1 API UseV1 bool + // FetchOwner indicates whether to return object owner information. + // It defaults to true when unset. + FetchOwner *bool + headers http.Header } diff --git a/src/vendor/github.com/minio/minio-go/v7/api-presigned.go b/src/vendor/github.com/minio/minio-go/v7/api-presigned.go index 29642200..1591fab3 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api-presigned.go +++ b/src/vendor/github.com/minio/minio-go/v7/api-presigned.go @@ -140,7 +140,7 @@ func (c *Client) PresignedPostPolicy(ctx context.Context, p *PostPolicy) (u *url } // Get credentials from the configured credentials provider. - credValues, err := c.credsProvider.GetWithContext(c.CredContext()) + credValues, err := c.credsProvider.GetWithContext(c.credContext(ctx)) if err != nil { return nil, nil, err } diff --git a/src/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go b/src/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go index 3023b949..cba0c069 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go +++ b/src/vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go @@ -69,6 +69,8 @@ type PutObjectFanOutResponse struct { // stream multiple objects are written, defined via a list of PutObjectFanOutRequests. Each entry // in PutObjectFanOutRequest carries an object keyname and its relevant metadata if any. `Key` is // mandatory, rest of the other options in PutObjectFanOutRequest are optional. +// +// Deprecated: Use PutObject instead. func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, fanOutData io.Reader, fanOutReq PutObjectFanOutRequest) ([]PutObjectFanOutResponse, error) { if len(fanOutReq.Entries) == 0 { return nil, errInvalidArgument("fan out requests cannot be empty") diff --git a/src/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go b/src/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go index fe503fb6..402319cb 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go +++ b/src/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go @@ -94,10 +94,16 @@ type Version struct { StorageClass string VersionID string `xml:"VersionId"` - // x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. - // Only returned by MinIO servers. + // UserMetadata contains the exact stored object metadata: x-amz-meta-* + // entries with their prefixed keys plus system entries such as + // content-type. Only returned by MinIO servers. UserMetadata StringMap `json:"userMetadata,omitempty"` + // UserMetadataStripped is UserMetadata in the keyed form StatObject and + // GetObject return in ObjectInfo.UserMetadata. See + // ObjectInfo.UserMetadataStripped. + UserMetadataStripped StringMap `json:"userMetadataStripped,omitempty" xml:"-"` + // x-amz-tagging values in their k/v values. // Only returned by MinIO servers. UserTags URLMap `json:"userTags,omitempty" xml:"UserTags"` @@ -280,6 +286,32 @@ type initiator struct { type copyObjectResult struct { ETag string LastModified time.Time // time string format "2006-01-02T15:04:05.000Z" + + // Checksum values returned in CopyObjectResult / CopyPartResult. + ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"` + ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"` + ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"` + ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"` + ChecksumCRC64NVME string `xml:",omitempty"` + ChecksumMD5 string `xml:",omitempty"` + ChecksumSHA512 string `xml:",omitempty"` + ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"` + ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"` + ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"` +} + +// setChecksums copies the copied part's checksums onto a CompletePart. +func (r *copyObjectResult) setChecksums(p *CompletePart) { + p.ChecksumCRC32 = r.ChecksumCRC32 + p.ChecksumCRC32C = r.ChecksumCRC32C + p.ChecksumSHA1 = r.ChecksumSHA1 + p.ChecksumSHA256 = r.ChecksumSHA256 + p.ChecksumCRC64NVME = r.ChecksumCRC64NVME + p.ChecksumMD5 = r.ChecksumMD5 + p.ChecksumSHA512 = r.ChecksumSHA512 + p.ChecksumXXHash64 = r.ChecksumXXHash64 + p.ChecksumXXHash3 = r.ChecksumXXHash3 + p.ChecksumXXHash128 = r.ChecksumXXHash128 } // ObjectPart container for particular part of an object. diff --git a/src/vendor/github.com/minio/minio-go/v7/api-stat.go b/src/vendor/github.com/minio/minio-go/v7/api-stat.go index a4b2af7a..ba9afa75 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api-stat.go +++ b/src/vendor/github.com/minio/minio-go/v7/api-stat.go @@ -57,7 +57,12 @@ func (c *Client) BucketExists(ctx context.Context, bucketName string) (bool, err } // StatObject verifies if object exists, you have permission to access it -// and returns information about the object. +// and returns information about the object. When the returned error is +// non-nil but a response was received, the ObjectInfo still carries the +// VersionID and IsDeleteMarker values parsed from the response headers, +// plus ReplicationReady on every error path except the versioned +// delete-marker 405 (an asymmetry preserved from the pre-v7.0.93 +// behavior, which populated ReplicationReady on the generic branch only). func (c *Client) StatObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { @@ -92,32 +97,32 @@ func (c *Client) StatObject(ctx context.Context, bucketName, objectName string, }) defer closeResponse(resp) if err != nil { - return ObjectInfo{}, err - } - - if resp != nil { + // executeMethod returns a non-nil error for every non-success + // status. When a response exists, its headers still carry the + // version and delete-marker fields — surface them with the error. + if resp == nil { + return ObjectInfo{}, err + } deleteMarker := resp.Header.Get(amzDeleteMarker) == "true" replicationReady := resp.Header.Get(minioTgtReplicationReady) == "true" - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - if resp.StatusCode == http.StatusMethodNotAllowed && opts.VersionID != "" && deleteMarker { - errResp := ErrorResponse{ - StatusCode: resp.StatusCode, - Code: MethodNotAllowed, - Message: s3ErrorResponseMap[MethodNotAllowed], - BucketName: bucketName, - Key: objectName, - } - return ObjectInfo{ - VersionID: resp.Header.Get(amzVersionID), - IsDeleteMarker: deleteMarker, - }, errResp + if resp.StatusCode == http.StatusMethodNotAllowed && opts.VersionID != "" && deleteMarker { + errResp := ErrorResponse{ + StatusCode: resp.StatusCode, + Code: MethodNotAllowed, + Message: s3ErrorResponseMap[MethodNotAllowed], + BucketName: bucketName, + Key: objectName, } return ObjectInfo{ - VersionID: resp.Header.Get(amzVersionID), - IsDeleteMarker: deleteMarker, - ReplicationReady: replicationReady, // whether delete marker can be replicated - }, httpRespToErrorResponse(resp, bucketName, objectName) + VersionID: resp.Header.Get(amzVersionID), + IsDeleteMarker: deleteMarker, + }, errResp } + return ObjectInfo{ + VersionID: resp.Header.Get(amzVersionID), + IsDeleteMarker: deleteMarker, + ReplicationReady: replicationReady, // whether delete marker can be replicated + }, err } return ToObjectInfo(bucketName, objectName, resp.Header) diff --git a/src/vendor/github.com/minio/minio-go/v7/api.go b/src/vendor/github.com/minio/minio-go/v7/api.go index dbf639ab..f7241af7 100644 --- a/src/vendor/github.com/minio/minio-go/v7/api.go +++ b/src/vendor/github.com/minio/minio-go/v7/api.go @@ -175,7 +175,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.98" + libraryVersion = "v7.3.0" ) // User Agent should always following the below style. @@ -403,7 +403,7 @@ func (c *Client) TraceOff() { // SetS3TransferAccelerate - turns s3 accelerated endpoint on or off for all your // requests. This feature is only specific to S3 for all other endpoints this // function does nothing. To read further details on s3 transfer acceleration -// please vist - +// please visit - // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) { if s3utils.IsAmazonEndpoint(*c.endpointURL) { @@ -554,7 +554,7 @@ type requestMetadata struct { } // dumpHTTP - dump HTTP request and response. -func (c *Client) dumpHTTP(req *http.Request, resp *http.Response) error { +func (c *Client) dumpHTTP(req *http.Request, resp *http.Response, doErr error) error { // Starts http dump. _, err := fmt.Fprintln(c.traceOutput, "---------START-HTTP---------") if err != nil { @@ -568,6 +568,9 @@ func (c *Client) dumpHTTP(req *http.Request, resp *http.Response) error { } // Only display request header. + if req.Body != nil { + req.Body = http.NoBody + } reqTrace, err := httputil.DumpRequestOut(req, false) if err != nil { return err @@ -579,28 +582,36 @@ func (c *Client) dumpHTTP(req *http.Request, resp *http.Response) error { return err } - // Only display response header. - var respTrace []byte + if resp != nil { + // Only display response header. + var respTrace []byte - // For errors we make sure to dump response body as well. - if resp.StatusCode != http.StatusOK && - resp.StatusCode != http.StatusPartialContent && - resp.StatusCode != http.StatusNoContent { - respTrace, err = httputil.DumpResponse(resp, true) - if err != nil { - return err + // For errors we make sure to dump response body as well. + if !successStatus.Contains(resp.StatusCode) { + respTrace, err = httputil.DumpResponse(resp, true) + if err != nil { + return err + } + } else { + respTrace, err = httputil.DumpResponse(resp, false) + if err != nil { + return err + } } - } else { - respTrace, err = httputil.DumpResponse(resp, false) + + // Write response to trace output. + _, err = fmt.Fprint(c.traceOutput, strings.TrimSuffix(string(respTrace), "\r\n")) if err != nil { return err } } - // Write response to trace output. - _, err = fmt.Fprint(c.traceOutput, strings.TrimSuffix(string(respTrace), "\r\n")) - if err != nil { - return err + if doErr != nil { + // Write error to trace output. + _, err = fmt.Fprintln(c.traceOutput, strings.TrimSuffix(string(doErr.Error()), "\r\n")) + if err != nil { + return err + } } // Ends the http dump. @@ -623,6 +634,9 @@ func (c *Client) do(req *http.Request) (resp *http.Response, err error) { resp, err = c.httpClient.Do(req) if err != nil { + if c.isTraceEnabled { + _ = c.dumpHTTP(req, nil, err) + } // Handle this specifically for now until future Golang versions fix this issue properly. if urlErr, ok := err.(*url.Error); ok { if strings.Contains(urlErr.Err.Error(), "EOF") { @@ -643,9 +657,9 @@ func (c *Client) do(req *http.Request) (resp *http.Response, err error) { } // If trace is enabled, dump http request and response, - // except when the traceErrorsOnly enabled and the response's status code is ok - if c.isTraceEnabled && (!c.traceErrorsOnly || resp.StatusCode != http.StatusOK) { - err = c.dumpHTTP(req, resp) + // except when traceErrorsOnly is enabled and the response has a success status code + if c.isTraceEnabled && (!c.traceErrorsOnly || !successStatus.Contains(resp.StatusCode)) { + err = c.dumpHTTP(req, resp, nil) if err != nil { return nil, err } @@ -657,6 +671,7 @@ func (c *Client) do(req *http.Request) (resp *http.Response, err error) { // List of success status. var successStatus = set.CreateIntSet( http.StatusOK, + http.StatusAccepted, http.StatusNoContent, http.StatusPartialContent, ) @@ -833,8 +848,24 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ return res, err } +// credsRetrievalPanic carries a panic value out of the de-duplicated +// credential retrieval goroutine so newRequest can resume it on each +// caller's goroutine. The value is re-raised verbatim, as if the provider +// had panicked on the caller's goroutine; the retrieval goroutine's stack +// is not carried along. +type credsRetrievalPanic struct{ value any } + +func (p credsRetrievalPanic) Error() string { + return fmt.Sprintf("credentials retrieval panicked: %v", p.value) +} + // newRequest - instantiate a new HTTP request for a given method. func (c *Client) newRequest(ctx context.Context, method string, metadata requestMetadata) (req *http.Request, err error) { + if ctx == nil { + // A nil context would be dereferenced below — by the bucket + // location lookup, the waiter select, and context.WithoutCancel. + return nil, errInvalidArgument("context cannot be nil") + } // If no method is supplied default to 'POST'. if method == "" { method = http.MethodPost @@ -867,23 +898,88 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request return nil, err } - if c.httpTrace != nil { - ctx = httptrace.WithClientTrace(ctx, c.httpTrace) - } - - // make sure to de-dup calls to credential services, this reduces - // the overall load to the endpoint generating credential service. - value, err, _ := c.credsGroup.Do(metadata.bucketName, func() (credentials.Value, error) { - if s3utils.IsS3ExpressBucket(metadata.bucketName) && s3utils.IsAmazonEndpoint(*c.endpointURL) { - return c.CreateSession(ctx, metadata.bucketName, SessionReadWrite) + // Cached, unexpired credentials (or an absent provider — anonymous + // access) are served inline; a retrieval that races expiry runs + // inline too, with the caller's live context, serialized by the + // Credentials mutex. A still-fresh cached S3 Express session is + // served inline as well; actual retrievals go through the de-dup + // group. + express := s3utils.IsS3ExpressBucket(metadata.bucketName) && s3utils.IsAmazonEndpoint(*c.endpointURL) + var value credentials.Value + var served bool + if express { + value, served = c.sessionFromCache(metadata.bucketName) + } else if c.credsProvider == nil || !c.credsProvider.IsExpired() { + value, err = c.credsProvider.GetWithContext(c.credContext(ctx)) + served = true + } + if !served { + // The provider retrieval is detached (context.WithoutCancel) so + // one caller's cancellation cannot fail concurrent waiters; each + // waiter stops waiting when its own context ends, though a + // caller arriving mid-retrieval blocks in IsExpired on the + // Credentials mutex until the retrieval completes. Detachment + // strips the caller's deadline along with its cancellation: on + // this path the caller context governs only the wait, and reaches + // the credential request itself only outside the group — on the + // inline not-expired path above and the direct call sites + // (presign, bucket location, express CreateSession). The S3 + // Express session request keeps the caller context: a full S3 + // operation's retries must stay cancellable, so its waiters + // share the winner's fate, and its trace for the same reason: + // the round trip was traced before this change. + // A retrieval panic resumes on each waiting caller's goroutine + // (credsRetrievalPanic); a runtime.Goexit reaches waiters as a + // terminal error from the de-dup group, since the retrieval + // goroutine cannot return one itself. + retrieveCtx := ctx + if express { + if c.httpTrace != nil { + retrieveCtx = httptrace.WithClientTrace(retrieveCtx, c.httpTrace) + } + } else { + retrieveCtx = context.WithoutCancel(ctx) + } + resCh := c.credsGroup.DoChan(metadata.bucketName, func() (v credentials.Value, rerr error) { + defer func() { + if r := recover(); r != nil { + rerr = credsRetrievalPanic{value: r} + } + }() + if express { + return c.CreateSession(retrieveCtx, metadata.bucketName, SessionReadWrite) + } + // Get credentials from the configured credentials provider. + return c.credsProvider.GetWithContext(c.credContext(retrieveCtx)) + }) + var res singleflight.Result[credentials.Value] + select { + case res = <-resCh: + case <-ctx.Done(): + // A result already delivered when the cancellation fires is + // preferred over the caller's context error. + select { + case res = <-resCh: + default: + return nil, ctx.Err() + } + } + var cp credsRetrievalPanic + if errors.As(res.Err, &cp) { + panic(cp.value) } - // Get credentials from the configured credentials provider. - return c.credsProvider.GetWithContext(c.CredContext()) - }) + value, err = res.Val, res.Err + } if err != nil { return nil, err } + // Attach the trace after retrieval: provider credential requests were + // never traced. The express session request is traced above. + if c.httpTrace != nil { + ctx = httptrace.WithClientTrace(ctx, c.httpTrace) + } + // Initialize a new HTTP request for the method. req, err = http.NewRequestWithContext(ctx, method, targetURL.String(), nil) if err != nil { @@ -1157,6 +1253,14 @@ func (c *Client) CredContext() *credentials.CredContext { } } +// credContext returns the client's CredContext with ctx attached as the +// caller context for credential retrieval. +func (c *Client) credContext(ctx context.Context) *credentials.CredContext { + cc := c.CredContext() + cc.Context = ctx + return cc +} + // GetCreds returns the access creds for the client func (c *Client) GetCreds() (credentials.Value, error) { if c.credsProvider == nil { diff --git a/src/vendor/github.com/minio/minio-go/v7/bucket-cache.go b/src/vendor/github.com/minio/minio-go/v7/bucket-cache.go index 7c649728..50b20c4d 100644 --- a/src/vendor/github.com/minio/minio-go/v7/bucket-cache.go +++ b/src/vendor/github.com/minio/minio-go/v7/bucket-cache.go @@ -76,33 +76,35 @@ func (c *Client) getBucketLocation(ctx context.Context, bucketName string) (stri // processes the getBucketLocation http response from the server. func processBucketLocationResponse(resp *http.Response, bucketName string) (bucketLocation string, err error) { - if resp != nil { - if resp.StatusCode != http.StatusOK { - err = httpRespToErrorResponse(resp, bucketName, "") - errResp := ToErrorResponse(err) - // For access denied error, it could be an anonymous - // request. Move forward and let the top level callers - // succeed if possible based on their policy. - switch errResp.Code { - case NotImplemented: - switch errResp.Server { - case "AmazonSnowball": - return "snowball", nil - case "cloudflare": - return "us-east-1", nil - } - case AuthorizationHeaderMalformed: - fallthrough - case InvalidRegion: - fallthrough - case AccessDenied: - if errResp.Region == "" { - return "us-east-1", nil - } - return errResp.Region, nil + if resp == nil { + return "", errInvalidArgument("Empty http response. " + reportIssue) + } + + if resp.StatusCode != http.StatusOK { + err = httpRespToErrorResponse(resp, bucketName, "") + errResp := ToErrorResponse(err) + // For access denied error, it could be an anonymous + // request. Move forward and let the top level callers + // succeed if possible based on their policy. + switch errResp.Code { + case NotImplemented: + switch errResp.Server { + case "AmazonSnowball": + return "snowball", nil + case "cloudflare": + return "us-east-1", nil } - return "", err + case AuthorizationHeaderMalformed: + fallthrough + case InvalidRegion: + fallthrough + case AccessDenied: + if errResp.Region == "" { + return "us-east-1", nil + } + return errResp.Region, nil } + return "", err } // Extract location. @@ -170,7 +172,7 @@ func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string c.setUserAgent(req) // Get credentials from the configured credentials provider. - value, err := c.credsProvider.GetWithContext(c.CredContext()) + value, err := c.credsProvider.GetWithContext(c.credContext(ctx)) if err != nil { return nil, err } diff --git a/src/vendor/github.com/minio/minio-go/v7/checksum.go b/src/vendor/github.com/minio/minio-go/v7/checksum.go index bb3ce27b..d6324b1c 100644 --- a/src/vendor/github.com/minio/minio-go/v7/checksum.go +++ b/src/vendor/github.com/minio/minio-go/v7/checksum.go @@ -25,6 +25,7 @@ import ( "encoding/base64" "encoding/binary" "errors" + "fmt" "hash" "io" "math/bits" @@ -316,6 +317,86 @@ func (c ChecksumType) String() string { return "" } +// checksumVerifyingReader verifies the checksum of data as it is read. +type checksumVerifyingReader struct { + io.ReadCloser + hash.Hash + expectChecksum string +} + +// newChecksumVerifyingReader returns a checksum-verifying reader for obj, +// wrapping the given response body, or nil if obj carries no usable checksum. +func (c *Client) newChecksumVerifyingReader(obj ObjectInfo) *checksumVerifyingReader { + switch { + case obj.ChecksumCRC32 != "": + return &checksumVerifyingReader{Hash: ChecksumCRC32.Hasher(), expectChecksum: obj.ChecksumCRC32} + case obj.ChecksumCRC32C != "": + return &checksumVerifyingReader{Hash: ChecksumCRC32C.Hasher(), expectChecksum: obj.ChecksumCRC32C} + case obj.ChecksumSHA1 != "": + return &checksumVerifyingReader{Hash: ChecksumSHA1.Hasher(), expectChecksum: obj.ChecksumSHA1} + case obj.ChecksumSHA256 != "": + return &checksumVerifyingReader{Hash: c.sha256Hasher(), expectChecksum: obj.ChecksumSHA256} + case obj.ChecksumCRC64NVME != "": + return &checksumVerifyingReader{Hash: ChecksumCRC64NVME.Hasher(), expectChecksum: obj.ChecksumCRC64NVME} + case obj.ChecksumMD5 != "": + return &checksumVerifyingReader{Hash: c.md5Hasher(), expectChecksum: obj.ChecksumMD5} + case obj.ChecksumSHA512 != "": + return &checksumVerifyingReader{Hash: ChecksumSHA512.Hasher(), expectChecksum: obj.ChecksumSHA512} + case obj.ChecksumXXHash64 != "": + return &checksumVerifyingReader{Hash: ChecksumXXHash64.Hasher(), expectChecksum: obj.ChecksumXXHash64} + case obj.ChecksumXXHash3 != "": + return &checksumVerifyingReader{Hash: ChecksumXXHash3.Hasher(), expectChecksum: obj.ChecksumXXHash3} + case obj.ChecksumXXHash128 != "": + return &checksumVerifyingReader{Hash: ChecksumXXHash128.Hasher(), expectChecksum: obj.ChecksumXXHash128} + default: + return nil + } +} + +func (c *checksumVerifyingReader) SetReader(r io.ReadCloser) { + c.ReadCloser = r +} + +// Close returns any pooled hasher and closes the underlying reader if one is set. +func (c *checksumVerifyingReader) Close() error { + if closer, ok := c.Hash.(interface{ Close() }); ok { + closer.Close() + } + if c.ReadCloser == nil { + return nil + } + return c.ReadCloser.Close() +} + +// Read reads data and hashes it. At EOF the checksum is verified; on +// mismatch the mismatch error is returned in place of io.EOF. +func (c *checksumVerifyingReader) Read(p []byte) (int, error) { + n, err := c.ReadCloser.Read(p) + if n > 0 { + n2, werr := c.Write(p[:n]) + if werr != nil { + return n, werr + } + if n2 != n { + return n, io.ErrShortWrite + } + } + if err == io.EOF { + if verr := c.VerifyChecksum(); verr != nil { + return n, verr + } + } + return n, err +} + +// VerifyChecksum returns an error if the computed checksum does not match. +func (c *checksumVerifyingReader) VerifyChecksum() error { + if got := base64.StdEncoding.EncodeToString(c.Sum(nil)); got != c.expectChecksum { + return fmt.Errorf("checksum mismatch, expected %s, got %s", c.expectChecksum, got) + } + return nil +} + // ChecksumReader reads all of r and returns a checksum of type c. // Returns any error that may have occurred while reading. func (c ChecksumType) ChecksumReader(r io.Reader) (Checksum, error) { diff --git a/src/vendor/github.com/minio/minio-go/v7/create-session.go b/src/vendor/github.com/minio/minio-go/v7/create-session.go index d85e79d4..3f1b3175 100644 --- a/src/vendor/github.com/minio/minio-go/v7/create-session.go +++ b/src/vendor/github.com/minio/minio-go/v7/create-session.go @@ -51,19 +51,30 @@ type createSessionResult struct { } `xml:",omitempty"` } +// expressSessionRenewalLeeway is how long before its expiration a cached +// S3 Express session stops being served, leaving room for renewal. +const expressSessionRenewalLeeway = 10 * time.Second + +// sessionFromCache returns the cached S3 Express session credentials for +// bucketName while they remain outside the renewal leeway. +func (c *Client) sessionFromCache(bucketName string) (credentials.Value, bool) { + v, ok := c.bucketSessionCache.Get(bucketName) + if !ok || !v.Expiration.After(time.Now().Add(expressSessionRenewalLeeway)) { + return credentials.Value{}, false + } + return v, true +} + // CreateSession - https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateSession.html // the returning credentials may be cached depending on the expiration of the original -// credential, credentials will get renewed 10 secs earlier than when its gonna expire -// allowing for some leeway in the renewal process. +// credential; a cached session stops being served expressSessionRenewalLeeway before +// it expires, allowing for some leeway in the renewal process. func (c *Client) CreateSession(ctx context.Context, bucketName string, sessionMode SessionMode) (cred credentials.Value, err error) { if err := s3utils.CheckValidBucketNameS3Express(bucketName); err != nil { return credentials.Value{}, err } - v, ok := c.bucketSessionCache.Get(bucketName) - if ok && v.Expiration.After(time.Now().Add(10*time.Second)) { - // Verify if the credentials will not expire - // in another 10 seconds, if not we renew it again. + if v, ok := c.sessionFromCache(bucketName); ok { return v, nil } @@ -143,7 +154,7 @@ func (c *Client) createSessionRequest(ctx context.Context, bucketName string, se c.setUserAgent(req) // Get credentials from the configured credentials provider. - value, err := c.credsProvider.GetWithContext(c.CredContext()) + value, err := c.credsProvider.GetWithContext(c.credContext(ctx)) if err != nil { return nil, err } diff --git a/src/vendor/github.com/minio/minio-go/v7/functional_tests.go b/src/vendor/github.com/minio/minio-go/v7/functional_tests.go index ef5f2377..80d98955 100644 --- a/src/vendor/github.com/minio/minio-go/v7/functional_tests.go +++ b/src/vendor/github.com/minio/minio-go/v7/functional_tests.go @@ -172,9 +172,12 @@ func logError(testName, function string, args map[string]interface{}, startTime // If server returns NotImplemented we assume it is gateway mode and hence log it as info and move on to next tests // Special case for ComposeObject API as it is implemented on client side and adds specific error details like `Error in upload-part-copy` in // addition to NotImplemented error returned from server - if isErrNotImplemented(err) { + switch { + case isErrNotImplemented(err): logIgnored(testName, function, args, startTime, message) - } else { + case isErrFreeTierLicense(err): + logNotAvailable(testName, function, args, startTime, message) + default: logFailure(testName, function, args, startTime, alert, message, err) if !isRunOnFail() { panic(fmt.Sprintf("Test failed with message: %s, err: %v", message, err)) @@ -206,6 +209,15 @@ func logIgnored(testName, function string, args map[string]interface{}, startTim ).Info("") } +// log not applicable test runs for license-gated features +func logNotAvailable(testName, function string, args map[string]interface{}, startTime time.Time, alert string) { + baseLogger(testName, function, args, startTime). + With( + "status", "NA", + "alert", strings.Split(alert, " ")[0]+" requires a paid-tier license", + ).Info("") +} + // Delete objects in given bucket, recursively func cleanupBucket(bucketName string, c *minio.Client) error { // Create a done channel to control 'ListObjectsV2' go routine. @@ -279,6 +291,11 @@ func isErrNotImplemented(err error) bool { return minio.ToErrorResponse(err).Code == minio.NotImplemented } +// isErrFreeTierLicense reports whether the server rejected the request because it requires a paid-tier license. +func isErrFreeTierLicense(err error) bool { + return minio.ToErrorResponse(err).Code == minio.XMinioPaidTierLicenseRequired +} + func isRunOnFail() bool { return os.Getenv("RUN_ON_FAIL") == "1" } @@ -795,6 +812,125 @@ func testListObjectVersions() { logSuccess(testName, function, args, startTime) } +func testStatObjectResponseHeaders() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "StatObject(bucketName, objectName, opts)" + args := map[string]interface{}{} + + c, err := NewClient(ClientConfig{}) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + defer cleanupBucket(bucketName, c) + + bufSize := dataFileMap["datafile-33-kB"] + reader := getDataReader("datafile-33-kB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + opts := minio.StatObjectOptions{} + err = opts.SetRange(0, 99) + if err != nil { + logError(testName, function, args, startTime, "", "SetRange failed", err) + return + } + args["range"] = "0-99" + + st, err := c.StatObject(context.Background(), bucketName, objectName, opts) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + + if st.Size != 100 { + logError(testName, function, args, startTime, "", "StatObject ranged size mismatch", fmt.Errorf("got %d, want 100", st.Size)) + return + } + wantContentRange := fmt.Sprintf("bytes 0-99/%d", bufSize) + gotContentRange := st.Headers.Get("Content-Range") + if gotContentRange != wantContentRange { + logError(testName, function, args, startTime, "", "StatObject response headers Content-Range mismatch", fmt.Errorf("got %q, want %q", gotContentRange, wantContentRange)) + return + } + if st.Headers.Get("ETag") == "" { + logError(testName, function, args, startTime, "", "StatObject response headers missing ETag", errors.New("empty ETag header")) + return + } + + gopts := minio.GetObjectOptions{} + err = gopts.SetRange(0, 99) + if err != nil { + logError(testName, function, args, startTime, "", "SetRange failed", err) + return + } + obj, err := c.GetObject(context.Background(), bucketName, objectName, gopts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + defer obj.Close() + // Read first: a first-op Stat issues an un-ranged StatObject by design, + // so the ranged GET response headers are observable only after a read. + buf := make([]byte, 100) + if _, err = io.ReadFull(obj, buf); err != nil { + logError(testName, function, args, startTime, "", "GetObject ranged read failed", err) + return + } + gst, err := obj.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "GetObject Stat failed", err) + return + } + if got := gst.Headers.Get("Content-Range"); got != wantContentRange { + logError(testName, function, args, startTime, "", "GetObject response headers Content-Range mismatch", fmt.Errorf("got %q, want %q", got, wantContentRange)) + return + } + + sawObject := false + for listInfo := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{}) { + if listInfo.Err != nil { + logError(testName, function, args, startTime, "", "ListObjects failed", listInfo.Err) + return + } + if listInfo.Key == objectName { + sawObject = true + } + if listInfo.Headers != nil { + logError(testName, function, args, startTime, "", "ListObjects ObjectInfo.Headers expected nil", fmt.Errorf("got %v", listInfo.Headers)) + return + } + } + if !sawObject { + logError(testName, function, args, startTime, "", "ListObjects did not list the uploaded object", errors.New("uploaded object missing from listing")) + return + } + + logSuccess(testName, function, args, startTime) +} + func testStatObjectWithVersioning() { // initialize logging params startTime := time.Now() @@ -5379,7 +5515,7 @@ func testGetObjectReadSeekFunctional() { } if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+strconv.Itoa(bufSize)+", got "+strconv.FormatInt(st.Size, 10), err) return } @@ -5418,50 +5554,77 @@ func testGetObjectReadSeekFunctional() { {0, 0, 0, nil, true, 0, 0}, // Start from offset 2048, fetch data and compare {2048, 0, 2048, nil, true, 2048, bufSize}, - // Start from offset larger than possible - {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0}, + // Start from offset larger than possible: Seek rejects it with io.EOF + // and keeps the offset unchanged. + {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, // Move to offset 0 without comparing {0, 0, 0, nil, false, 0, 0}, // Move one step forward and compare {1, 1, 1, nil, true, 1, bufSize}, - // Move larger than possible - {int64(bufSize), 1, 0, seekErr, false, 0, 0}, - // Provide negative offset with CUR_SEEK - {int64(-1), 1, 0, seekErr, false, 0, 0}, - // Test with whence SEEK_END and with positive offset - {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0}, + // Move larger than possible: Seek returns io.EOF. + {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, + // Provide negative offset with CUR_SEEK: valid since the resulting + // absolute position is within the object. + {int64(-1), 1, int64(bufSize) - 1, nil, false, 0, 0}, + // Test with whence SEEK_END and with positive offset: Seek returns + // io.EOF. + {1024, 2, 0, io.EOF, false, 0, 0}, // Test with whence SEEK_END and with negative offset {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, // Test with whence SEEK_END and with large negative offset {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0}, + // Seek to exactly the object end; the subsequent read reports io.EOF. + {0, 2, int64(bufSize), nil, false, 0, 0}, } for i, testCase := range testCases { + // Snapshot the cursor so rejected seeks can be asserted side-effect + // free. + prevPos, err := r.Seek(0, io.SeekCurrent) + if err != nil { + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, reading the current position failed", i+1), err) + return + } // Perform seek operation n, err := r.Seek(testCase.offset, testCase.whence) // We expect an error if testCase.err == seekErr && err == nil { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %v, found: %v", i+1, testCase.err, err), err) return } // We expect a specific error if testCase.err != seekErr && testCase.err != err { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %v, found: %v", i+1, testCase.err, err), err) return } // If we expect an error go to the next loop if testCase.err != nil { + // A rejected seek must leave the cursor unchanged. + curPos, cerr := r.Seek(0, io.SeekCurrent) + if cerr != nil || curPos != prevPos { + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, cursor after rejected seek: expected %d, got %d", i+1, prevPos, curPos), cerr) + return + } continue } // Check the returned seek pos if n != testCase.pos { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", number of bytes seeked does not match, expected "+string(testCase.pos)+", got "+string(n), err) + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) return } // Compare only if shouldCmp is activated if testCase.shouldCmp { cmpData(r, testCase.start, testCase.end) } + // A successful seek to the object end must defer io.EOF to the + // subsequent read. + if !testCase.shouldCmp && testCase.err == nil && testCase.pos >= int64(bufSize) { + readN, readErr := r.Read(make([]byte, 1)) + if readN != 0 || readErr != io.EOF { + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, read after seek to object end expected zero bytes and io.EOF, got %d bytes and %v", i+1, readN, readErr), readErr) + return + } + } } logSuccess(testName, function, args, startTime) } @@ -5569,8 +5732,8 @@ func testGetObjectReadAtFunctional() { return } - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + if st.Size != (int64(bufSize) - int64(len(bufRead))) { + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+strconv.Itoa(bufSize)+", got "+strconv.Itoa(bufSize-len(bufRead)), err) return } @@ -5665,6 +5828,228 @@ func testGetObjectReadAtFunctional() { logSuccess(testName, function, args, startTime) } +// testGetObjectWithRange - get object with range +func testGetObjectWithRange() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + c, err := NewClient(ClientConfig{}) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + defer cleanupBucket(bucketName, c) + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + reader := getDataReader("datafile-33-kB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := io.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + var r *minio.Object + testIndex := 0 + baseSize := 0 + New := func() { + opts := minio.GetObjectOptions{} + switch testIndex { + case 0: + baseSize = bufSize + case 1: + opts.SetRange(100, 1000) + baseSize = 1000 - 100 + 1 + case 2: + opts.SetRange(100, 0) + baseSize = bufSize - 100 + case 3: + opts.SetRange(0, 1000) + baseSize = 1000 + 1 + } + r, err = c.GetObject(context.Background(), bucketName, objectName, opts) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to create MinIO client object", err) + } + } + + Size := func(size int) { + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Failed to get object stat", err) + } + if int(st.Size) != size { + logError(testName, function, args, startTime, "", "Incorrect size returned", fmt.Errorf("Test index %d Expected size %d, got %d", testIndex, size, int(st.Size))) + } + } + Read := func(size int) { + b := make([]byte, size) + _, err := r.Read(b) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to read from object", err) + } + } + + ReadFull := func(size int) { + b := make([]byte, size) + _, err := r.Read(b) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to read from object", err) + } + } + + ReadAll := func() { + _, err := io.ReadAll(r) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to read from object", err) + } + } + + Seek := func(offset int64, whence int) { + _, err := r.Seek(offset, whence) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to seek in object", err) + } + } + ReadAt := func(offset int64, size int64) { + b := make([]byte, size) + _, err := r.ReadAt(b, offset) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to read from object", err) + } + } + for index := range 4 { + testIndex = index + // case 1: stat first and then read 100 bytes for per read + { + New() + Size(baseSize) + Read(100) + Size(baseSize - 100) + Read(100) + Size(baseSize - 200) + Read(100) + Size(baseSize - 300) + ReadFull(100) + Size(baseSize - 400) + } + // case 2: read 100 bytes for per read + { + New() + Read(100) + Size(baseSize - 100) + Read(100) + Size(baseSize - 200) + Read(100) + Size(baseSize - 300) + ReadFull(100) + Size(baseSize - 400) + } + // case 3: Stat -> Read -> ReadAt -> Read + { + New() + Size(baseSize) + Read(100) + Size(baseSize - 100) + // should not move the offset, so next stat is not changed + ReadAt(100, 100) + Size(baseSize - 100) + Read(100) + Size(baseSize - 200) + } + // case 4: Read -> ReadAt -> Read + { + New() + Read(100) + Size(baseSize - 100) + // should not move the offset, so next stat is not changed + ReadAt(100, 100) + Size(baseSize - 100) + Read(100) + Size(baseSize - 200) + } + // case 5: Stat -> Read -> ReadAt -> Read -> Seek -> Read + { + New() + Size(baseSize) + Read(100) + Size(baseSize - 100) + // should not move the offset, so next stat is not changed + ReadAt(100, 100) + Size(baseSize - 100) + Read(100) + Size(baseSize - 200) + Seek(100, io.SeekCurrent) + Size(baseSize - 300) + Read(100) + Size(baseSize - 400) + } + // case 6: Read -> ReadAt -> Read -> Seek -> Read + { + New() + Read(100) + Size(baseSize - 100) + // should not move the offset, so next stat is not changed + ReadAt(100, 100) + Size(baseSize - 100) + Read(100) + Size(baseSize - 200) + Seek(100, io.SeekCurrent) + Size(baseSize - 300) + Read(100) + Size(baseSize - 400) + } + // case 7: Stat -> ReadAll -> ReadAt + { + New() + Size(baseSize) + ReadAll() + ReadAt(100, 100) + } + // case 8: ReadAll -> ReadAt + { + New() + ReadAll() + ReadAt(100, 100) + } + // case9: Seek first + { + New() + Seek(100, io.SeekCurrent) + Size(baseSize - 100) + Read(100) + Size(baseSize - 200) + } + } +} + // Reproduces issue https://github.com/minio/minio-go/issues/1137 func testGetObjectReadAtWhenEOFWasReached() { // initialize logging params @@ -5746,7 +6131,7 @@ func testGetObjectReadAtWhenEOFWasReached() { return } - if st.Size != int64(bufSize) { + if st.Size != (int64(bufSize) - int64(len(buf1))) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } @@ -6527,7 +6912,7 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { } if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+strconv.Itoa(bufSize)+", got "+strconv.FormatInt(st.Size, 10), err) return } @@ -6563,17 +6948,20 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { {0, 0, 0, nil, true, 0, 0}, // Start from offset 2048, fetch data and compare {2048, 0, 2048, nil, true, 2048, bufSize}, - // Start from offset larger than possible + // Start from offset larger than possible: Seek rejects it with io.EOF + // and keeps the offset unchanged. {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, // Move to offset 0 without comparing {0, 0, 0, nil, false, 0, 0}, // Move one step forward and compare {1, 1, 1, nil, true, 1, bufSize}, - // Move larger than possible + // Move larger than possible: Seek returns io.EOF. {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, - // Provide negative offset with CUR_SEEK - {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, - // Test with whence SEEK_END and with positive offset + // Provide negative offset with CUR_SEEK: valid since the resulting + // absolute position is within the object. + {int64(-1), 1, int64(bufSize) - 1, nil, false, 0, 0}, + // Test with whence SEEK_END and with positive offset: Seek returns + // io.EOF. {1024, 2, 0, io.EOF, false, 0, 0}, // Test with whence SEEK_END and with negative offset {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, @@ -6581,9 +6969,18 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, // Test with invalid whence {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, + // Seek to exactly the object end; the subsequent read reports io.EOF. + {0, 2, int64(bufSize), nil, false, 0, 0}, } for i, testCase := range testCases { + // Snapshot the cursor so rejected seeks can be asserted side-effect + // free. + prevPos, err := r.Seek(0, io.SeekCurrent) + if err != nil { + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, reading the current position failed", i+1), err) + return + } // Perform seek operation n, err := r.Seek(testCase.offset, testCase.whence) if err != nil && testCase.err == nil { @@ -6605,6 +7002,12 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } + // A rejected seek must leave the cursor unchanged. + curPos, cerr := r.Seek(0, io.SeekCurrent) + if cerr != nil || curPos != prevPos { + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, cursor after rejected seek: expected %d, got %d", i+1, prevPos, curPos), cerr) + return + } } // Check the returned seek pos if n != testCase.pos { @@ -6616,6 +7019,15 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { if testCase.shouldCmp { cmpData(r, testCase.start, testCase.end) } + // A successful seek to the object end must defer io.EOF to the + // subsequent read. + if !testCase.shouldCmp && testCase.err == nil && testCase.pos >= int64(bufSize) { + readN, readErr := r.Read(make([]byte, 1)) + if readN != 0 || readErr != io.EOF { + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, read after seek to object end expected zero bytes and io.EOF, got %d bytes and %v", i+1, readN, readErr), readErr) + return + } + } } logSuccess(testName, function, args, startTime) @@ -6693,7 +7105,7 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { } if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+strconv.Itoa(bufSize)+", got "+strconv.FormatInt(st.Size, 10), err) return } @@ -6729,17 +7141,20 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { {0, 0, 0, nil, true, 0, 0}, // Start from offset 2048, fetch data and compare {2048, 0, 2048, nil, true, 2048, bufSize}, - // Start from offset larger than possible + // Start from offset larger than possible: Seek rejects it with io.EOF + // and keeps the offset unchanged. {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, // Move to offset 0 without comparing {0, 0, 0, nil, false, 0, 0}, // Move one step forward and compare {1, 1, 1, nil, true, 1, bufSize}, - // Move larger than possible + // Move larger than possible: Seek returns io.EOF. {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, - // Provide negative offset with CUR_SEEK - {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, - // Test with whence SEEK_END and with positive offset + // Provide negative offset with CUR_SEEK: valid since the resulting + // absolute position is within the object. + {int64(-1), 1, int64(bufSize) - 1, nil, false, 0, 0}, + // Test with whence SEEK_END and with positive offset: Seek returns + // io.EOF. {1024, 2, 0, io.EOF, false, 0, 0}, // Test with whence SEEK_END and with negative offset {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, @@ -6747,9 +7162,18 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, // Test with invalid whence {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, + // Seek to exactly the object end; the subsequent read reports io.EOF. + {0, 2, int64(bufSize), nil, false, 0, 0}, } for i, testCase := range testCases { + // Snapshot the cursor so rejected seeks can be asserted side-effect + // free. + prevPos, err := r.Seek(0, io.SeekCurrent) + if err != nil { + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, reading the current position failed", i+1), err) + return + } // Perform seek operation n, err := r.Seek(testCase.offset, testCase.whence) if err != nil && testCase.err == nil { @@ -6771,6 +7195,12 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } + // A rejected seek must leave the cursor unchanged. + curPos, cerr := r.Seek(0, io.SeekCurrent) + if cerr != nil || curPos != prevPos { + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, cursor after rejected seek: expected %d, got %d", i+1, prevPos, curPos), cerr) + return + } } // Check the returned seek pos if n != testCase.pos { @@ -6782,6 +7212,15 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { if testCase.shouldCmp { cmpData(r, testCase.start, testCase.end) } + // A successful seek to the object end must defer io.EOF to the + // subsequent read. + if !testCase.shouldCmp && testCase.err == nil && testCase.pos >= int64(bufSize) { + readN, readErr := r.Read(make([]byte, 1)) + if readN != 0 || readErr != io.EOF { + logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, read after seek to object end expected zero bytes and io.EOF, got %d bytes and %v", i+1, readN, readErr), readErr) + return + } + } } logSuccess(testName, function, args, startTime) @@ -6879,7 +7318,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() { } if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+strconv.Itoa(bufSize)+", got "+strconv.Itoa(int(st.Size)), err) return } @@ -8856,7 +9295,7 @@ func testGetObjectReadSeekFunctionalV2() { } if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+strconv.Itoa(bufSize)+" got "+strconv.FormatInt(st.Size, 10), err) return } @@ -8867,7 +9306,7 @@ func testGetObjectReadSeekFunctionalV2() { return } if n != offset { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+strconv.FormatInt(offset, 10)+" got "+strconv.FormatInt(n, 10), err) return } n, err = r.Seek(0, 1) @@ -8876,12 +9315,44 @@ func testGetObjectReadSeekFunctionalV2() { return } if n != offset { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+strconv.FormatInt(offset, 10)+" got "+strconv.FormatInt(n, 10), err) return } - _, err = r.Seek(offset, 2) - if err == nil { - logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err) + // Seeking past the object end with SEEK_END is rejected with io.EOF and + // leaves the offset unchanged. + n, err = r.Seek(offset, 2) + if err != io.EOF { + logError(testName, function, args, startTime, "", "Seek past object end should return io.EOF", err) + return + } + if n != 0 { + logError(testName, function, args, startTime, "", "Seek past object end should return offset 0, got "+strconv.FormatInt(n, 10), err) + return + } + // A rejected seek must leave the cursor unchanged. + n, err = r.Seek(0, 1) + if err != nil { + logError(testName, function, args, startTime, "", "Seek failed", err) + return + } + if n != offset { + logError(testName, function, args, startTime, "", "Cursor moved after rejected seek, expected "+strconv.FormatInt(offset, 10)+" got "+strconv.FormatInt(n, 10), err) + return + } + // Seeking to exactly the object end succeeds; the subsequent read reports + // io.EOF. + n, err = r.Seek(0, 2) + if err != nil { + logError(testName, function, args, startTime, "", "Seek failed", err) + return + } + if n != st.Size { + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+strconv.FormatInt(st.Size, 10)+" got "+strconv.FormatInt(n, 10), err) + return + } + readN, readErr := r.Read(make([]byte, 1)) + if readN != 0 || readErr != io.EOF { + logError(testName, function, args, startTime, "", "Read after seeking to object end should return zero bytes and EOF", readErr) return } n, err = r.Seek(-offset, 2) @@ -8890,7 +9361,7 @@ func testGetObjectReadSeekFunctionalV2() { return } if n != st.Size-offset { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err) + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+strconv.FormatInt(st.Size-offset, 10)+" got "+strconv.FormatInt(n, 10), err) return } @@ -8913,7 +9384,7 @@ func testGetObjectReadSeekFunctionalV2() { return } if n != (offset - 1) { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err) + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+strconv.FormatInt(offset-1, 10)+" got "+strconv.FormatInt(n, 10), err) return } @@ -15084,6 +15555,7 @@ func main() { testMakeBucketError() testMakeBucketRegions() testPutObjectWithMetadata() + testGetObjectWithRange() testPutObjectReadAt() testPutObjectStreaming() testPutObjectPreconditionOnNonExistent() @@ -15122,6 +15594,7 @@ func main() { testRemoveObjects() testRemoveObjectsIter() testListObjectVersions() + testStatObjectResponseHeaders() testStatObjectWithVersioning() testGetObjectWithVersioning() testCopyObjectWithVersioning() diff --git a/src/vendor/github.com/minio/minio-go/v7/hook-reader.go b/src/vendor/github.com/minio/minio-go/v7/hook-reader.go index 06dbbb74..0d78e9af 100644 --- a/src/vendor/github.com/minio/minio-go/v7/hook-reader.go +++ b/src/vendor/github.com/minio/minio-go/v7/hook-reader.go @@ -25,22 +25,31 @@ import ( // hookReader hooks additional reader in the source stream. It is // useful for making progress bars. Second reader is appropriately // notified about the exact number of bytes read from the primary -// source on each Read operation. +// source on each Read operation. It deliberately implements neither +// io.Seeker nor io.Closer: retry logic treats a seekable body as +// rewindable, and executeMethod closes request bodies that implement +// io.Closer — a caller-supplied reader must be shielded from both. type hookReader struct { source io.Reader hook io.Reader } +// hookReadSeeker extends hookReader with seeking support. It is +// constructed only when the source implements io.Seeker, so a wrapped +// reader exposes Seek if and only if it can actually rewind. This lets +// retry logic disable retries for non-seekable bodies instead of +// retrying over a drained reader. +type hookReadSeeker struct { + hookReader + seeker io.Seeker +} + // Seek implements io.Seeker. Seeks source first, and if necessary // seeks hook if Seek method is appropriately found. -func (hr *hookReader) Seek(offset int64, whence int) (n int64, err error) { - // Verify for source has embedded Seeker, use it. - sourceSeeker, ok := hr.source.(io.Seeker) - if ok { - n, err = sourceSeeker.Seek(offset, whence) - if err != nil { - return 0, err - } +func (hr *hookReadSeeker) Seek(offset int64, whence int) (n int64, err error) { + n, err = hr.seeker.Seek(offset, whence) + if err != nil { + return 0, err } if hr.hook != nil { @@ -53,7 +62,7 @@ func (hr *hookReader) Seek(offset int64, whence int) (n int64, err error) { return 0, err } if n != m { - return 0, fmt.Errorf("hook seeker seeked %d bytes, expected source %d bytes", m, n) + return 0, fmt.Errorf("hook seeker sought to offset %d, expected source offset %d", m, n) } } } @@ -80,14 +89,13 @@ func (hr *hookReader) Read(b []byte) (n int, err error) { return n, err } -// newHook returns a io.ReadSeeker which implements hookReader that -// reports the data read from the source to the hook. +// newHook returns an io.Reader that reports the data read from the +// source to the hook. The returned reader implements io.Seeker only +// when the source does. func newHook(source, hook io.Reader) io.Reader { - if hook == nil { - return &hookReader{source: source} - } - return &hookReader{ - source: source, - hook: hook, + hr := hookReader{source: source, hook: hook} + if seeker, ok := source.(io.Seeker); ok { + return &hookReadSeeker{hookReader: hr, seeker: seeker} } + return &hr } diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go index 415b0709..41225e4c 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go @@ -19,6 +19,7 @@ package credentials import ( "bytes" + "context" "crypto/sha256" "encoding/hex" "encoding/xml" @@ -142,7 +143,7 @@ func closeResponse(resp *http.Response) { } } -func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssumeRoleOptions) (AssumeRoleResponse, error) { +func getAssumeRoleCredentials(ctx context.Context, clnt *http.Client, endpoint string, opts STSAssumeRoleOptions) (AssumeRoleResponse, error) { v := url.Values{} v.Set("Action", "AssumeRole") v.Set("Version", STSVersion) @@ -180,7 +181,7 @@ func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssume } postBody.Seek(0, 0) - req, err := http.NewRequest(http.MethodPost, u.String(), postBody) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), postBody) if err != nil { return AssumeRoleResponse{}, err } @@ -245,7 +246,7 @@ func (m *STSAssumeRole) RetrieveWithCredContext(cc *CredContext) (Value, error) return Value{}, errors.New("STS endpoint unknown") } - a, err := getAssumeRoleCredentials(client, stsEndpoint, m.Options) + a, err := getAssumeRoleCredentials(cc.requestContext(), client, stsEndpoint, m.Options) if err != nil { return Value{}, err } diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials-sso.sample b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials-sso.sample new file mode 100644 index 00000000..46af924b --- /dev/null +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials-sso.sample @@ -0,0 +1,57 @@ +[profile p1] +sso_session = main +sso_account_id = 123456789 +sso_role_name = myrole + +[sso-session main] +sso_region = us-test-2 +sso_start_url = https://testacct.awsapps.com/start + +[profile p2-legacy] +sso_start_url = https://legacy.awsapps.com/start +sso_region = us-test-1 +sso_account_id = 987654321 +sso_role_name = legacyrole + +[profile p3-broken] +sso_account_id = 111111111 +sso_role_name = brokenrole + +[profile p4-noregion] +sso_start_url = https://noregion.awsapps.com/start +sso_account_id = 222222222 +sso_role_name = noregionrole + +[profile p5-ghost-session] +sso_session = ghost +sso_account_id = 333333333 +sso_role_name = ghostrole + +[profile p6-norole] +sso_start_url = https://norole.awsapps.com/start +sso_region = us-test-1 +sso_account_id = 444444444 + +[profile p7-noaccount] +sso_session = main +sso_role_name = noaccountrole + +[profile p8-mixed] +sso_start_url = https://mixed.awsapps.com/start +aws_access_key_id = mixedAccessKey +aws_secret_access_key = mixedSecret + +[profile p9-mixed-complete] +sso_session = main +sso_account_id = 555555555 +sso_role_name = mixedcompleterole +aws_access_key_id = completeAccessKey +aws_secret_access_key = completeSecret + +[profile p10-partial-key] +sso_start_url = https://partial.awsapps.com/start +aws_access_key_id = partialAccessKey + +[profile p11-partial-secret] +sso_session = main +aws_secret_access_key = partialSecret diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go index 52aff9a5..dfc25c07 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go @@ -18,6 +18,7 @@ package credentials import ( + "context" "net/http" "sync" "time" @@ -85,6 +86,21 @@ type CredContext struct { // Endpoint specifies the MinIO endpoint that will be used if no // explicit endpoint is provided. Endpoint string + + // Context is the optional caller context. Cancellation and deadlines + // on it propagate to the HTTP requests the built-in providers make + // to fetch credentials, in addition to any provider-level timeout + // bound. A nil Context means no caller cancellation applies. + Context context.Context +} + +// requestContext returns the caller context carried by cc, or +// context.Background() when no Context is set. +func (cc *CredContext) requestContext() context.Context { + if cc.Context == nil { + return context.Background() + } + return cc.Context } // A Expiry provides shared expiration logic to be used by credentials diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/file_aws_credentials.go b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/file_aws_credentials.go index 2d0336f9..204bcfa2 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/file_aws_credentials.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/file_aws_credentials.go @@ -18,8 +18,15 @@ package credentials import ( + "context" + "crypto/sha1" + "encoding/hex" "encoding/json" "errors" + "fmt" + "io" + "io/fs" + "net/http" "os" "os/exec" "path/filepath" @@ -29,7 +36,10 @@ import ( "gopkg.in/ini.v1" ) -// A externalProcessCredentials stores the output of a credential_process +// ssoPortalRequestTimeout bounds the AWS SSO portal role-credentials request. +const ssoPortalRequestTimeout = time.Minute + +// An externalProcessCredentials stores the output of a credential_process type externalProcessCredentials struct { Version int SessionToken string @@ -38,6 +48,34 @@ type externalProcessCredentials struct { Expiration time.Time } +// An ssoCredentials stores the response of fetching role credentials for +// an AWS SSO role from the SSO portal. +type ssoCredentials struct { + RoleCredentials ssoRoleCredentials `json:"roleCredentials"` +} + +// An ssoRoleCredentials stores the role-specific credentials portion of +// an SSO role credentials response. Expiration is milliseconds since +// the Unix epoch. +type ssoRoleCredentials struct { + AccessKeyID string `json:"accessKeyId"` + Expiration int64 `json:"expiration"` + SecretAccessKey string `json:"secretAccessKey"` + SessionToken string `json:"sessionToken"` +} + +func (s ssoRoleCredentials) expirationTime() time.Time { + return time.Unix(0, s.Expiration*int64(time.Millisecond)) +} + +// ssoCachedToken is the subset of the cached token file written by +// `aws sso login` under ~/.aws/sso/cache/ that we need. +type ssoCachedToken struct { + AccessToken string `json:"accessToken"` + ExpiresAt time.Time `json:"expiresAt"` + Region string `json:"region"` +} + // A FileAWSCredentials retrieves credentials from the current user's home // directory, and keeps track if those credentials are expired. // @@ -47,10 +85,15 @@ type FileAWSCredentials struct { // Path to the shared credentials file. // - // If empty will look for "AWS_SHARED_CREDENTIALS_FILE" env variable. If the - // env value is empty will default to current user's home directory. - // Linux/OSX: "$HOME/.aws/credentials" - // Windows: "%USERPROFILE%\.aws\credentials" + // If empty, the default AWS files are merged instead: the AWS config + // file ("AWS_CONFIG_FILE" env variable or "$HOME/.aws/config") loaded + // first, then the shared credentials file + // ("AWS_SHARED_CREDENTIALS_FILE" env variable or + // "$HOME/.aws/credentials") overriding matching keys, the same + // resolution the AWS SDK uses. Profiles written by `aws configure sso` + // live in the config file and are discovered this way. + // + // If set, only this one file is read. Filename string // AWS Profile to extract credentials from the shared credentials file. If empty @@ -60,6 +103,14 @@ type FileAWSCredentials struct { // retrieved states if the credentials have been successfully retrieved. retrieved bool + + // overrideSSOCacheDir overrides, for tests, the directory holding the + // cached SSO tokens (defaults to ~/.aws/sso/cache). + overrideSSOCacheDir string + + // overrideSSOPortalURL overrides, for tests, the AWS SSO portal URL + // serving role credentials. + overrideSSOPortalURL string } // NewFileAWSCredentials returns a pointer to a new Credentials object @@ -71,27 +122,27 @@ func NewFileAWSCredentials(filename, profile string) *Credentials { }) } -func (p *FileAWSCredentials) retrieve() (Value, error) { - if p.Filename == "" { - p.Filename = os.Getenv("AWS_SHARED_CREDENTIALS_FILE") - if p.Filename == "" { - homeDir, err := os.UserHomeDir() - if err != nil { - return Value{}, err - } - p.Filename = filepath.Join(homeDir, ".aws", "credentials") - } - } - if p.Profile == "" { - p.Profile = os.Getenv("AWS_PROFILE") - if p.Profile == "" { - p.Profile = "default" +func (p *FileAWSCredentials) retrieve(cc *CredContext) (Value, error) { + profile := p.Profile + if profile == "" { + profile = os.Getenv("AWS_PROFILE") + if profile == "" { + profile = "default" } } p.retrieved = false - iniProfile, err := loadProfile(p.Filename, p.Profile) + var ( + iniConfig *ini.File + iniProfile *ini.Section + err error + ) + if p.Filename != "" { + iniConfig, iniProfile, err = loadProfile(p.Filename, profile) + } else { + iniConfig, iniProfile, err = loadDefaultProfiles(profile) + } if err != nil { return Value{}, err } @@ -103,10 +154,53 @@ func (p *FileAWSCredentials) retrieve() (Value, error) { // Default to empty string if not found. token := iniProfile.Key("aws_session_token") - // If credential_process is defined, obtain credentials by executing - // the external process + // A complete static key pair takes precedence over SSO configuration + // and credential_process in the same profile, matching aws-sdk-go-v2's + // resolution order: static credentials, then SSO, then + // credential_process. + hasStaticKeys := id.String() != "" && secret.String() != "" + + // Exactly one half of a static key pair is never usable and poisons + // downstream consumers — Chain accepts a Value when either field is + // set — so it is rejected before any resolution, matching + // aws-sdk-go-v2's partial-credentials validation. + if !hasStaticKeys && (id.String() != "" || secret.String() != "") { + return Value{}, errors.New("profile has partial static credentials: aws_access_key_id and aws_secret_access_key must both be set") + } + + // If the profile is configured for AWS SSO, obtain credentials from the + // token cached by `aws sso login`. + if !hasStaticKeys && iniProfile.Key("sso_role_name").String() != "" { + ssoCreds, err := p.getSSOCredentials(cc, iniConfig, iniProfile) + if err != nil { + return Value{}, err + } + expiration := ssoCreds.RoleCredentials.expirationTime() + p.retrieved = true + p.SetExpiration(expiration, DefaultExpiryWindow) + return Value{ + AccessKeyID: ssoCreds.RoleCredentials.AccessKeyID, + SecretAccessKey: ssoCreds.RoleCredentials.SecretAccessKey, + SessionToken: ssoCreds.RoleCredentials.SessionToken, + Expiration: expiration, + SignerType: SignatureV4, + }, nil + } + + // A profile carrying SSO configuration without sso_role_name cannot + // engage the SSO flow above; without a complete static key pair the + // values below would be anonymous. + if !hasStaticKeys && + (iniProfile.Key("sso_session").String() != "" || iniProfile.Key("sso_start_url").String() != "" || + iniProfile.Key("sso_account_id").String() != "" || iniProfile.Key("sso_region").String() != "") { + return Value{}, errors.New("profile has SSO configuration but no sso_role_name, and no complete static credentials") + } + + // If credential_process is defined, obtain credentials by executing the + // external process. Static credentials and SSO in the same profile both + // take precedence over it, matching aws-sdk-go-v2. credentialProcess := strings.TrimSpace(iniProfile.Key("credential_process").String()) - if credentialProcess != "" { + if !hasStaticKeys && credentialProcess != "" { args := strings.Fields(credentialProcess) if len(args) <= 1 { return Value{}, errors.New("invalid credential process args") @@ -131,6 +225,7 @@ func (p *FileAWSCredentials) retrieve() (Value, error) { SignerType: SignatureV4, }, nil } + p.retrieved = true return Value{ AccessKeyID: id.String(), @@ -140,28 +235,319 @@ func (p *FileAWSCredentials) retrieve() (Value, error) { }, nil } +// getSSOCredentials fetches role credentials for an SSO-configured profile +// from the AWS SSO portal, using the access token cached by `aws sso login`. +func (p *FileAWSCredentials) getSSOCredentials(cc *CredContext, iniConfig *ini.File, iniProfile *ini.Section) (ssoCredentials, error) { + ssoAccountID := iniProfile.Key("sso_account_id").String() + ssoRoleName := iniProfile.Key("sso_role_name").String() + if ssoAccountID == "" { + return ssoCredentials{}, errors.New("profile defines sso_role_name but no sso_account_id") + } + + // Modern config: the profile references an [sso-session ] section + // and the cached token file is named after the SHA1 of the session name. + // Legacy config: sso_start_url/sso_region live directly on the profile + // and the cached token file is named after the SHA1 of the start URL. + ssoSessionName := iniProfile.Key("sso_session").String() + cacheKey := ssoSessionName + ssoRegion := iniProfile.Key("sso_region").String() + if ssoSessionName != "" { + if sessionSection, err := iniConfig.GetSection("sso-session " + ssoSessionName); err == nil { + // aws-sdk-go-v2 rejects profile sso_region/sso_start_url + // values that contradict the sso-session; guessing between + // the two could silently query the wrong portal. + if region := sessionSection.Key("sso_region").String(); region != "" { + if ssoRegion != "" && ssoRegion != region { + return ssoCredentials{}, fmt.Errorf("sso_region %q in profile must match sso_region %q in sso-session %q", ssoRegion, region, ssoSessionName) + } + ssoRegion = region + } + if sessionStartURL := sessionSection.Key("sso_start_url").String(); sessionStartURL != "" { + if profileStartURL := iniProfile.Key("sso_start_url").String(); profileStartURL != "" && profileStartURL != sessionStartURL { + return ssoCredentials{}, fmt.Errorf("sso_start_url %q in profile must match sso_start_url %q in sso-session %q", profileStartURL, sessionStartURL, ssoSessionName) + } + } + } + } else { + startURL := iniProfile.Key("sso_start_url").String() + if startURL == "" { + return ssoCredentials{}, errors.New("profile defines sso_role_name but neither sso_session nor sso_start_url") + } + cacheKey = startURL + } + + hash := sha1.Sum([]byte(cacheKey)) + cachedTokenFilename := hex.EncodeToString(hash[:]) + ".json" + + cacheDir := p.overrideSSOCacheDir + if cacheDir == "" { + homeDir, err := os.UserHomeDir() + if err != nil { + return ssoCredentials{}, fmt.Errorf("getting home dir: %w", err) + } + cacheDir = filepath.Join(homeDir, ".aws", "sso", "cache") + } + cachedTokenPath := filepath.Join(cacheDir, cachedTokenFilename) + cachedTokenRaw, err := os.ReadFile(cachedTokenPath) + if err != nil { + return ssoCredentials{}, fmt.Errorf("reading cached SSO token %q (try `aws sso login`): %w", cachedTokenPath, err) + } + + var cachedToken ssoCachedToken + if err := json.Unmarshal(cachedTokenRaw, &cachedToken); err != nil { + return ssoCredentials{}, fmt.Errorf("parsing cached SSO token %q: %w", cachedTokenPath, err) + } + now := time.Now + if p.CurrentTime != nil { + now = p.CurrentTime + } + if cachedToken.ExpiresAt.Before(now()) { + return ssoCredentials{}, errors.New("cached SSO token is expired, refresh it with `aws sso login`") + } + + if ssoRegion == "" { + ssoRegion = cachedToken.Region + } + if ssoRegion == "" { + return ssoCredentials{}, errors.New("unable to determine AWS SSO region from profile, sso-session or cached token") + } + + portalURL := p.overrideSSOPortalURL + if portalURL == "" { + portalURL = ssoPortalBaseURL(ssoRegion) + } + if cc == nil { + cc = defaultCredContext + } + ctx, cancel := context.WithTimeout(cc.requestContext(), ssoPortalRequestTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, portalURL+"/federation/credentials", nil) + if err != nil { + return ssoCredentials{}, fmt.Errorf("creating SSO role credentials request: %w", err) + } + req.Header.Set("x-amz-sso_bearer_token", cachedToken.AccessToken) + query := req.URL.Query() + query.Add("account_id", ssoAccountID) + query.Add("role_name", ssoRoleName) + req.URL.RawQuery = query.Encode() + + client := cc.Client + if client == nil { + client = defaultCredContext.Client + } + resp, err := client.Do(req) + if err != nil { + return ssoCredentials{}, fmt.Errorf("fetching SSO role credentials: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) + return ssoCredentials{}, fmt.Errorf("fetching SSO role credentials: %s: %s", resp.Status, strings.TrimSpace(string(body))) + } + + var ssoCreds ssoCredentials + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&ssoCreds); err != nil { + return ssoCredentials{}, fmt.Errorf("parsing SSO role credentials response: %w", err) + } + if ssoCreds.RoleCredentials.AccessKeyID == "" || ssoCreds.RoleCredentials.SecretAccessKey == "" { + return ssoCredentials{}, errors.New("SSO portal returned empty role credentials") + } + return ssoCreds, nil +} + +// ssoPortalBaseURL returns the AWS SSO portal endpoint for region. The DNS +// suffix follows the region's AWS partition per aws-sdk-go-v2 endpoint +// metadata; regions of unlisted partitions, including aws-us-gov, use the +// standard amazonaws.com suffix. +func ssoPortalBaseURL(region string) string { + suffix := "amazonaws.com" + switch { + case strings.HasPrefix(region, "cn-"): + suffix = "amazonaws.com.cn" + case strings.HasPrefix(region, "us-iso-"): + suffix = "c2s.ic.gov" + case strings.HasPrefix(region, "us-isob-"): + suffix = "sc2s.sgov.gov" + case strings.HasPrefix(region, "eu-isoe-"): + suffix = "cloud.adc-e.uk" + case strings.HasPrefix(region, "us-isof-"): + suffix = "csp.hci.ic.gov" + case strings.HasPrefix(region, "eusc-"): + suffix = "amazonaws.eu" + } + return fmt.Sprintf("https://portal.sso.%s.%s", region, suffix) +} + // Retrieve reads and extracts the shared credentials from the current // users home directory. +// +// Deprecated: Retrieve() exists for historical compatibility and should not +// be used. To get new credentials use the RetrieveWithCredContext function. func (p *FileAWSCredentials) Retrieve() (Value, error) { - return p.retrieve() + return p.retrieve(nil) } -// RetrieveWithCredContext is like Retrieve(), cred context is no-op for File credentials -func (p *FileAWSCredentials) RetrieveWithCredContext(_ *CredContext) (Value, error) { - return p.retrieve() +// RetrieveWithCredContext retrieves credentials from the file like Retrieve, +// using the context's HTTP client for any SSO portal call. +func (p *FileAWSCredentials) RetrieveWithCredContext(cc *CredContext) (Value, error) { + return p.retrieve(cc) } -// loadProfiles loads from the file pointed to by shared credentials filename for profile. +// loadProfile loads from the file pointed to by shared credentials filename for profile. // The credentials retrieved from the profile will be returned or error. Error will be // returned if it fails to read from the file, or the data is invalid. -func loadProfile(filename, profile string) (*ini.Section, error) { +func loadProfile(filename, profile string) (*ini.File, *ini.Section, error) { config, err := ini.Load(filename) if err != nil { - return nil, err + return nil, nil, err } iniProfile, err := config.GetSection(profile) if err != nil { - return nil, err + // AWS config files (~/.aws/config) name non-default profile + // sections "profile ". + var sectionErr error + iniProfile, sectionErr = config.GetSection("profile " + profile) + if sectionErr != nil { + return nil, nil, err + } + } + return config, iniProfile, nil +} + +// loadDefaultProfiles loads profile from the default AWS files, mirroring +// the AWS SDK's shared-config resolution: the config file (AWS_CONFIG_FILE +// or ~/.aws/config) is loaded first and the shared credentials file +// (AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials) overrides matching +// keys. A missing file is tolerated as long as the other one loads; a file +// that exists but fails to parse aborts resolution, like aws-sdk-go-v2, +// because silently skipping it could resolve credentials from the wrong +// (lower-precedence) source. +func loadDefaultProfiles(profile string) (*ini.File, *ini.Section, error) { + configFilename := os.Getenv("AWS_CONFIG_FILE") + credsFilename := os.Getenv("AWS_SHARED_CREDENTIALS_FILE") + var loadErrs []error + if configFilename == "" || credsFilename == "" { + // A home-dir failure only rules out the files defaulted under it; + // a file named via env var must still load. + homeDir, err := os.UserHomeDir() + if err != nil { + loadErrs = append(loadErrs, err) + } else { + if configFilename == "" { + configFilename = filepath.Join(homeDir, ".aws", "config") + } + if credsFilename == "" { + credsFilename = filepath.Join(homeDir, ".aws", "credentials") + } + } + } + + merged := ini.Empty() + loaded := false + if configFilename != "" { + config, err := ini.Load(configFilename) + switch { + case err == nil: + loaded = true + mergeAWSConfigSections(merged, config) + case errors.Is(err, fs.ErrNotExist): + loadErrs = append(loadErrs, err) + default: + return nil, nil, err + } + } + if credsFilename != "" { + config, err := ini.Load(credsFilename) + switch { + case err == nil: + loaded = true + mergeAWSCredentialsSections(merged, config) + case errors.Is(err, fs.ErrNotExist): + loadErrs = append(loadErrs, err) + default: + return nil, nil, err + } + } + if !loaded { + // Prefer the home-dir error when there is one (the informative + // failure); otherwise return the last not-exist error bare so the + // legacy os.IsNotExist probe on a missing-files result keeps working. + err := loadErrs[len(loadErrs)-1] + for _, e := range loadErrs { + if !errors.Is(e, fs.ErrNotExist) { + err = e + break + } + } + return nil, nil, err + } + iniProfile, err := merged.GetSection(profile) + if err != nil { + return nil, nil, err + } + return merged, iniProfile, nil +} + +// mergeAWSConfigSections copies the AWS config file's sections into dst +// under the SDK's config-file rules: bare sections other than "default" and +// "sso-session " are invalid profile names and ignored, and a +// "profile " section is renamed to , replacing a bare +// same-named section wholesale rather than merging with it. +func mergeAWSConfigSections(dst, src *ini.File) { + for _, section := range src.Sections() { + name := section.Name() + if name == ini.DefaultSection || strings.HasPrefix(name, "profile ") || + (!strings.EqualFold(name, "default") && !strings.HasPrefix(name, "sso-session ")) { + continue + } + copySectionKeys(dst.Section(name), section) + } + for _, section := range src.Sections() { + name := section.Name() + if !strings.HasPrefix(name, "profile ") { + continue + } + dstSection := dst.Section(strings.TrimPrefix(name, "profile ")) + for _, key := range dstSection.Keys() { + dstSection.DeleteKey(key.Name()) + } + copySectionKeys(dstSection, section) + } +} + +// mergeAWSCredentialsSections copies the shared credentials file's sections +// into dst, overriding matching keys; "profile "-prefixed section names are +// invalid in the credentials file and ignored. When a section overrides an +// existing profile, the static credentials move atomically: +// aws_access_key_id, aws_secret_access_key and aws_session_token are only +// taken from an overriding section carrying the complete key pair, so a +// partial pair cannot clobber half of an existing one. A section new to +// the merge is copied as-is; a partial pair it carries is rejected at +// resolution time, matching aws-sdk-go-v2's partial-credentials error. +func mergeAWSCredentialsSections(dst, src *ini.File) { + for _, section := range src.Sections() { + name := section.Name() + if name == ini.DefaultSection || strings.HasPrefix(name, "profile ") { + continue + } + _, err := dst.GetSection(name) + newSection := err != nil + hasPair := section.HasKey("aws_access_key_id") && section.HasKey("aws_secret_access_key") + dstSection := dst.Section(name) + for _, key := range section.Keys() { + switch key.Name() { + case "aws_access_key_id", "aws_secret_access_key", "aws_session_token": + if !newSection && !hasPair { + continue + } + } + dstSection.Key(key.Name()).SetValue(key.Value()) + } + } +} + +func copySectionKeys(dst, src *ini.Section) { + for _, key := range src.Keys() { + dst.Key(key.Name()).SetValue(key.Value()) } - return iniProfile, nil } diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go index f4f7c8f7..f35d9d65 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go @@ -58,6 +58,17 @@ type IAM struct { // Region configurable custom region for STS Region string + // ExpiryWindow configures how long before the actual credential + // expiration the credentials are considered expired and refreshed. + // The zero value keeps the default behavior of refreshing once 80% + // of the credential lifetime has elapsed. A positive value refreshes + // that long before the actual expiration; a value that meets or + // exceeds the credential lifetime places the effective expiration at + // or before the moment of retrieval, so the credentials are refreshed + // on every subsequent retrieval. A negative value behaves like the + // default. + ExpiryWindow time.Duration + // Support for container authorization token https://docs.aws.amazon.com/sdkref/latest/guide/feature-container-credentials.html Container struct { AuthorizationToken string @@ -94,6 +105,15 @@ func NewIAM(endpoint string) *Credentials { }) } +// expiryWindow returns the configured ExpiryWindow, falling back to +// DefaultExpiryWindow when unset or negative. +func (m *IAM) expiryWindow() time.Duration { + if m.ExpiryWindow <= 0 { + return DefaultExpiryWindow + } + return m.ExpiryWindow +} + // RetrieveWithCredContext is like Retrieve with Cred Context func (m *IAM) RetrieveWithCredContext(cc *CredContext) (Value, error) { if cc == nil { @@ -151,6 +171,8 @@ func (m *IAM) RetrieveWithCredContext(cc *CredContext) (Value, error) { client = defaultCredContext.Client } + ctx := cc.requestContext() + endpoint := m.Endpoint switch { @@ -184,7 +206,11 @@ func (m *IAM) RetrieveWithCredContext(cc *CredContext) (Value, error) { stsWebIdentityCreds, err := creds.RetrieveWithCredContext(cc) if err == nil { - m.SetExpiration(creds.Expiration(), DefaultExpiryWindow) + // Use the raw credential expiration, not creds.Expiration(): + // the inner provider has already reduced its own expiration by + // DefaultExpiryWindow, and applying a window to the reduced + // value would compound the two. + m.SetExpiration(stsWebIdentityCreds.Expiration, m.expiryWindow()) } return stsWebIdentityCreds, err @@ -193,11 +219,11 @@ func (m *IAM) RetrieveWithCredContext(cc *CredContext) (Value, error) { endpoint = fmt.Sprintf("%s%s", DefaultECSRoleEndpoint, relativeURI) } - roleCreds, err = getEcsTaskCredentials(client, endpoint, token) + roleCreds, err = getEcsTaskCredentials(ctx, client, endpoint, token) case tokenFile != "" && fullURI != "": endpoint = fullURI - roleCreds, err = getEKSPodIdentityCredentials(client, endpoint, tokenFile) + roleCreds, err = getEKSPodIdentityCredentials(ctx, client, endpoint, tokenFile) case fullURI != "": if len(endpoint) == 0 { @@ -211,17 +237,16 @@ func (m *IAM) RetrieveWithCredContext(cc *CredContext) (Value, error) { } } - roleCreds, err = getEcsTaskCredentials(client, endpoint, token) + roleCreds, err = getEcsTaskCredentials(ctx, client, endpoint, token) default: - roleCreds, err = getCredentials(client, endpoint) + roleCreds, err = getCredentials(ctx, client, endpoint) } if err != nil { return Value{}, err } - // Expiry window is set to 10secs. - m.SetExpiration(roleCreds.Expiration, DefaultExpiryWindow) + m.SetExpiration(roleCreds.Expiration, m.expiryWindow()) return Value{ AccessKeyID: roleCreds.AccessKeyID, @@ -273,8 +298,8 @@ func getIAMRoleURL(endpoint string) (*url.URL, error) { // with the current EC2 service. If there are no credentials, // or there is an error making or receiving the request. // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html -func listRoleNames(client *http.Client, u *url.URL, token string) ([]string, error) { - req, err := http.NewRequest(http.MethodGet, u.String(), nil) +func listRoleNames(ctx context.Context, client *http.Client, u *url.URL, token string) ([]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return nil, err } @@ -303,8 +328,8 @@ func listRoleNames(client *http.Client, u *url.URL, token string) ([]string, err return credsList, nil } -func getEcsTaskCredentials(client *http.Client, endpoint, token string) (ec2RoleCredRespBody, error) { - req, err := http.NewRequest(http.MethodGet, endpoint, nil) +func getEcsTaskCredentials(ctx context.Context, client *http.Client, endpoint, token string) (ec2RoleCredRespBody, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return ec2RoleCredRespBody{}, err } @@ -330,20 +355,20 @@ func getEcsTaskCredentials(client *http.Client, endpoint, token string) (ec2Role return respCreds, nil } -func getEKSPodIdentityCredentials(client *http.Client, endpoint string, tokenFile string) (ec2RoleCredRespBody, error) { +func getEKSPodIdentityCredentials(ctx context.Context, client *http.Client, endpoint string, tokenFile string) (ec2RoleCredRespBody, error) { if tokenFile != "" { bytes, err := os.ReadFile(tokenFile) if err != nil { return ec2RoleCredRespBody{}, fmt.Errorf("getEKSPodIdentityCredentials: failed to read token file:%s", err) } token := string(bytes) - return getEcsTaskCredentials(client, endpoint, token) + return getEcsTaskCredentials(ctx, client, endpoint, token) } return ec2RoleCredRespBody{}, fmt.Errorf("getEKSPodIdentityCredentials: no tokenFile found") } -func fetchIMDSToken(client *http.Client, endpoint string) (string, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) +func fetchIMDSToken(ctx context.Context, client *http.Client, endpoint string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodPut, endpoint+TokenPath, nil) @@ -371,14 +396,18 @@ func fetchIMDSToken(client *http.Client, endpoint string) (string, error) { // // If the credentials cannot be found, or there is an error // reading the response an error will be returned. -func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { +func getCredentials(ctx context.Context, client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { if endpoint == "" { endpoint = DefaultIAMRoleEndpoint } // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html - token, err := fetchIMDSToken(client, endpoint) + token, err := fetchIMDSToken(ctx, client, endpoint) if err != nil { + // A dead caller context is not an IMDSv2 availability signal. + if cerr := ctx.Err(); cerr != nil { + return ec2RoleCredRespBody{}, fmt.Errorf("%w (imds token fetch: %v)", cerr, err) + } // Return only errors for valid situations, if the IMDSv2 is not enabled // we will not be able to get the token, in such a situation we have // to rely on IMDSv1 behavior as a fallback, this check ensures that. @@ -395,7 +424,7 @@ func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, } // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html - roleNames, err := listRoleNames(client, u, token) + roleNames, err := listRoleNames(ctx, client, u, token) if err != nil { return ec2RoleCredRespBody{}, err } @@ -415,7 +444,7 @@ func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, // $ curl http://169.254.169.254/latest/meta-data/iam/security-credentials/s3access // u.Path = path.Join(u.Path, roleName) - req, err := http.NewRequest(http.MethodGet, u.String(), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return ec2RoleCredRespBody{}, err } diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go index ef6f436b..7ab7d68e 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go @@ -19,6 +19,7 @@ package credentials import ( "bytes" + "context" "encoding/xml" "errors" "fmt" @@ -100,7 +101,7 @@ func NewSTSClientGrants(stsEndpoint string, getClientGrantsTokenExpiry func() (* }), nil } -func getClientGrantsCredentials(clnt *http.Client, endpoint string, +func getClientGrantsCredentials(ctx context.Context, clnt *http.Client, endpoint string, getClientGrantsTokenExpiry func() (*ClientGrantsToken, error), ) (AssumeRoleWithClientGrantsResponse, error) { accessToken, err := getClientGrantsTokenExpiry() @@ -119,7 +120,7 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string, return AssumeRoleWithClientGrantsResponse{}, err } - req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode())) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), strings.NewReader(v.Encode())) if err != nil { return AssumeRoleWithClientGrantsResponse{}, err } @@ -179,7 +180,7 @@ func (m *STSClientGrants) RetrieveWithCredContext(cc *CredContext) (Value, error return Value{}, errors.New("STS endpoint unknown") } - a, err := getClientGrantsCredentials(client, stsEndpoint, m.GetClientGrantsTokenExpiry) + a, err := getClientGrantsCredentials(cc.requestContext(), client, stsEndpoint, m.GetClientGrantsTokenExpiry) if err != nil { return Value{}, err } diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_custom_identity.go b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_custom_identity.go index e9e7a115..b986c175 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_custom_identity.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_custom_identity.go @@ -107,7 +107,7 @@ func (c *CustomTokenIdentity) RetrieveWithCredContext(cc *CredContext) (value Va u.RawQuery = v.Encode() - req, err := http.NewRequest(http.MethodPost, u.String(), nil) + req, err := http.NewRequestWithContext(cc.requestContext(), http.MethodPost, u.String(), nil) if err != nil { return value, err } diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go index 7e80cd6a..648c95d2 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go @@ -172,7 +172,7 @@ func (k *LDAPIdentity) RetrieveWithCredContext(cc *CredContext) (value Value, er v.Set("ConfigName", k.ConfigName) } - req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode())) + req, err := http.NewRequestWithContext(cc.requestContext(), http.MethodPost, u.String(), strings.NewReader(v.Encode())) if err != nil { return value, err } diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_tls_identity.go b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_tls_identity.go index beab4a6a..ebaae128 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_tls_identity.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_tls_identity.go @@ -131,7 +131,7 @@ func (i *STSCertificateIdentity) RetrieveWithCredContext(cc *CredContext) (Value } endpointURL.RawQuery = queryValues.Encode() - req, err := http.NewRequest(http.MethodPost, endpointURL.String(), nil) + req, err := http.NewRequestWithContext(cc.requestContext(), http.MethodPost, endpointURL.String(), nil) if err != nil { return Value{}, err } diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go index a9987255..c3afdc62 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go @@ -19,6 +19,7 @@ package credentials import ( "bytes" + "context" "encoding/xml" "errors" "fmt" @@ -137,7 +138,7 @@ func WithPolicy(policy string) func(*STSWebIdentity) { } } -func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSessionName string, policy string, +func getWebIdentityCredentials(ctx context.Context, clnt *http.Client, endpoint, roleARN, roleSessionName string, policy string, getWebIDTokenExpiry func() (*WebIdentityToken, error), tokenRevokeType string, ) (AssumeRoleWithWebIdentityResponse, error) { idToken, err := getWebIDTokenExpiry() @@ -180,7 +181,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession return AssumeRoleWithWebIdentityResponse{}, err } - req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode())) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), strings.NewReader(v.Encode())) if err != nil { return AssumeRoleWithWebIdentityResponse{}, err } @@ -242,7 +243,7 @@ func (m *STSWebIdentity) RetrieveWithCredContext(cc *CredContext) (Value, error) return Value{}, errors.New("STS endpoint unknown") } - a, err := getWebIdentityCredentials(client, stsEndpoint, m.RoleARN, m.roleSessionName, m.Policy, m.GetWebIDTokenExpiry, m.TokenRevokeType) + a, err := getWebIdentityCredentials(cc.requestContext(), client, stsEndpoint, m.RoleARN, m.roleSessionName, m.Policy, m.GetWebIDTokenExpiry, m.TokenRevokeType) if err != nil { return Value{}, err } diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go b/src/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go index 01d4e6aa..2ae38b85 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go @@ -209,6 +209,58 @@ func (t Transition) MarshalXML(en *xml.Encoder, startElement xml.StartElement) e return en.EncodeElement(transitionWrapper(t), startElement) } +// Compression is a MinIO AIStor extension with no S3 equivalent: it compresses +// matching current object versions in place. Presence of the element enables the +// action, so Rule carries it as a pointer — unlike Transition, there is no +// StorageClass to key presence off and Days: 0 legitimately means "immediately". +// +// HighCompression selects the strongest level the server offers; without it the +// rule writes the level ingest uses and only compresses objects stored +// uncompressed. +// +// Unlike Transition and Expiration there is no absolute Date trigger — a one-off +// sweep is what the compress batch job is for. That also means no custom +// MarshalJSON: with no embedded time.Time to suppress, every field honors +// omitempty, and a zero element still marshals as {} because Rule holds it as a +// pointer and its presence alone enables the action. +type Compression struct { + XMLName xml.Name `xml:"Compression" json:"-"` + Days ExpirationDays `xml:"Days,omitempty" json:"Days,omitempty"` + HighCompression bool `xml:"HighCompression,omitempty" json:"HighCompression,omitempty"` + SkipCompressed bool `xml:"SkipCompressed,omitempty" json:"SkipCompressed,omitempty"` + IncludeEncrypted bool `xml:"IncludeEncrypted,omitempty" json:"IncludeEncrypted,omitempty"` +} + +// IsDaysNull returns true if days field is null +func (c *Compression) IsDaysNull() bool { + return c == nil || c.Days == ExpirationDays(0) +} + +// IsNull returns true if the element is absent. +func (c *Compression) IsNull() bool { + return c == nil +} + +// NoncurrentVersionCompression is the Compression counterpart for noncurrent +// object versions. See Compression for why Rule holds it as a pointer. +type NoncurrentVersionCompression struct { + XMLName xml.Name `xml:"NoncurrentVersionCompression" json:"-"` + NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty" json:"NoncurrentDays,omitempty"` + HighCompression bool `xml:"HighCompression,omitempty" json:"HighCompression,omitempty"` + SkipCompressed bool `xml:"SkipCompressed,omitempty" json:"SkipCompressed,omitempty"` + IncludeEncrypted bool `xml:"IncludeEncrypted,omitempty" json:"IncludeEncrypted,omitempty"` +} + +// IsDaysNull returns true if noncurrent days field is null +func (n *NoncurrentVersionCompression) IsDaysNull() bool { + return n == nil || n.NoncurrentDays == ExpirationDays(0) +} + +// IsNull returns true if the element is absent. +func (n *NoncurrentVersionCompression) IsNull() bool { + return n == nil +} + // And And Rule for LifecycleTag, to be used in LifecycleRuleFilter type And struct { XMLName xml.Name `xml:"And" json:"-"` @@ -265,12 +317,11 @@ func (f Filter) MarshalJSON() ([]byte, error) { } // MarshalXML - produces the xml representation of the Filter struct -// only one of Prefix, And and Tag should be present in the output. +// at most one of Prefix, And, Tag, ObjectSizeGreaterThan and +// ObjectSizeLessThan is present in the output. +// A zero-value Filter marshals as ; +// suppressing the element entirely is the caller's responsibility. func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - if f.IsNull() { - return nil - } - if err := e.EncodeToken(start); err != nil { return err } @@ -473,6 +524,8 @@ func (r Rule) MarshalJSON() ([]byte, error) { Prefix string `json:"Prefix,omitempty"` Status string `json:"Status"` Transition *Transition `json:"Transition,omitempty"` + Compression *Compression `json:"Compression,omitempty"` + NoncurrentVersionCompression *NoncurrentVersionCompression `json:"NoncurrentVersionCompression,omitempty"` } newr := rule{ Prefix: r.Prefix, @@ -504,10 +557,45 @@ func (r Rule) MarshalJSON() ([]byte, error) { if !r.AllVersionsExpiration.IsNull() { newr.AllVersionsExpiration = &r.AllVersionsExpiration } + newr.Compression = r.Compression + newr.NoncurrentVersionCompression = r.NoncurrentVersionCompression return json.Marshal(newr) } +// ruleAlias drops Rule's methods so EncodeElement falls back to the default +// struct-tag encoding. +type ruleAlias Rule + +// ruleWrapper embeds ruleAlias, so it needs no updating when Rule gains a +// field. Its RuleFilter is shallower than the embedded one, and encoding/xml's +// depth-based field shadowing suppresses the deeper field, so is +// emitted only through this pointer: nil omits the element, a set pointer +// emits it (empty) as the last child of . +type ruleWrapper struct { + ruleAlias + RuleFilter *Filter `xml:"Filter,omitempty"` +} + +// MarshalXML customizes XML encoding of Rule: a rule with neither Filter nor +// Prefix set emits an empty as the last +// child of — S3 requires a Filter when no Prefix element is present; +// the empty child is this library's representation of an empty +// filter. A rule with only a top-level Prefix omits the Filter element; a +// non-null Filter marshals through the default encoding in its declared +// field position. MarshalJSON is unaffected and still omits a null Filter. +func (r Rule) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if !r.RuleFilter.IsNull() { + return e.EncodeElement(ruleAlias(r), start) + } + + w := ruleWrapper{ruleAlias: ruleAlias(r)} + if r.Prefix == "" { + w.RuleFilter = &w.ruleAlias.RuleFilter + } + return e.EncodeElement(w, start) +} + // Rule represents a single rule in lifecycle configuration type Rule struct { XMLName xml.Name `xml:"Rule,omitempty" json:"-"` @@ -522,6 +610,8 @@ type Rule struct { Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` Status string `xml:"Status" json:"Status"` Transition Transition `xml:"Transition,omitempty" json:"Transition,omitempty"` + Compression *Compression `xml:"Compression,omitempty" json:"Compression,omitempty"` + NoncurrentVersionCompression *NoncurrentVersionCompression `xml:"NoncurrentVersionCompression,omitempty" json:"NoncurrentVersionCompression,omitempty"` } // Configuration is a collection of Rule objects. diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/policy/bucket-policy-condition.go b/src/vendor/github.com/minio/minio-go/v7/pkg/policy/bucket-policy-condition.go new file mode 100644 index 00000000..4d6d85cb --- /dev/null +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/policy/bucket-policy-condition.go @@ -0,0 +1,114 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package policy + +import ( + "github.com/minio/minio-go/v7/pkg/set" +) + +// ConditionKeyMap - map of policy condition key and value. +type ConditionKeyMap map[string]set.StringSet + +// Add - adds key and value. The value is appended If key already exists. +func (ckm ConditionKeyMap) Add(key string, value set.StringSet) { + if v, ok := ckm[key]; ok { + ckm[key] = v.Union(value) + } else { + ckm[key] = set.CopyStringSet(value) + } +} + +// Remove - removes value of given key. If key has empty after removal, the key is also removed. +func (ckm ConditionKeyMap) Remove(key string, value set.StringSet) { + if v, ok := ckm[key]; ok { + if value != nil { + ckm[key] = v.Difference(value) + } + + if ckm[key].IsEmpty() { + delete(ckm, key) + } + } +} + +// RemoveKey - removes key and its value. +func (ckm ConditionKeyMap) RemoveKey(key string) { + delete(ckm, key) +} + +// CopyConditionKeyMap - returns new copy of given ConditionKeyMap. +func CopyConditionKeyMap(condKeyMap ConditionKeyMap) ConditionKeyMap { + out := make(ConditionKeyMap) + + for k, v := range condKeyMap { + out[k] = set.CopyStringSet(v) + } + + return out +} + +// mergeConditionKeyMap - returns a new ConditionKeyMap which contains merged key/value of given two ConditionKeyMap. +func mergeConditionKeyMap(condKeyMap1, condKeyMap2 ConditionKeyMap) ConditionKeyMap { + out := CopyConditionKeyMap(condKeyMap1) + + for k, v := range condKeyMap2 { + if ev, ok := out[k]; ok { + out[k] = ev.Union(v) + } else { + out[k] = set.CopyStringSet(v) + } + } + + return out +} + +// ConditionMap - map of condition and conditional values. +type ConditionMap map[string]ConditionKeyMap + +// Add - adds condition key and condition value. The value is appended if key already exists. +func (cond ConditionMap) Add(condKey string, condKeyMap ConditionKeyMap) { + if v, ok := cond[condKey]; ok { + cond[condKey] = mergeConditionKeyMap(v, condKeyMap) + } else { + cond[condKey] = CopyConditionKeyMap(condKeyMap) + } +} + +// Remove - removes condition key and its value. +func (cond ConditionMap) Remove(condKey string) { + delete(cond, condKey) +} + +// mergeConditionMap - returns new ConditionMap which contains merged key/value of two ConditionMap. +func mergeConditionMap(condMap1, condMap2 ConditionMap) ConditionMap { + out := make(ConditionMap) + + for k, v := range condMap1 { + out[k] = CopyConditionKeyMap(v) + } + + for k, v := range condMap2 { + if ev, ok := out[k]; ok { + out[k] = mergeConditionKeyMap(ev, v) + } else { + out[k] = CopyConditionKeyMap(v) + } + } + + return out +} diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/policy/bucket-policy.go b/src/vendor/github.com/minio/minio-go/v7/pkg/policy/bucket-policy.go new file mode 100644 index 00000000..762c9015 --- /dev/null +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/policy/bucket-policy.go @@ -0,0 +1,680 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package policy + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + + "github.com/minio/minio-go/v7/pkg/set" +) + +// BucketPolicy - Bucket level policy. +type BucketPolicy string + +// Different types of Policies currently supported for buckets. +const ( + BucketPolicyNone BucketPolicy = "none" + BucketPolicyReadOnly BucketPolicy = "readonly" + BucketPolicyReadWrite BucketPolicy = "readwrite" + BucketPolicyWriteOnly BucketPolicy = "writeonly" +) + +// IsValidBucketPolicy - returns true if policy is valid and supported, false otherwise. +func (p BucketPolicy) IsValidBucketPolicy() bool { + switch p { + case BucketPolicyNone, BucketPolicyReadOnly, BucketPolicyReadWrite, BucketPolicyWriteOnly: + return true + } + return false +} + +// Resource prefix for all aws resources. +const awsResourcePrefix = "arn:aws:s3:::" + +// Common bucket actions for both read and write policies. +var commonBucketActions = set.CreateStringSet("s3:GetBucketLocation") + +// Read only bucket actions. +var readOnlyBucketActions = set.CreateStringSet("s3:ListBucket") + +// Write only bucket actions. +var writeOnlyBucketActions = set.CreateStringSet("s3:ListBucketMultipartUploads") + +// Read only object actions. +var readOnlyObjectActions = set.CreateStringSet("s3:GetObject") + +// Write only object actions. +var writeOnlyObjectActions = set.CreateStringSet("s3:AbortMultipartUpload", "s3:DeleteObject", "s3:ListMultipartUploadParts", "s3:PutObject") + +// Read and write object actions. +var readWriteObjectActions = readOnlyObjectActions.Union(writeOnlyObjectActions) + +// All valid bucket and object actions. +var validActions = commonBucketActions. + Union(readOnlyBucketActions). + Union(writeOnlyBucketActions). + Union(readOnlyObjectActions). + Union(writeOnlyObjectActions) + +var startsWithFunc = func(resource, resourcePrefix string) bool { + return strings.HasPrefix(resource, resourcePrefix) +} + +// User - canonical users list. +type User struct { + AWS set.StringSet `json:"AWS,omitempty"` + CanonicalUser set.StringSet `json:"CanonicalUser,omitempty"` +} + +// UnmarshalJSON is a custom json unmarshaler for Principal field, +// the reason is that Principal can take a json struct represented by +// User string but it can also take a string. +func (u *User) UnmarshalJSON(data []byte) error { + // Try to unmarshal data in a struct equal to User, + // to avoid infinite recursive call of this function + type AliasUser User + var au AliasUser + err := json.Unmarshal(data, &au) + if err == nil { + *u = User(au) + return nil + } + // Data type is not known, check if it is a json string + // which contains a star, which is permitted in the spec + var str string + err = json.Unmarshal(data, &str) + if err == nil { + if str != "*" { + return errors.New("unrecognized Principal field") + } + *u = User{AWS: set.CreateStringSet("*")} + return nil + } + return err +} + +// Statement - minio policy statement +type Statement struct { + Actions set.StringSet `json:"Action"` + Conditions ConditionMap `json:"Condition,omitempty"` + Effect string + Principal User `json:"Principal"` + Resources set.StringSet `json:"Resource"` + Sid string +} + +// BucketAccessPolicy - minio policy collection +type BucketAccessPolicy struct { + Version string // date in YYYY-MM-DD format + Statements []Statement `json:"Statement"` +} + +// isValidStatement - returns whether given statement is valid to process for given bucket name. +func isValidStatement(statement Statement, bucketName string) bool { + if statement.Actions.Intersection(validActions).IsEmpty() { + return false + } + + if statement.Effect != "Allow" { + return false + } + + if statement.Principal.AWS == nil || !statement.Principal.AWS.Contains("*") { + return false + } + + bucketResource := awsResourcePrefix + bucketName + if statement.Resources.Contains(bucketResource) { + return true + } + + if statement.Resources.FuncMatch(startsWithFunc, bucketResource+"/").IsEmpty() { + return false + } + + return true +} + +// Returns new statements with bucket actions for given policy. +func newBucketStatement(policy BucketPolicy, bucketName, prefix string) (statements []Statement) { + statements = []Statement{} + if policy == BucketPolicyNone || bucketName == "" { + return statements + } + + bucketResource := set.CreateStringSet(awsResourcePrefix + bucketName) + + statement := Statement{ + Actions: commonBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: bucketResource, + Sid: "", + } + statements = append(statements, statement) + + if policy == BucketPolicyReadOnly || policy == BucketPolicyReadWrite { + statement = Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: bucketResource, + Sid: "", + } + if prefix != "" { + condKeyMap := make(ConditionKeyMap) + condKeyMap.Add("s3:prefix", set.CreateStringSet(prefix+"*")) + condMap := make(ConditionMap) + condMap.Add("StringLike", condKeyMap) + statement.Conditions = condMap + } + statements = append(statements, statement) + } + + if policy == BucketPolicyWriteOnly || policy == BucketPolicyReadWrite { + statement = Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: bucketResource, + Sid: "", + } + statements = append(statements, statement) + } + + return statements +} + +// Returns new statements contains object actions for given policy. +func newObjectStatement(policy BucketPolicy, bucketName, prefix string) (statements []Statement) { + statements = []Statement{} + if policy == BucketPolicyNone || bucketName == "" { + return statements + } + + statement := Statement{ + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet(awsResourcePrefix + bucketName + "/" + prefix + "*"), + Sid: "", + } + + switch policy { + case BucketPolicyReadOnly: + statement.Actions = readOnlyObjectActions + case BucketPolicyWriteOnly: + statement.Actions = writeOnlyObjectActions + case BucketPolicyReadWrite: + statement.Actions = readWriteObjectActions + } + + statements = append(statements, statement) + return statements +} + +// Returns new statements for given policy, bucket and prefix. +func newStatements(policy BucketPolicy, bucketName, prefix string) (statements []Statement) { + statements = []Statement{} + ns := newBucketStatement(policy, bucketName, prefix) + statements = append(statements, ns...) + + ns = newObjectStatement(policy, bucketName, prefix) + statements = append(statements, ns...) + + return statements +} + +// Returns whether given bucket statements are used by other than given prefix statements. +func getInUsePolicy(statements []Statement, bucketName, prefix string) (readOnlyInUse, writeOnlyInUse bool) { + resourcePrefix := awsResourcePrefix + bucketName + "/" + objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*" + + for _, s := range statements { + if !s.Resources.Contains(objectResource) && !s.Resources.FuncMatch(startsWithFunc, resourcePrefix).IsEmpty() { + if s.Actions.Intersection(readOnlyObjectActions).Equals(readOnlyObjectActions) { + readOnlyInUse = true + } + + if s.Actions.Intersection(writeOnlyObjectActions).Equals(writeOnlyObjectActions) { + writeOnlyInUse = true + } + } + if readOnlyInUse && writeOnlyInUse { + break + } + } + + return readOnlyInUse, writeOnlyInUse +} + +// Removes object actions in given statement. +func removeObjectActions(statement Statement, objectResource string) Statement { + if statement.Conditions == nil { + if len(statement.Resources) > 1 { + statement.Resources.Remove(objectResource) + } else { + statement.Actions = statement.Actions.Difference(readOnlyObjectActions) + statement.Actions = statement.Actions.Difference(writeOnlyObjectActions) + } + } + + return statement +} + +// Removes bucket actions for given policy in given statement. +func removeBucketActions(statement Statement, prefix, bucketResource string, readOnlyInUse, writeOnlyInUse bool) Statement { + removeReadOnly := func() { + if !statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) { + return + } + + if statement.Conditions == nil { + statement.Actions = statement.Actions.Difference(readOnlyBucketActions) + return + } + + if prefix != "" { + stringEqualsValue := statement.Conditions["StringEquals"] + values := set.NewStringSet() + if stringEqualsValue != nil { + values = stringEqualsValue["s3:prefix"] + if values == nil { + values = set.NewStringSet() + } + } + + values.Remove(prefix) + + if stringEqualsValue != nil { + if values.IsEmpty() { + delete(stringEqualsValue, "s3:prefix") + } + if len(stringEqualsValue) == 0 { + delete(statement.Conditions, "StringEquals") + } + } + + if len(statement.Conditions) == 0 { + statement.Conditions = nil + statement.Actions = statement.Actions.Difference(readOnlyBucketActions) + } + } + } + + removeWriteOnly := func() { + if statement.Conditions == nil { + statement.Actions = statement.Actions.Difference(writeOnlyBucketActions) + } + } + + if len(statement.Resources) > 1 { + statement.Resources.Remove(bucketResource) + } else { + if !readOnlyInUse { + removeReadOnly() + } + + if !writeOnlyInUse { + removeWriteOnly() + } + } + + return statement +} + +// Returns statements containing removed actions/statements for given +// policy, bucket name and prefix. +func removeStatements(statements []Statement, bucketName, prefix string) []Statement { + bucketResource := awsResourcePrefix + bucketName + objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*" + readOnlyInUse, writeOnlyInUse := getInUsePolicy(statements, bucketName, prefix) + + out := []Statement{} + readOnlyBucketStatements := []Statement{} + s3PrefixValues := set.NewStringSet() + + for _, statement := range statements { + if !isValidStatement(statement, bucketName) { + out = append(out, statement) + continue + } + + if statement.Resources.Contains(bucketResource) { + if statement.Conditions != nil { + statement = removeBucketActions(statement, prefix, bucketResource, false, false) + } else { + statement = removeBucketActions(statement, prefix, bucketResource, readOnlyInUse, writeOnlyInUse) + } + } else if statement.Resources.Contains(objectResource) { + statement = removeObjectActions(statement, objectResource) + } + + if !statement.Actions.IsEmpty() { + if statement.Resources.Contains(bucketResource) && + statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) && + statement.Effect == "Allow" && + statement.Principal.AWS.Contains("*") { + if statement.Conditions != nil { + stringEqualsValue := statement.Conditions["StringEquals"] + values := set.NewStringSet() + if stringEqualsValue != nil { + values = stringEqualsValue["s3:prefix"] + if values == nil { + values = set.NewStringSet() + } + } + s3PrefixValues = s3PrefixValues.Union(values.ApplyFunc(func(v string) string { + return bucketResource + "/" + v + "*" + })) + } else if !s3PrefixValues.IsEmpty() { + readOnlyBucketStatements = append(readOnlyBucketStatements, statement) + continue + } + } + out = append(out, statement) + } + } + + skipBucketStatement := true + resourcePrefix := awsResourcePrefix + bucketName + "/" + for _, statement := range out { + if !statement.Resources.FuncMatch(startsWithFunc, resourcePrefix).IsEmpty() && + s3PrefixValues.Intersection(statement.Resources).IsEmpty() { + skipBucketStatement = false + break + } + } + + for _, statement := range readOnlyBucketStatements { + if skipBucketStatement && + statement.Resources.Contains(bucketResource) && + statement.Effect == "Allow" && + statement.Principal.AWS.Contains("*") && + statement.Conditions == nil { + continue + } + + out = append(out, statement) + } + + if len(out) == 1 { + statement := out[0] + if statement.Resources.Contains(bucketResource) && + statement.Actions.Intersection(commonBucketActions).Equals(commonBucketActions) && + statement.Effect == "Allow" && + statement.Principal.AWS.Contains("*") && + statement.Conditions == nil { + out = []Statement{} + } + } + + return out +} + +// Appends given statement into statement list to have unique statements. +// - If statement already exists in statement list, it ignores. +// - If statement exists with different conditions, they are merged. +// - Else the statement is appended to statement list. +func appendStatement(statements []Statement, statement Statement) []Statement { + for i, s := range statements { + if s.Actions.Equals(statement.Actions) && + s.Effect == statement.Effect && + s.Principal.AWS.Equals(statement.Principal.AWS) && + reflect.DeepEqual(s.Conditions, statement.Conditions) { + statements[i].Resources = s.Resources.Union(statement.Resources) + return statements + } else if s.Resources.Equals(statement.Resources) && + s.Effect == statement.Effect && + s.Principal.AWS.Equals(statement.Principal.AWS) && + reflect.DeepEqual(s.Conditions, statement.Conditions) { + statements[i].Actions = s.Actions.Union(statement.Actions) + return statements + } + + if s.Resources.Intersection(statement.Resources).Equals(statement.Resources) && + s.Actions.Intersection(statement.Actions).Equals(statement.Actions) && + s.Effect == statement.Effect && + s.Principal.AWS.Intersection(statement.Principal.AWS).Equals(statement.Principal.AWS) { + if reflect.DeepEqual(s.Conditions, statement.Conditions) { + return statements + } + if s.Conditions != nil && statement.Conditions != nil { + if s.Resources.Equals(statement.Resources) { + statements[i].Conditions = mergeConditionMap(s.Conditions, statement.Conditions) + return statements + } + } + } + } + + if !statement.Actions.IsEmpty() || !statement.Resources.IsEmpty() { + return append(statements, statement) + } + + return statements +} + +// Appends two statement lists. +func appendStatements(statements, appendStatements []Statement) []Statement { + for _, s := range appendStatements { + statements = appendStatement(statements, s) + } + + return statements +} + +// Returns policy of given bucket statement. +func getBucketPolicy(statement Statement, prefix string) (commonFound, readOnly, writeOnly bool) { + if statement.Effect != "Allow" || !statement.Principal.AWS.Contains("*") { + return commonFound, readOnly, writeOnly + } + + if statement.Actions.Intersection(commonBucketActions).Equals(commonBucketActions) && + statement.Conditions == nil { + commonFound = true + } + + if statement.Actions.Intersection(writeOnlyBucketActions).Equals(writeOnlyBucketActions) && + statement.Conditions == nil { + writeOnly = true + } + + if statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) { + if prefix != "" && statement.Conditions != nil { + if stringEqualsValue, ok := statement.Conditions["StringEquals"]; ok { + if s3PrefixValues, ok := stringEqualsValue["s3:prefix"]; ok { + if s3PrefixValues.Contains(prefix) { + readOnly = true + } + } + } else if stringNotEqualsValue, ok := statement.Conditions["StringNotEquals"]; ok { + if s3PrefixValues, ok := stringNotEqualsValue["s3:prefix"]; ok { + if !s3PrefixValues.Contains(prefix) { + readOnly = true + } + } + } else if stringLikeValue, ok := statement.Conditions["StringLike"]; ok { + if s3PrefixValues, ok := stringLikeValue["s3:prefix"]; ok { + if s3PrefixValues.Contains(prefix + "*") { + readOnly = true + } + } + } else if stringNotLikeValue, ok := statement.Conditions["StringNotLike"]; ok { + if s3PrefixValues, ok := stringNotLikeValue["s3:prefix"]; ok { + if !s3PrefixValues.Contains(prefix + "*") { + readOnly = true + } + } + } + } else if prefix == "" && statement.Conditions == nil { + readOnly = true + } else if prefix != "" && statement.Conditions == nil { + readOnly = true + } + } + + return commonFound, readOnly, writeOnly +} + +// Returns policy of given object statement. +func getObjectPolicy(statement Statement) (readOnly, writeOnly bool) { + if statement.Effect == "Allow" && + statement.Principal.AWS.Contains("*") && + statement.Conditions == nil { + if statement.Actions.Intersection(readOnlyObjectActions).Equals(readOnlyObjectActions) { + readOnly = true + } + if statement.Actions.Intersection(writeOnlyObjectActions).Equals(writeOnlyObjectActions) { + writeOnly = true + } + } + + return readOnly, writeOnly +} + +// GetPolicy - Returns policy of given bucket name, prefix in given statements. +func GetPolicy(statements []Statement, bucketName, prefix string) BucketPolicy { + bucketResource := awsResourcePrefix + bucketName + objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*" + + bucketCommonFound := false + bucketReadOnly := false + bucketWriteOnly := false + matchedResource := "" + objReadOnly := false + objWriteOnly := false + + for _, s := range statements { + matchedObjResources := set.NewStringSet() + if s.Resources.Contains(objectResource) { + matchedObjResources.Add(objectResource) + } else { + matchedObjResources = s.Resources.FuncMatch(resourceMatch, objectResource) + } + if !matchedObjResources.IsEmpty() { + readOnly, writeOnly := getObjectPolicy(s) + for resource := range matchedObjResources { + if len(matchedResource) < len(resource) { + objReadOnly = readOnly + objWriteOnly = writeOnly + matchedResource = resource + } else if len(matchedResource) == len(resource) { + objReadOnly = objReadOnly || readOnly + objWriteOnly = objWriteOnly || writeOnly + matchedResource = resource + } + } + } + if s.Resources.Contains(bucketResource) { + commonFound, readOnly, writeOnly := getBucketPolicy(s, prefix) + bucketCommonFound = bucketCommonFound || commonFound + bucketReadOnly = bucketReadOnly || readOnly + bucketWriteOnly = bucketWriteOnly || writeOnly + } + } + + policy := BucketPolicyNone + if bucketCommonFound { + if bucketReadOnly && bucketWriteOnly && objReadOnly && objWriteOnly { + policy = BucketPolicyReadWrite + } else if bucketReadOnly && objReadOnly { + policy = BucketPolicyReadOnly + } else if bucketWriteOnly && objWriteOnly { + policy = BucketPolicyWriteOnly + } + } + + return policy +} + +// GetPolicies - returns a map of policies of given bucket name, prefix in given statements. +func GetPolicies(statements []Statement, bucketName, prefix string) map[string]BucketPolicy { + policyRules := map[string]BucketPolicy{} + objResources := set.NewStringSet() + // Search all resources related to objects policy + for _, s := range statements { + for r := range s.Resources { + if strings.HasPrefix(r, awsResourcePrefix+bucketName+"/"+prefix) { + objResources.Add(r) + } + } + } + + // Pretend that policy resource as an actual object and fetch its policy + for r := range objResources { + // Put trailing * if exists in asterisk + asterisk := "" + if strings.HasSuffix(r, "*") { + r = r[:len(r)-1] + asterisk = "*" + } + var objectPath string + if len(r) >= len(awsResourcePrefix+bucketName)+1 { + objectPath = r[len(awsResourcePrefix+bucketName)+1:] + } + p := GetPolicy(statements, bucketName, objectPath) + policyRules[bucketName+"/"+objectPath+asterisk] = p + } + return policyRules +} + +// SetPolicy - Returns new statements containing policy of given bucket name and prefix are appended. +func SetPolicy(statements []Statement, policy BucketPolicy, bucketName, prefix string) []Statement { + out := removeStatements(statements, bucketName, prefix) + // fmt.Println("out = ") + // printstatement(out) + ns := newStatements(policy, bucketName, prefix) + // fmt.Println("ns = ") + // printstatement(ns) + + rv := appendStatements(out, ns) + // fmt.Println("rv = ") + // printstatement(rv) + + return rv +} + +// Match function matches wild cards in 'pattern' for resource. +func resourceMatch(pattern, resource string) bool { + if pattern == "" { + return resource == pattern + } + if pattern == "*" { + return true + } + parts := strings.Split(pattern, "*") + if len(parts) == 1 { + return resource == pattern + } + tGlob := strings.HasSuffix(pattern, "*") + end := len(parts) - 1 + if !strings.HasPrefix(resource, parts[0]) { + return false + } + for i := 1; i < end; i++ { + if !strings.Contains(resource, parts[i]) { + return false + } + idx := strings.Index(resource, parts[i]) + len(parts[i]) + resource = resource[idx:] + } + return tGlob || strings.HasSuffix(resource, parts[end]) +} diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming-unsigned-trailer.go b/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming-unsigned-trailer.go index e18002b8..35c50b0c 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming-unsigned-trailer.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming-unsigned-trailer.go @@ -61,10 +61,10 @@ func getUSStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 { return streamLen } -// prepareStreamingRequest - prepares a request with appropriate -// headers before computing the seed signature. +// prepareUSStreamingRequest - prepares a request with the headers +// required for an unsigned aws-chunked streaming upload. func prepareUSStreamingRequest(req *http.Request, sessionToken string, dataLen int64, timestamp time.Time) { - req.TransferEncoding = []string{"aws-chunked"} + setAwsChunkedContentEncoding(req) if sessionToken != "" { req.Header.Set("X-Amz-Security-Token", sessionToken) } diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go b/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go index 4fe0debf..8d86d4f1 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go @@ -132,8 +132,8 @@ func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int for k := range req.Trailer { req.Header.Add("X-Amz-Trailer", strings.ToLower(k)) } - req.TransferEncoding = []string{"aws-chunked"} } + setAwsChunkedContentEncoding(req) if sessionToken != "" { req.Header.Set("X-Amz-Security-Token", sessionToken) diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go b/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go index 2a060c6e..0ae2256d 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go @@ -331,7 +331,7 @@ func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, locati req.Header.Add("X-Amz-Trailer", strings.ToLower(k)) } - req.Header.Set("Content-Encoding", "aws-chunked") + setAwsChunkedContentEncoding(&req) req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(req.ContentLength, 10)) } @@ -395,7 +395,7 @@ func UnsignedTrailer(req http.Request, trailer http.Header) *http.Request { req.Header.Add("X-Amz-Trailer", strings.ToLower(k)) } - req.Header.Set("Content-Encoding", "aws-chunked") + setAwsChunkedContentEncoding(&req) req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(req.ContentLength, 10)) // Use custom chunked encoding. diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/signer/utils.go b/src/vendor/github.com/minio/minio-go/v7/pkg/signer/utils.go index 87c99398..20172c2f 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/signer/utils.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/signer/utils.go @@ -22,6 +22,8 @@ import ( "crypto/sha256" "net/http" "strings" + + "golang.org/x/net/http/httpguts" ) // unsignedPayload - value to be set to X-Amz-Content-Sha256 header when @@ -60,3 +62,26 @@ func signV4TrimAll(input string) string { // unicode.IsSpace() internally here) to one space and return return strings.Join(strings.Fields(input), " ") } + +// awsChunkedEncoding is the content coding AWS SigV4 streaming uploads +// declare on the Content-Encoding header. +const awsChunkedEncoding = "aws-chunked" + +// setAwsChunkedContentEncoding marks the request payload as aws-chunked +// encoded, keeping any content encoding the caller already set, for example +// "aws-chunked,gzip". AWS SigV4 streaming uploads require this header. +// Assigning "aws-chunked" to http.Request.TransferEncoding never reaches +// the wire — net/http silently drops assigned values other than "chunked" — +// so this header is the only signal that declares the aws-chunked framing. +func setAwsChunkedContentEncoding(req *http.Request) { + encodings := req.Header.Values("Content-Encoding") + if httpguts.HeaderValuesContainsToken(encodings, awsChunkedEncoding) { + return + } + existing := strings.TrimSpace(strings.Join(encodings, ",")) + if existing == "" { + req.Header.Set("Content-Encoding", awsChunkedEncoding) + return + } + req.Header.Set("Content-Encoding", awsChunkedEncoding+","+existing) +} diff --git a/src/vendor/github.com/minio/minio-go/v7/pkg/singleflight/singleflight.go b/src/vendor/github.com/minio/minio-go/v7/pkg/singleflight/singleflight.go index 49260327..ece42b9e 100644 --- a/src/vendor/github.com/minio/minio-go/v7/pkg/singleflight/singleflight.go +++ b/src/vendor/github.com/minio/minio-go/v7/pkg/singleflight/singleflight.go @@ -172,10 +172,12 @@ func (g *Group[K, V]) doCall(c *call[V], key K, fn func() (V, error)) { } else { panic(e) } - } else if c.err == errGoexit { - // Already in the process of goexit, no need to call again } else { - // Normal return + // Normal return, and runtime.Goexit, which reaches here with + // c.err set to errGoexit. This goroutine is already being torn + // down and must not call runtime.Goexit again: Do callers + // re-Goexit themselves once they see c.err, but DoChan callers + // hold a channel and have nothing else to wake them. for _, ch := range c.chans { ch <- Result[V]{c.val, c.err, c.dups > 0} } diff --git a/src/vendor/github.com/minio/minio-go/v7/rdma.go b/src/vendor/github.com/minio/minio-go/v7/rdma.go index 450c9620..48d2c87f 100644 --- a/src/vendor/github.com/minio/minio-go/v7/rdma.go +++ b/src/vendor/github.com/minio/minio-go/v7/rdma.go @@ -26,6 +26,25 @@ import ( var ErrRDMANotConnected = errors.New("RDMA infrastructure not connected") +// maxRDMABufferSize is the largest transfer a single RDMA descriptor can +// describe: the x-amz-rdma-token carries the window size in a 32-bit field. +// libminiocpp itself does not fail on a larger buffer: it skips RDMA and +// transfers it over one ordinary HTTP request. The guards below reject it +// instead, so an oversize buffer does not silently become a much slower HTTP +// transfer on a path the caller explicitly opted into for RDMA. Keep this in +// step with kRDMAMaxMemoryRegSize in minio-cpp -- a value even one byte larger +// lets a buffer through here that libminiocpp then declines. The lower bound +// also stops a negative size from wrapping to a huge C.size_t. +const maxRDMABufferSize int64 = 1<<32 - 1 // 4 GiB - 1 + +// rdmaBufferSizeInRange reports whether a descriptor can name this many bytes. +// Both entry points gate on it before converting to C.size_t, so it is the one +// place the bound is expressed -- and the only part of the guard that can be +// exercised without an RDMA device. +func rdmaBufferSizeInRange(size int) bool { + return size >= 0 && int64(size) <= maxRDMABufferSize +} + type rdmaClientHandle struct { cptr *C.miniocpp_client } @@ -63,6 +82,12 @@ func newRDMAClient(c *Client) (*rdmaClientHandle, error) { func (c *Client) putObjectRDMA(_ context.Context, bucketName, objectName string, opts PutObjectOptions, ) (UploadInfo, error) { + if !rdmaBufferSizeInRange(opts.RDMABufferSize) { + return UploadInfo{}, fmt.Errorf( + "RDMA put: buffer size %d must be between 0 and the %d bytes an RDMA descriptor can describe", + opts.RDMABufferSize, maxRDMABufferSize) + } + h, err := c.rdma() if err != nil { return UploadInfo{}, err @@ -93,6 +118,12 @@ func (c *Client) putObjectRDMA(_ context.Context, bucketName, objectName string, func (c *Client) getObjectRDMA(_ context.Context, bucketName, objectName string, opts GetObjectOptions, ) (int64, error) { + if !rdmaBufferSizeInRange(opts.RDMABufferSize) { + return 0, fmt.Errorf( + "RDMA get: buffer size %d must be between 0 and the %d bytes an RDMA descriptor can describe", + opts.RDMABufferSize, maxRDMABufferSize) + } + h, err := c.rdma() if err != nil { return 0, err @@ -136,5 +167,7 @@ func AlignedBuffer(n int) unsafe.Pointer { // FreeAlignedBuffer releases a buffer from AlignedBuffer. func FreeAlignedBuffer(p unsafe.Pointer) { C.miniocpp_free_aligned(p) } -// IsRDMAAvailable reports whether cuObj is connected to a cuObjServer. +// IsRDMAAvailable reports whether this host has a usable RDMA device, so an +// RDMA transfer is worth attempting. Whether the server serves RDMA for a +// given object is answered per request, by x-amz-rdma-reply. func IsRDMAAvailable() bool { return C.miniocpp_rdma_available() != 0 } diff --git a/src/vendor/github.com/minio/minio-go/v7/s3-error.go b/src/vendor/github.com/minio/minio-go/v7/s3-error.go index 4bcc47d8..4ff34156 100644 --- a/src/vendor/github.com/minio/minio-go/v7/s3-error.go +++ b/src/vendor/github.com/minio/minio-go/v7/s3-error.go @@ -62,6 +62,7 @@ const ( InvalidDuration = "InvalidDuration" XAmzContentSHA256Mismatch = "XAmzContentSHA256Mismatch" XMinioInvalidObjectName = "XMinioInvalidObjectName" + XMinioPaidTierLicenseRequired = "XMinioPaidTierLicenseRequired" NoSuchCORSConfiguration = "NoSuchCORSConfiguration" BucketAlreadyExists = "BucketAlreadyExists" NoSuchVersion = "NoSuchVersion" diff --git a/src/vendor/github.com/minio/minio-go/v7/utils.go b/src/vendor/github.com/minio/minio-go/v7/utils.go index 3361464c..48e4a47b 100644 --- a/src/vendor/github.com/minio/minio-go/v7/utils.go +++ b/src/vendor/github.com/minio/minio-go/v7/utils.go @@ -33,6 +33,7 @@ import ( "mime" "net" "net/http" + "net/textproto" "net/url" "regexp" "strconv" @@ -63,7 +64,7 @@ func amzExpirationToExpiryDateRuleID(expiration string) (time.Time, string) { return time.Time{}, "" } -var restoreRegex = regexp.MustCompile(`ongoing-request="(.*?)"(, expiry-date="(.*?)")?`) +var restoreRegex = regexp.MustCompile(`ongoing-request="(.*?)"(, ?expiry-date="(.*?)")?`) func amzRestoreToStruct(restore string) (ongoing bool, expTime time.Time, err error) { matches := restoreRegex.FindStringSubmatch(restore) @@ -105,8 +106,32 @@ func sumMD5Base64(data []byte) string { return base64.StdEncoding.EncodeToString(hash.Sum(nil)) } -// getEndpointURL - construct a new endpoint. +// getEndpointURL - construct a new endpoint from a host[:port] endpoint +// or an http(s):// URL whose scheme agrees with the secure option. func getEndpointURL(endpoint string, secure bool) (*url.URL, error) { + // An endpoint that already carries a scheme is parsed directly instead + // of prefixing another scheme. It must agree with the secure option + // since signing and transport behavior are derived from that option. + // A "://" preceded by a path, query, or fragment delimiter is endpoint + // data rather than a scheme, so such endpoints keep the scheme-less path. + if i := strings.Index(endpoint, "://"); i == 0 || (i > 0 && !strings.ContainsAny(endpoint[:i], "/?#")) { + scheme := strings.ToLower(endpoint[:i]) + if scheme != "http" && scheme != "https" { + return nil, errInvalidArgument("Endpoint url scheme \"" + scheme + "\" is unsupported; use http or https or omit the scheme.") + } + endpointURL, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + if secure != (endpointURL.Scheme == "https") { + return nil, errInvalidArgument("Endpoint url scheme \"" + endpointURL.Scheme + "\" conflicts with the secure option; remove the scheme from the endpoint or align the secure option.") + } + if err := isValidEndpointURL(*endpointURL); err != nil { + return nil, err + } + return endpointURL, nil + } + // If secure is false, use 'http' scheme. scheme := "https" if !secure { @@ -199,6 +224,9 @@ func isValidExpiry(expires time.Duration) error { return nil } +// amzMetaPrefix is the canonical prefix of S3 user metadata headers. +const amzMetaPrefix = "X-Amz-Meta-" + // Extract only necessary metadata header key/values by // filtering them out with a list of custom header keys. func extractObjMetadata(header http.Header) http.Header { @@ -215,7 +243,7 @@ func extractObjMetadata(header http.Header) http.Header { "X-Amz-Website-Redirect-Location", "X-Amz-Server-Side-Encryption", "X-Amz-Tagging-Count", - "X-Amz-Meta-", + amzMetaPrefix, "X-Minio-Meta-", // Add new headers to be preserved. // if you add new headers here, please extend @@ -230,7 +258,7 @@ func extractObjMetadata(header http.Header) http.Header { continue } found = true - if prefix == "X-Amz-Meta-" || prefix == "X-Minio-Meta-" { + if prefix == amzMetaPrefix || prefix == "X-Minio-Meta-" { for index, val := range v { if strings.HasPrefix(val, "=?") { decoder := mime.WordDecoder{} @@ -249,8 +277,29 @@ func extractObjMetadata(header http.Header) http.Header { return filteredHeader } +// stripUserMetadata converts the raw element of MinIO list +// responses into the keyed form StatObject and GetObject return in +// ObjectInfo.UserMetadata: only "X-Amz-Meta-*" entries are kept, with the +// prefix stripped and values passed through verbatim (list responses carry +// the stored values, so no decoding applies). Returns nil if raw contains +// no user metadata. +func stripUserMetadata(raw StringMap) StringMap { + var stripped StringMap + for k, v := range raw { + k = textproto.CanonicalMIMEHeaderKey(k) + if !strings.HasPrefix(k, amzMetaPrefix) { + continue + } + if stripped == nil { + stripped = make(StringMap, len(raw)) + } + stripped[strings.TrimPrefix(k, amzMetaPrefix)] = v + } + return stripped +} + const ( - // RFC 7231#section-7.1.1.1 timetamp format. e.g Tue, 29 Apr 2014 18:30:38 GMT + // RFC 7231#section-7.1.1.1 timestamp format. e.g Tue, 29 Apr 2014 18:30:38 GMT rfc822TimeFormat = "Mon, 2 Jan 2006 15:04:05 GMT" rfc822TimeFormatSingleDigitDay = "Mon, _2 Jan 2006 15:04:05 GMT" rfc822TimeFormatSingleDigitDayTwoDigitYear = "Mon, _2 Jan 06 15:04:05 GMT" @@ -351,8 +400,8 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err metadata := extractObjMetadata(h) userMetadata := make(map[string]string) for k, v := range metadata { - if strings.HasPrefix(k, "X-Amz-Meta-") { - userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0] + if strings.HasPrefix(k, amzMetaPrefix) { + userMetadata[strings.TrimPrefix(k, amzMetaPrefix)] = v[0] } } @@ -412,6 +461,7 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err // following function filters out a list of standard set of keys // which are not part of object metadata. Metadata: metadata, + Headers: h, UserMetadata: userMetadata, UserTags: userTags.ToMap(), UserTagCount: tagCount, diff --git a/src/vendor/go.yaml.in/yaml/v3/parserc.go b/src/vendor/go.yaml.in/yaml/v3/parserc.go index 25fe8236..f35829db 100644 --- a/src/vendor/go.yaml.in/yaml/v3/parserc.go +++ b/src/vendor/go.yaml.in/yaml/v3/parserc.go @@ -226,9 +226,9 @@ func yaml_parser_state_machine(parser *yaml_parser_t, event *yaml_event_t) bool } // Parse the production: -// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END // -// ************ +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// ************ func yaml_parser_parse_stream_start(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -249,13 +249,11 @@ func yaml_parser_parse_stream_start(parser *yaml_parser_t, event *yaml_event_t) } // Parse the productions: -// implicit_document ::= block_node DOCUMENT-END* -// -// * // -// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* -// -// ************************* +// implicit_document ::= block_node DOCUMENT-END* +// * +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// ************************* func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t, implicit bool) bool { token := peek_token(parser) @@ -359,9 +357,9 @@ func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t } // Parse the productions: -// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* // -// *********** +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// *********** func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -382,11 +380,10 @@ func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event } // Parse the productions: -// implicit_document ::= block_node DOCUMENT-END* -// -// ************* // -// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// implicit_document ::= block_node DOCUMENT-END* +// ************* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* func yaml_parser_parse_document_end(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -432,42 +429,32 @@ func yaml_parser_set_event_comments(parser *yaml_parser_t, event *yaml_event_t) } // Parse the productions: -// block_node_or_indentless_sequence ::= -// -// ALIAS -// ***** -// | properties (block_content | indentless_block_sequence)? -// ********** * -// | block_content | indentless_block_sequence -// * -// -// block_node ::= ALIAS -// -// ***** -// | properties block_content? -// ********** * -// | block_content -// * -// -// flow_node ::= ALIAS -// -// ***** -// | properties flow_content? -// ********** * -// | flow_content -// * -// -// properties ::= TAG ANCHOR? | ANCHOR TAG? -// -// ************************* -// -// block_content ::= block_collection | flow_collection | SCALAR -// -// ****** // -// flow_content ::= flow_collection | SCALAR -// -// ****** +// block_node_or_indentless_sequence ::= +// ALIAS +// ***** +// | properties (block_content | indentless_block_sequence)? +// ********** * +// | block_content | indentless_block_sequence +// * +// block_node ::= ALIAS +// ***** +// | properties block_content? +// ********** * +// | block_content +// * +// flow_node ::= ALIAS +// ***** +// | properties flow_content? +// ********** * +// | flow_content +// * +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// ************************* +// block_content ::= block_collection | flow_collection | SCALAR +// ****** +// flow_content ::= flow_collection | SCALAR +// ****** func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, indentless_sequence bool) bool { //defer trace("yaml_parser_parse_node", "block:", block, "indentless_sequence:", indentless_sequence)() @@ -697,9 +684,9 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i } // Parse the productions: -// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END // -// ******************** *********** * ********* +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// ******************** *********** * ********* func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { if first { token := peek_token(parser) @@ -755,9 +742,9 @@ func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_e } // Parse the productions: -// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ // -// *********** * +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// *********** * func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -821,15 +808,15 @@ func yaml_parser_split_stem_comment(parser *yaml_parser_t, stem_len int) { } // Parse the productions: -// block_mapping ::= BLOCK-MAPPING_START // -// ******************* -// ((KEY block_node_or_indentless_sequence?)? -// *** * -// (VALUE block_node_or_indentless_sequence?)?)* +// block_mapping ::= BLOCK-MAPPING_START +// ******************* +// ((KEY block_node_or_indentless_sequence?)? +// *** * +// (VALUE block_node_or_indentless_sequence?)?)* // -// BLOCK-END -// ********* +// BLOCK-END +// ********* func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { if first { token := peek_token(parser) @@ -896,13 +883,14 @@ func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_even } // Parse the productions: -// block_mapping ::= BLOCK-MAPPING_START // -// ((KEY block_node_or_indentless_sequence?)? +// block_mapping ::= BLOCK-MAPPING_START +// +// ((KEY block_node_or_indentless_sequence?)? // -// (VALUE block_node_or_indentless_sequence?)?)* -// ***** * -// BLOCK-END +// (VALUE block_node_or_indentless_sequence?)?)* +// ***** * +// BLOCK-END func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -929,19 +917,17 @@ func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_ev } // Parse the productions: -// flow_sequence ::= FLOW-SEQUENCE-START -// -// ******************* -// (flow_sequence_entry FLOW-ENTRY)* -// * ********** -// flow_sequence_entry? -// * -// FLOW-SEQUENCE-END -// ***************** // -// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? -// -// * +// flow_sequence ::= FLOW-SEQUENCE-START +// ******************* +// (flow_sequence_entry FLOW-ENTRY)* +// * ********** +// flow_sequence_entry? +// * +// FLOW-SEQUENCE-END +// ***************** +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { if first { token := peek_token(parser) @@ -1005,9 +991,9 @@ func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_ev } // Parse the productions: -// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? // -// *** * +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// *** * func yaml_parser_parse_flow_sequence_entry_mapping_key(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -1026,9 +1012,9 @@ func yaml_parser_parse_flow_sequence_entry_mapping_key(parser *yaml_parser_t, ev } // Parse the productions: -// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? // -// ***** * +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// ***** * func yaml_parser_parse_flow_sequence_entry_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -1050,9 +1036,9 @@ func yaml_parser_parse_flow_sequence_entry_mapping_value(parser *yaml_parser_t, } // Parse the productions: -// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? // -// * +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * func yaml_parser_parse_flow_sequence_entry_mapping_end(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -1068,18 +1054,17 @@ func yaml_parser_parse_flow_sequence_entry_mapping_end(parser *yaml_parser_t, ev } // Parse the productions: -// flow_mapping ::= FLOW-MAPPING-START -// -// ****************** -// (flow_mapping_entry FLOW-ENTRY)* -// * ********** -// flow_mapping_entry? -// ****************** -// FLOW-MAPPING-END -// **************** // -// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? -// - *** * +// flow_mapping ::= FLOW-MAPPING-START +// ****************** +// (flow_mapping_entry FLOW-ENTRY)* +// * ********** +// flow_mapping_entry? +// ****************** +// FLOW-MAPPING-END +// **************** +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * *** * func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { if first { token := peek_token(parser) @@ -1144,8 +1129,9 @@ func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event } // Parse the productions: -// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? -// - ***** * +// +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * ***** * func yaml_parser_parse_flow_mapping_value(parser *yaml_parser_t, event *yaml_event_t, empty bool) bool { token := peek_token(parser) if token == nil { diff --git a/src/vendor/go.yaml.in/yaml/v3/yamlh.go b/src/vendor/go.yaml.in/yaml/v3/yamlh.go index f59aa40f..07c44236 100644 --- a/src/vendor/go.yaml.in/yaml/v3/yamlh.go +++ b/src/vendor/go.yaml.in/yaml/v3/yamlh.go @@ -433,21 +433,19 @@ type yaml_document_t struct { // The prototype of a read handler. // -// The read handler is called when the parser needs to read more bytes from the -// source. The handler should write not more than size bytes to the buffer. -// The number of written bytes should be set to the size_read variable. +// The read handler is called when the parser needs to read more bytes from the +// source. The handler should write not more than size bytes to the buffer. +// The number of written bytes should be set to the size_read variable. // -// [in,out] data A pointer to an application data specified by +// [in,out] data A pointer to an application data specified by +// yaml_parser_set_input(). +// [out] buffer The buffer to write the data from the source. +// [in] size The size of the buffer. +// [out] size_read The actual number of bytes read from the source. // -// yaml_parser_set_input(). -// -// [out] buffer The buffer to write the data from the source. -// [in] size The size of the buffer. -// [out] size_read The actual number of bytes read from the source. -// -// On success, the handler should return 1. If the handler failed, -// the returned value should be 0. On EOF, the handler should set the -// size_read to 0 and return 1. +// On success, the handler should return 1. If the handler failed, +// the returned value should be 0. On EOF, the handler should set the +// size_read to 0 and return 1. type yaml_read_handler_t func(parser *yaml_parser_t, buffer []byte) (n int, err error) // This structure holds information about a potential simple key. @@ -655,19 +653,17 @@ type yaml_comment_t struct { // The prototype of a write handler. // -// The write handler is called when the emitter needs to flush the accumulated -// characters to the output. The handler should write @a size bytes of the -// @a buffer to the output. -// -// @param[in,out] data A pointer to an application data specified by -// -// yaml_emitter_set_output(). +// The write handler is called when the emitter needs to flush the accumulated +// characters to the output. The handler should write @a size bytes of the +// @a buffer to the output. // -// @param[in] buffer The buffer with bytes to be written. -// @param[in] size The size of the buffer. +// @param[in,out] data A pointer to an application data specified by +// yaml_emitter_set_output(). +// @param[in] buffer The buffer with bytes to be written. +// @param[in] size The size of the buffer. // -// @returns On success, the handler should return @c 1. If the handler failed, -// the returned value should be @c 0. +// @returns On success, the handler should return @c 1. If the handler failed, +// the returned value should be @c 0. type yaml_write_handler_t func(emitter *yaml_emitter_t, buffer []byte) error type yaml_emitter_state_t int diff --git a/src/vendor/golang.org/x/crypto/acme/types.go b/src/vendor/golang.org/x/crypto/acme/types.go index 65d69b26..4df7e6df 100644 --- a/src/vendor/golang.org/x/crypto/acme/types.go +++ b/src/vendor/golang.org/x/crypto/acme/types.go @@ -114,14 +114,15 @@ type Error struct { } func (e *Error) Error() string { - str := fmt.Sprintf("%d %s: %s", e.StatusCode, e.ProblemType, e.Detail) + var sb strings.Builder + fmt.Fprintf(&sb, "%d %s: %s", e.StatusCode, e.ProblemType, e.Detail) if len(e.Subproblems) > 0 { - str += fmt.Sprintf("; subproblems:") + sb.WriteString("; subproblems:") for _, sp := range e.Subproblems { - str += fmt.Sprintf("\n\t%s", sp) + fmt.Fprintf(&sb, "\n\t%s", sp) } } - return str + return sb.String() } // AuthorizationError indicates that an authorization for an identifier diff --git a/src/vendor/golang.org/x/crypto/ocsp/ocsp.go b/src/vendor/golang.org/x/crypto/ocsp/ocsp.go index e6c645e7..23c39e73 100644 --- a/src/vendor/golang.org/x/crypto/ocsp/ocsp.go +++ b/src/vendor/golang.org/x/crypto/ocsp/ocsp.go @@ -85,7 +85,8 @@ type certID struct { // https://tools.ietf.org/html/rfc2560#section-4.1.1 type ocspRequest struct { - TBSRequest tbsRequest + TBSRequest tbsRequest + OptionalSignature asn1.RawValue `asn1:"explicit,tag:0,optional"` } type tbsRequest struct { @@ -321,10 +322,10 @@ type Request struct { func (req *Request) Marshal() ([]byte, error) { hashAlg := getOIDFromHashAlgorithm(req.HashAlgorithm) if hashAlg == nil { - return nil, errors.New("Unknown hash algorithm") + return nil, errors.New("unknown hash algorithm") } return asn1.Marshal(ocspRequest{ - tbsRequest{ + TBSRequest: tbsRequest{ Version: 0, RequestList: []request{ { @@ -418,8 +419,10 @@ func (p ParseError) Error() string { } // ParseRequest parses an OCSP request in DER form. It only supports -// requests for a single certificate. Signed requests are not supported. -// If a request includes a signature, it will result in a ParseError. +// requests for a single certificate identifier. If a request includes +// multiple certificate identifiers, only the first will be included in +// the parsed Request. Signed requests are not supported. If a request +// includes a signature, it will result in a ParseError. func ParseRequest(bytes []byte) (*Request, error) { var req ocspRequest rest, err := asn1.Unmarshal(bytes, &req) @@ -430,6 +433,10 @@ func ParseRequest(bytes []byte) (*Request, error) { return nil, ParseError("trailing data in OCSP request") } + if len(req.OptionalSignature.FullBytes) > 0 { + return nil, ParseError("signed OCSP requests are not supported") + } + if len(req.TBSRequest.RequestList) == 0 { return nil, ParseError("OCSP request contains no request body") } diff --git a/src/vendor/golang.org/x/text/secure/precis/nickname.go b/src/vendor/golang.org/x/text/secure/precis/nickname.go index 11e0ccbb..73b8f0e7 100644 --- a/src/vendor/golang.org/x/text/secure/precis/nickname.go +++ b/src/vendor/golang.org/x/text/secure/precis/nickname.go @@ -44,24 +44,25 @@ func (t *nickAdditionalMapping) Transform(dst, src []byte, atEOF bool) (nDst, nS // to a single ASCII space character (e.g., "St Peter" is // mapped to "St Peter"). for nSrc < len(src) { - r, size := utf8.DecodeRune(src[nSrc:]) - if size == 0 { // Incomplete UTF-8 encoding - if !atEOF { - return nDst, nSrc, transform.ErrShortSrc - } - size = 1 + if !utf8.FullRune(src[nSrc:]) && !atEOF { + return nDst, nSrc, transform.ErrShortSrc } + r, size := utf8.DecodeRune(src[nSrc:]) if unicode.Is(unicode.Zs, r) { t.prevSpace = true } else { if t.prevSpace && t.notStart { + if nDst >= len(dst) { + return nDst, nSrc, transform.ErrShortDst + } dst[nDst] = ' ' nDst += 1 + t.prevSpace = false } - if size != copy(dst[nDst:], src[nSrc:nSrc+size]) { - nDst += size + if len(dst)-nDst < size { return nDst, nSrc, transform.ErrShortDst } + copy(dst[nDst:], src[nSrc:nSrc+size]) nDst += size t.prevSpace = false t.notStart = true diff --git a/src/vendor/golang.org/x/text/secure/precis/profile.go b/src/vendor/golang.org/x/text/secure/precis/profile.go index bdd991bb..e62e79ba 100644 --- a/src/vendor/golang.org/x/text/secure/precis/profile.go +++ b/src/vendor/golang.org/x/text/secure/precis/profile.go @@ -349,13 +349,13 @@ func (c *checker) Reset() { func (c *checker) span(src []byte, atEOF bool) (n int, err error) { for n < len(src) { e, sz := dpTrie.lookup(src[n:]) - d := categoryTransitions[category(e&catMask)] if sz == 0 { if !atEOF { return n, transform.ErrShortSrc } return n, errDisallowedRune } + d := categoryTransitions[category(e&catMask)] doLookAhead := false if property(e) < c.p.class.validFrom { if d.rule == nil { @@ -389,6 +389,9 @@ func (c *checker) span(src []byte, atEOF bool) (n int, err error) { n += sz } if m := c.beforeBits >> finalShift; c.beforeBits&m != m || c.termBits != 0 { + if !atEOF { + return n, transform.ErrShortSrc + } err = errContext } return n, err @@ -396,8 +399,9 @@ func (c *checker) span(src []byte, atEOF bool) (n int, err error) { // TODO: we may get rid of this transform if transform.Chain understands // something like a Spanner interface. -func (c checker) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { +func (c *checker) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { short := false + if len(dst) < len(src) { src = src[:len(dst)] atEOF = false diff --git a/src/vendor/modules.txt b/src/vendor/modules.txt index c4e2edb0..bb46eccd 100644 --- a/src/vendor/modules.txt +++ b/src/vendor/modules.txt @@ -119,7 +119,7 @@ github.com/jackc/puddle/v2/internal/genstack # github.com/joho/godotenv v1.5.1 ## explicit; go 1.12 github.com/joho/godotenv -# github.com/klauspost/compress v1.19.1 +# github.com/klauspost/compress v1.19.2 ## explicit; go 1.24 github.com/klauspost/compress github.com/klauspost/compress/flate @@ -160,7 +160,7 @@ github.com/minio/crc64nvme # github.com/minio/md5-simd v1.1.2 ## explicit; go 1.14 github.com/minio/md5-simd -# github.com/minio/minio-go/v7 v7.2.1 +# github.com/minio/minio-go/v7 v7.3.0 ## explicit; go 1.25.0 github.com/minio/minio-go/v7 github.com/minio/minio-go/v7/pkg/cors @@ -169,6 +169,7 @@ github.com/minio/minio-go/v7/pkg/encrypt github.com/minio/minio-go/v7/pkg/kvcache github.com/minio/minio-go/v7/pkg/lifecycle github.com/minio/minio-go/v7/pkg/notification +github.com/minio/minio-go/v7/pkg/policy github.com/minio/minio-go/v7/pkg/replication github.com/minio/minio-go/v7/pkg/s3utils github.com/minio/minio-go/v7/pkg/set @@ -267,10 +268,10 @@ go.uber.org/atomic # go.uber.org/multierr v1.11.0 ## explicit; go 1.19 go.uber.org/multierr -# go.yaml.in/yaml/v3 v3.0.4 +# go.yaml.in/yaml/v3 v3.0.5 ## explicit; go 1.16 go.yaml.in/yaml/v3 -# golang.org/x/crypto v0.54.0 +# golang.org/x/crypto v0.55.0 ## explicit; go 1.25.0 golang.org/x/crypto/acme golang.org/x/crypto/acme/autocert @@ -285,7 +286,7 @@ golang.org/x/image/font/opentype golang.org/x/image/font/sfnt golang.org/x/image/math/fixed golang.org/x/image/vector -# golang.org/x/net v0.57.0 +# golang.org/x/net v0.58.0 ## explicit; go 1.25.0 golang.org/x/net/html golang.org/x/net/html/atom @@ -304,7 +305,7 @@ golang.org/x/sync/semaphore golang.org/x/sys/cpu golang.org/x/sys/unix golang.org/x/sys/windows -# golang.org/x/text v0.40.0 +# golang.org/x/text v0.41.0 ## explicit; go 1.25.0 golang.org/x/text/cases golang.org/x/text/encoding