Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 30 additions & 1 deletion ndarray.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ package ndarray
import (
"errors"
"fmt"
"math"
"strconv"
"strings"

Expand Down Expand Up @@ -45,12 +46,28 @@ func prod(dims []int) int {
return n
}

// validateShape reports whether every dimension is non-negative.
// maxSize is the largest element count whose float64 storage size in bytes
// still fits in an int — numpy's "array is too big" bound.
const maxSize = math.MaxInt / 8

// validateShape reports whether every dimension is non-negative and the
// element count is representable. The count check multiplies the non-zero
// dimensions with an overflow guard, so a shape whose product would wrap (e.g.
// (1<<62+1, 4), which wraps to exactly 4) is rejected rather than accepted
// with a wrong Size; like numpy, a zero dimension does not excuse the others.
func validateShape(shape []int) error {
n := 1
for _, d := range shape {
if d < 0 {
return fmt.Errorf("%w: negative dimension %d", ErrShapeMismatch, d)
}
if d == 0 {
continue
}
if d > maxSize/n {
return fmt.Errorf("%w: shape %v is too big", ErrShapeMismatch, shape)
}
n *= d
}
return nil
}
Expand Down Expand Up @@ -327,6 +344,9 @@ func (a *Array) Reshape(shape ...int) (*Array, error) {
if err != nil {
return nil, err
}
if err := validateShape(shape); err != nil {
return nil, err
}
if prod(shape) != a.Size() {
return nil, fmt.Errorf("%w: cannot reshape size %d into %v",
ErrShapeMismatch, a.Size(), shape)
Expand Down Expand Up @@ -427,6 +447,10 @@ func broadcastShape(s1, s2 []int) ([]int, error) {
return nil, fmt.Errorf("%w: %v vs %v", ErrBroadcast, s1, s2)
}
}
// Two representable shapes can broadcast to one that is not.
if err := validateShape(out); err != nil {
return nil, err
}
return out, nil
}

Expand All @@ -450,6 +474,11 @@ func (a *Array) broadcastTo(shape []int) []float64 {
if len(out) == 0 {
return out
}
if n == 0 {
// 0-d target: the receiver is a 0-d (possibly offset) view too.
out[0] = a.data[a.offset]
return out
}
// Fast path: the innermost axis is materialised with unit stride (it is
// either a real, non-broadcast last axis or a broadcast length-1 one). Then
// each contiguous inner run is a single copy() — for a broadcast last axis the
Expand Down
90 changes: 90 additions & 0 deletions overflow_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package ndarray

import (
"errors"
"math"
"testing"
)

// TestShapeSizeOverflow checks that a shape whose element count overflows int
// is rejected instead of wrapping. Before the fix prod() wrapped silently:
// New(1<<62, 4) returned an array of Size 0, and FromData/Reshape accepted
// (1<<62+1, 4) for 4 elements because the product wrapped to exactly 4, after
// which At(1, 0) panicked. numpy raises "array is too big" for all of these.
func TestShapeSizeOverflow(t *testing.T) {
huge := [][]int{
{1 << 62, 4},
{1<<62 + 1, 4},
{1 << 59, 8},
{math.MaxInt, 2},
{0, 1 << 62, 1 << 62}, // numpy rejects this too, despite the zero
}
for _, s := range huge {
if _, err := New(s...); !errors.Is(err, ErrShapeMismatch) {
t.Errorf("New(%v) err = %v, want ErrShapeMismatch", s, err)
}
}
if _, err := FromData([]float64{1, 2, 3, 4}, 1<<62+1, 4); !errors.Is(err, ErrShapeMismatch) {
t.Errorf("FromData wrapped shape err = %v", err)
}
a := mustArr(t, ok(Arange(0, 4, 1)))
if _, err := a.Reshape(1<<62+1, 4); !errors.Is(err, ErrShapeMismatch) {
t.Errorf("Reshape wrapped shape err = %v", err)
}
if _, err := a.Reshape(-1, 1<<62+1, 4); !errors.Is(err, ErrShapeMismatch) {
t.Errorf("Reshape wrapped shape with -1 err = %v", err)
}
// Two valid shapes can broadcast to an unrepresentable one.
col := &Array{data: []float64{1}, shape: []int{1 << 40, 1}, strides: []int{0, 0}}
row := &Array{data: []float64{1}, shape: []int{1, 1 << 40}, strides: []int{0, 0}}
if _, err := col.Add(row); !errors.Is(err, ErrShapeMismatch) {
t.Errorf("broadcast to (2^40, 2^40) err = %v", err)
}
// Large but representable shapes are still fine.
if _, err := New(0, 1<<40); err != nil {
t.Errorf("New(0, 1<<40) err = %v", err)
}
}

// TestSliceHugeStep checks the element count of a slice whose |step| exceeds
// the axis length. Before the fix ceil((stop-start)/step) was computed as
// (stop-start+step-1)/step, which overflows for a step near MaxInt and gave an
// empty view; numpy gives one element (Python a[0:5:sys.maxsize] == a[0:1]).
func TestSliceHugeStep(t *testing.T) {
x := mustArr(t, ok(Arange(0, 6, 1)))
for _, c := range []struct {
name string
ix Index
want []float64
}{
{"0:5:MaxInt", Rng(0, 5, math.MaxInt), []float64{0}},
{"4:0:-MaxInt", Rng(4, 0, -math.MaxInt), []float64{4}},
{"::-MaxInt", Step(-math.MaxInt), []float64{5}},
{"::MaxInt", Step(math.MaxInt), []float64{0}},
{"::MinInt", Step(math.MinInt), []float64{5}},
{"5:0:MaxInt", Rng(5, 0, math.MaxInt), []float64{}},
} {
s, err := x.Slice(c.ix)
if err != nil {
t.Fatalf("%s: %v", c.name, err)
}
wantData(t, s, []int{len(c.want)}, c.want)
}
}

// TestZeroDimViewBroadcast checks elementwise ops on a 0-d view that does not
// start at offset 0 (x[2] of a 1-D x). Such a view is not "contiguous", so it
// takes the broadcasting path, which indexed shape[n-1] with n == 0 and
// panicked with "index out of range [-1]".
func TestZeroDimViewBroadcast(t *testing.T) {
x := mustArr(t, ok(Arange(0, 6, 1)))
s := mustArr(t, ok(x.Slice(A(2))))
r := mustArr(t, ok(s.Add(s)))
wantData(t, r, nil, []float64{4}) // Shape() of a 0-d array is nil
r = mustArr(t, ok(s.Mul(x)))
wantData(t, r, []int{6}, []float64{0, 2, 4, 6, 8, 10})
m := mustArr(t, ok(s.MaskSelect(s)))
wantData(t, m, []int{1}, []float64{2})
w := mustArr(t, ok(Where(s, s, x)))
wantData(t, w, []int{6}, []float64{2, 2, 2, 2, 2, 2})
}
9 changes: 7 additions & 2 deletions slicing.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,11 @@ func (ix Index) resolveRange(length int) (first, count, step int, err error) {
if ix.hasStop {
stop = normBound(ix.stop, length, 0, length)
}
// ceil((stop-first)/step), written so that it cannot overflow for a
// step near MaxInt (stop-first+step-1 would).
count = 0
if stop > first {
count = (stop - first + step - 1) / step
count = (stop-first-1)/step + 1
}
return first, count, step, nil
}
Expand All @@ -115,9 +117,12 @@ func (ix Index) resolveRange(length int) (first, count, step int, err error) {
if ix.hasStop {
stop = normBound(ix.stop, length, -1, length-1)
}
// ceil((first-stop)/-step) without overflow. For step == MinInt, -step
// wraps to MinInt; the quotient is then 0 (the dividend is smaller in
// magnitude), which still yields the right count of 1.
count = 0
if first > stop {
count = (first - stop - step - 1) / (-step)
count = (first-stop-1)/(-step) + 1
}
return first, count, step, nil
}
Expand Down
Loading