From e8589c124472c5f0319b623a105baad1347cb7dd Mon Sep 17 00:00:00 2001 From: Matthew Nibecker Date: Tue, 8 Sep 2026 15:42:59 -0700 Subject: [PATCH 1/3] Add -static flag This commit adds the -static flag which when enabled, forces static type checking on all input files. By default csup and parquet are statically checked- all other file types are dynamic. If the -static flag is enabled the entire file is read and the full type information is gathered. Non-seekable inputs (e.g., /dev/stdin) cannot be statically checked an will return an error -static is enabled. This commit removes the -samplesize flag. --- cli/inputflags/flags.go | 7 +- cmd/super/compile/shared.go | 9 ++- cmd/super/root/command.go | 2 +- cmd/super/ztests/samplesize.yaml | 38 ---------- compiler/dag/op.go | 10 ++- compiler/semantic/dagen.go | 1 + compiler/semantic/op.go | 6 +- compiler/semantic/ztests/checker-case.yaml | 4 +- .../semantic/ztests/checker-cond-fail.yaml | 2 +- compiler/semantic/ztests/from-json.yaml | 2 +- compiler/semantic/ztests/from-type.yaml | 76 +++++++++++++++++++ compiler/sfmt/dag.go | 12 +++ compiler/sfmt/ztests/input-files.yaml | 4 +- compiler/ztests/pruner.yaml | 4 +- compiler/ztests/quoted-type.yaml | 2 +- compiler/ztests/sql/case.yaml | 2 +- compiler/ztests/sql/cte.yaml | 2 +- compiler/ztests/sql/drop.yaml | 6 +- compiler/ztests/sql/embedded-pipe.yaml | 2 +- .../ztests/sql/join-using-table-star.yaml | 6 +- compiler/ztests/sql/like-newline.yaml | 2 +- compiler/ztests/sql/precedence.yaml | 2 +- compiler/ztests/sql/search.yaml | 6 +- compiler/ztests/sql/select-record.yaml | 2 +- compiler/ztests/sql/select-star.yaml | 4 +- compiler/ztests/sql/select.yaml | 4 +- .../ztests/sql/subqueries-select-star.yaml | 2 +- compiler/ztests/sql/where.yaml | 4 +- runtime/exec/environment.go | 2 +- sio/anyio/file.go | 45 ++++++----- ztest/ztest.go | 2 +- 31 files changed, 171 insertions(+), 101 deletions(-) delete mode 100644 cmd/super/ztests/samplesize.yaml create mode 100644 compiler/semantic/ztests/from-type.yaml diff --git a/cli/inputflags/flags.go b/cli/inputflags/flags.go index 07163abd2f..1c229ecc3e 100644 --- a/cli/inputflags/flags.go +++ b/cli/inputflags/flags.go @@ -12,9 +12,9 @@ import ( type Flags struct { Dynamic bool ReaderOpts anyio.ReaderOpts - SampleSize int bsupReadMax auto.Bytes bsupReadSize auto.Bytes + Static bool } func (f *Flags) SetFlags(fs *flag.FlagSet, validate bool) { @@ -36,7 +36,7 @@ func (f *Flags) SetFlags(fs *flag.FlagSet, validate bool) { }) fs.BoolVar(&f.Dynamic, "dynamic", false, "disable static type checking of inputs") fs.StringVar(&opts.Format, "i", "auto", "format of input data [auto,arrows,bsup,csup,csv,json,line,parquet,sup,tsv,zeek]") - fs.IntVar(&f.SampleSize, "samplesize", 1000, "values to read per input file to determine type (<1 for all)") + fs.BoolVar(&f.Static, "static", false, "force static type checking of query inputs") } // Init is called after flags have been parsed. @@ -50,5 +50,8 @@ func (f *Flags) Init() error { if bsup.Size < 0 { return errors.New("target read buffer size must be greater than zero") } + if f.Dynamic && f.Static { + return errors.New("-static and -dynamic flags cannot both be enabled") + } return nil } diff --git a/cmd/super/compile/shared.go b/cmd/super/compile/shared.go index 6e269091b5..6ac744fdea 100644 --- a/cmd/super/compile/shared.go +++ b/cmd/super/compile/shared.go @@ -31,7 +31,7 @@ type Shared struct { parallel int query bool runtime bool - sampleSize int + static bool queryFlags queryflags.QueryTextFlags OutputFlags outputflags.Flags } @@ -42,7 +42,7 @@ func (s *Shared) SetFlags(fs *flag.FlagSet) { fs.BoolVar(&s.optimize, "O", false, "display optimized DAG") fs.IntVar(&s.parallel, "P", 0, "display parallelized DAG") fs.BoolVar(&s.query, "C", false, "display DAG or AST as query text") - fs.IntVar(&s.sampleSize, "samplesize", 1000, "values to read per input file to determine type (<1 for all)") + fs.BoolVar(&s.static, "static", false, "force static type checking on query inputs") s.OutputFlags.SetFlags(fs) s.queryFlags.SetFlags(fs) } @@ -51,6 +51,9 @@ func (s *Shared) Run(ctx context.Context, args []string, dbFlags *dbflags.Flags, if len(s.queryFlags.Query) == 0 && len(args) == 0 { return errors.New("no query specified") } + if s.dynamic && s.static { + return errors.New("-static and -dynamic flags cannot both be enabled") + } var inputs []string if len(args) > 0 { s.queryFlags.Query = append(s.queryFlags.Query, &srcfiles.PlainInput{Text: args[0]}) @@ -87,7 +90,7 @@ func (s *Shared) Run(ctx context.Context, args []string, dbFlags *dbflags.Flags, rctx := runtime.DefaultContext() env := exec.NewEnvironment(storage.NewLocalEngine(), root) env.Dynamic = s.dynamic - env.SampleSize = s.sampleSize + env.Static = s.static dag, err := compiler.Analyze(rctx, ast, env, false) if err != nil { return err diff --git a/cmd/super/root/command.go b/cmd/super/root/command.go index 994a9dccd8..6ee9dc5e53 100644 --- a/cmd/super/root/command.go +++ b/cmd/super/root/command.go @@ -90,7 +90,7 @@ func (c *Command) Run(args []string) error { env.Dynamic = c.inputFlags.Dynamic env.IgnoreOpenErrors = !c.stopErr env.ReaderOpts = c.inputFlags.ReaderOpts - env.SampleSize = c.inputFlags.SampleSize + env.Static = c.inputFlags.Static comp := compiler.NewCompilerWithEnv(env) query, err := runtime.CompileQuery(ctx, super.NewContext(), comp, ast, nil) if err != nil { diff --git a/cmd/super/ztests/samplesize.yaml b/cmd/super/ztests/samplesize.yaml deleted file mode 100644 index f988c58bce..0000000000 --- a/cmd/super/ztests/samplesize.yaml +++ /dev/null @@ -1,38 +0,0 @@ -script: | - echo === -1 >&2 - ! super -samplesize -1 -s -c 'values {a,b,c}' in.sup - echo === 0 >&2 - ! super -samplesize 0 -s -c 'values {a,b,c}' in.sup - echo === 1 >&2 - ! super -samplesize 1 -s -c 'values {a,b,c}' in.sup - echo === 2 >&2 - ! super -samplesize 2 -s -c 'values {a,b,c}' in.sup - -inputs: - - name: in.sup - data: | - {a:1} - {b:2} - -outputs: - - name: stderr - data: | - === -1 - no such field "c" at line 1, column 13: - values {a,b,c} - ~ - === 0 - no such field "c" at line 1, column 13: - values {a,b,c} - ~ - === 1 - no such field "b" at line 1, column 11: - values {a,b,c} - ~ - no such field "c" at line 1, column 13: - values {a,b,c} - ~ - === 2 - no such field "c" at line 1, column 13: - values {a,b,c} - ~ diff --git a/compiler/dag/op.go b/compiler/dag/op.go index bcdc1a45c4..776241cb6e 100644 --- a/compiler/dag/op.go +++ b/compiler/dag/op.go @@ -12,6 +12,7 @@ import ( "reflect" "slices" + "github.com/brimdata/super" "github.com/brimdata/super/pkg/field" "github.com/segmentio/ksuid" ) @@ -262,10 +263,11 @@ type ( Commit ksuid.KSUID `json:"commit"` } FileScan struct { - Kind string `json:"kind" unpack:""` - Paths []string `json:"paths"` - Format string `json:"format"` - Pushdown Pushdown `json:"pushdown"` + Kind string `json:"kind" unpack:""` + Paths []string `json:"paths"` + Type super.Type `json:"-"` + Format string `json:"format"` + Pushdown Pushdown `json:"pushdown"` } ListerScan struct { Kind string `json:"kind" unpack:""` diff --git a/compiler/semantic/dagen.go b/compiler/semantic/dagen.go index 69fdc72bda..82673737d9 100644 --- a/compiler/semantic/dagen.go +++ b/compiler/semantic/dagen.go @@ -85,6 +85,7 @@ func (d *dagen) op(op sem.Op) dag.Op { Kind: "FileScan", Paths: op.Paths, Format: op.Format, + Type: op.Type, } case *sem.HTTPScan: return &dag.HTTPScan{ diff --git a/compiler/semantic/op.go b/compiler/semantic/op.go index c8f3d38089..7bbb8b9b56 100644 --- a/compiler/semantic/op.go +++ b/compiler/semantic/op.go @@ -308,7 +308,11 @@ func (t *translator) fileType(path, format string) (super.Type, error) { } opts := t.env.ReaderOpts opts.Format = format - return anyio.FileType(t.ctx, t.sctx, engine, path, opts, t.env.SampleSize) + typ, err := anyio.FileType(t.ctx, t.sctx, engine, path, opts, t.env.Static) + if typ != nil || err != nil { + return typ, err + } + return t.checker.unknown, nil } func (t *translator) fromFileGlob(globLoc ast.Node, pattern string, args []ast.OpArg) sem.Op { diff --git a/compiler/semantic/ztests/checker-case.yaml b/compiler/semantic/ztests/checker-case.yaml index c697a405a6..d4c43e8e05 100644 --- a/compiler/semantic/ztests/checker-case.yaml +++ b/compiler/semantic/ztests/checker-case.yaml @@ -1,6 +1,6 @@ script: | - super -s -I walk.spq in.sup - ! super -s -I fail.spq in.sup + super -static -s -I walk.spq in.sup + ! super -static -s -I fail.spq in.sup inputs: - name: walk.spq diff --git a/compiler/semantic/ztests/checker-cond-fail.yaml b/compiler/semantic/ztests/checker-cond-fail.yaml index 91c223e837..37e53304a7 100644 --- a/compiler/semantic/ztests/checker-cond-fail.yaml +++ b/compiler/semantic/ztests/checker-cond-fail.yaml @@ -1,5 +1,5 @@ script: | - ! super -s -c "values is_ok(n) ? this+1 : this+2" in.sup + ! super -static -s -c "values is_ok(n) ? this+1 : this+2" in.sup inputs: - name: in.sup diff --git a/compiler/semantic/ztests/from-json.yaml b/compiler/semantic/ztests/from-json.yaml index d75d1fc2dc..2afcc8e227 100644 --- a/compiler/semantic/ztests/from-json.yaml +++ b/compiler/semantic/ztests/from-json.yaml @@ -1,5 +1,5 @@ script: | - super -s -c 'select a.x from a.json a' + super -static -s -c 'select a.x from a.json a' inputs: - name: a.json diff --git a/compiler/semantic/ztests/from-type.yaml b/compiler/semantic/ztests/from-type.yaml new file mode 100644 index 0000000000..6deb815255 --- /dev/null +++ b/compiler/semantic/ztests/from-type.yaml @@ -0,0 +1,76 @@ +script: | + super -o test.csup test.sup + super -f parquet -o test.parquet -c blend test.sup + super -f json -o test.json test.sup + for format in "sup" "json" "csup" "parquet"; do + echo "=== $format (default)" + super compile -dag -C "from test.$format" + echo "=== $format (static)" + super compile -static -dag -C "from test.$format" + echo "=== $format (default /dev/stdin)" + cat "test.$format" | super compile -dag -C "from /dev/stdin" 2>&1 + echo "=== $format (static /dev/stdin)" + cat "test.$format" | super compile -static -dag -C "from /dev/stdin" 2>&1 || true + done + +inputs: + - name: test.sup + data: | + {x:1} + {y:1} + +outputs: + - name: stdout + data: | + === sup (default) + file test.sup format sup + | output main + === sup (static) + file test.sup format sup type {x?:int64,y?:int64} + | output main + === sup (default /dev/stdin) + file /dev/stdin + | output main + === sup (static /dev/stdin) + cannot get file type of non-seekable input at line 1, column 6: + from /dev/stdin + ~~~~~~~~~~ + === json (default) + file test.json format json + | output main + === json (static) + file test.json format json type {x?:int64,y?:int64} + | output main + === json (default /dev/stdin) + file /dev/stdin + | output main + === json (static /dev/stdin) + cannot get file type of non-seekable input at line 1, column 6: + from /dev/stdin + ~~~~~~~~~~ + === csup (default) + file test.csup format csup type {x?:int64,y?:int64} + | output main + === csup (static) + file test.csup format csup type {x?:int64,y?:int64} + | output main + === csup (default /dev/stdin) + file /dev/stdin + | output main + === csup (static /dev/stdin) + cannot get file type of non-seekable input at line 1, column 6: + from /dev/stdin + ~~~~~~~~~~ + === parquet (default) + file test.parquet format parquet type {x:int64|null,y:int64|null} + | output main + === parquet (static) + file test.parquet format parquet type {x:int64|null,y:int64|null} + | output main + === parquet (default /dev/stdin) + file /dev/stdin + | output main + === parquet (static /dev/stdin) + cannot get file type of non-seekable input at line 1, column 6: + from /dev/stdin + ~~~~~~~~~~ diff --git a/compiler/sfmt/dag.go b/compiler/sfmt/dag.go index a18cacc2c7..99b74d4c60 100644 --- a/compiler/sfmt/dag.go +++ b/compiler/sfmt/dag.go @@ -307,6 +307,9 @@ func (c *canonDAG) op(p dag.Op) { if p.Pushdown.Unordered { c.write(" unordered") } + if !isUnknown(p.Type) { + c.write(" type %s", sup.FormatType(p.Type)) + } if len(p.Pushdown.Projection) > 0 { c.fields(p.Pushdown.Projection) } @@ -640,3 +643,12 @@ func (c *canonDAG) sortExprs(sortExprs []dag.SortExpr) { c.write(" %s nulls %s", s.Order, s.Nulls) } } + +func isUnknown(typ super.Type) bool { + if err, ok := super.TypeUnder(typ).(*super.TypeError); ok { + if rec, ok := err.Type.(*super.TypeRecord); ok { + return len(rec.Fields) == 0 + } + } + return false +} diff --git a/compiler/sfmt/ztests/input-files.yaml b/compiler/sfmt/ztests/input-files.yaml index 6cdf6c8736..35a93908bb 100644 --- a/compiler/sfmt/ztests/input-files.yaml +++ b/compiler/sfmt/ztests/input-files.yaml @@ -2,13 +2,13 @@ script: | # Command line input files do not appear in the AST. super compile -C pass /dev/null /dev/zero echo === - super compile -C -dag pass /dev/null /dev/zero + super compile -C -dag pass /dev/null /dev/null outputs: - name: stdout data: | pass === - file /dev/null,/dev/zero + file /dev/null,/dev/null | pass | output main diff --git a/compiler/ztests/pruner.yaml b/compiler/ztests/pruner.yaml index 31608640ec..76d5d94d3b 100644 --- a/compiler/ztests/pruner.yaml +++ b/compiler/ztests/pruner.yaml @@ -10,7 +10,7 @@ outputs: - name: stdout data: | === Test pruner flips comparator with literal/path - file test.parquet format parquet filter (1<=v and 1>=v) + file test.parquet format parquet type {v:int64} filter (1<=v and 1>=v) pruner ( expr compare(v.max, 1, true)>=0 and compare(v.min, 1, true)<=0 fields v.max,v.min @@ -20,7 +20,7 @@ outputs: === Test pruner optimization works with in subquery expressions null | values {x:( - file test.parquet format parquet filter (1<=v and 1>=v) + file test.parquet format parquet type {v:int64} filter (1<=v and 1>=v) pruner ( expr compare(v.max, 1, true)>=0 and compare(v.min, 1, true)<=0 fields v.max,v.min diff --git a/compiler/ztests/quoted-type.yaml b/compiler/ztests/quoted-type.yaml index e51c4cf7f1..b8e77d6c4a 100644 --- a/compiler/ztests/quoted-type.yaml +++ b/compiler/ztests/quoted-type.yaml @@ -1,5 +1,5 @@ script: | - super -s -c 'is(this, <"@foo">)' in.sup + super -static -s -c 'is(this, <"@foo">)' in.sup echo === super -s -c 'type `@foo`={x:int64} const foo = <"@foo"> type `Y Z`={y:"@foo"} const yz = <"Y Z"> is(this, <"Y Z">)' in.sup diff --git a/compiler/ztests/sql/case.yaml b/compiler/ztests/sql/case.yaml index 91276d8b16..9898538467 100644 --- a/compiler/ztests/sql/case.yaml +++ b/compiler/ztests/sql/case.yaml @@ -1,5 +1,5 @@ script: | - super -s -c 'Select * FROM "a.sup" | droP c' + super -static -s -c 'Select * FROM "a.sup" | droP c' inputs: - name: a.sup diff --git a/compiler/ztests/sql/cte.yaml b/compiler/ztests/sql/cte.yaml index edf10ec7e5..0f6cf1b0ec 100644 --- a/compiler/ztests/sql/cte.yaml +++ b/compiler/ztests/sql/cte.yaml @@ -1,5 +1,5 @@ script: | - super -s -I sales.spq + super -static -s -I sales.spq echo // === super -s -c 'with x as ( select 1 as y ) select z.y from x as z' ! super -c 'with x as ( select 1 as y ), x as ( select 2 as y ) select * from x' diff --git a/compiler/ztests/sql/drop.yaml b/compiler/ztests/sql/drop.yaml index b156ceb16a..bf4e850528 100644 --- a/compiler/ztests/sql/drop.yaml +++ b/compiler/ztests/sql/drop.yaml @@ -1,9 +1,9 @@ script: | - super -s -c 'select * from "a.sup" | drop c' + super -static -s -c 'select * from "a.sup" | drop c' echo === - super -dynamic -s -c 'select this from "messy.sup" | values that | drop s,t' + super -s -c 'select this from "messy.sup" | values that | drop s,t' echo === - super -s -c 'select * from "b.sup" | drop b' + super -static -s -c 'select * from "b.sup" | drop b' inputs: - name: a.sup diff --git a/compiler/ztests/sql/embedded-pipe.yaml b/compiler/ztests/sql/embedded-pipe.yaml index a55e8baa58..f8794fdffe 100644 --- a/compiler/ztests/sql/embedded-pipe.yaml +++ b/compiler/ztests/sql/embedded-pipe.yaml @@ -1,5 +1,5 @@ script: | - super -s -I query.sql + super -static -s -I query.sql inputs: - name: query.sql diff --git a/compiler/ztests/sql/join-using-table-star.yaml b/compiler/ztests/sql/join-using-table-star.yaml index 27c24edfb8..d0104ccaa9 100644 --- a/compiler/ztests/sql/join-using-table-star.yaml +++ b/compiler/ztests/sql/join-using-table-star.yaml @@ -1,9 +1,9 @@ script: | - super -s -c "select * from j0.json join j1.json using (a)" + super -static -s -c "select * from j0.json join j1.json using (a)" echo === - super -s -c "select j0.* from j0.json join j1.json using (a)" + super -static -s -c "select j0.* from j0.json join j1.json using (a)" echo === - super -s -c "select j1.* from j0.json join j1.json using (a)" + super -static -s -c "select j1.* from j0.json join j1.json using (a)" inputs: - name: j0.json diff --git a/compiler/ztests/sql/like-newline.yaml b/compiler/ztests/sql/like-newline.yaml index 5981075430..39fa2a8804 100644 --- a/compiler/ztests/sql/like-newline.yaml +++ b/compiler/ztests/sql/like-newline.yaml @@ -1,5 +1,5 @@ script: | - super -s -c "select * from a.json where a like '%bar%'" + super -static -s -c "select * from a.json where a like '%bar%'" inputs: - name: a.json diff --git a/compiler/ztests/sql/precedence.yaml b/compiler/ztests/sql/precedence.yaml index 9ccfff1ec3..f47b57346c 100644 --- a/compiler/ztests/sql/precedence.yaml +++ b/compiler/ztests/sql/precedence.yaml @@ -1,7 +1,7 @@ script: | super -s -I query.sql echo === - super -s -c "SELECT * FROM 'data.csv' WHERE NOT Country = 'Spain';" + super -static -s -c "SELECT * FROM 'data.csv' WHERE NOT Country = 'Spain';" echo === super -s -c "select 'a' || 'b' | count()" echo === diff --git a/compiler/ztests/sql/search.yaml b/compiler/ztests/sql/search.yaml index 171ea21a08..5280a227f9 100644 --- a/compiler/ztests/sql/search.yaml +++ b/compiler/ztests/sql/search.yaml @@ -1,9 +1,9 @@ script: | - super -s -c 'select * from "a.sup" | search 13' + super -static -s -c 'select * from "a.sup" | search 13' echo === - super -s -c 'select * from "messy.sup" | search bar or s==4' + super -static -s -c 'select * from "messy.sup" | search bar or s==4' echo === - super -s -c 'select * from "b.sup" | search len(b) >= 3' + super -static -s -c 'select * from "b.sup" | search len(b) >= 3' inputs: - name: a.sup diff --git a/compiler/ztests/sql/select-record.yaml b/compiler/ztests/sql/select-record.yaml index 1e28fc3706..1310cbc5dc 100644 --- a/compiler/ztests/sql/select-record.yaml +++ b/compiler/ztests/sql/select-record.yaml @@ -1,5 +1,5 @@ script: | - super -s -c 'select {x:T.a,y:c,c} as outer from "a.sup" T' + super -static -s -c 'select {x:T.a,y:c,c} as outer from "a.sup" T' inputs: - name: a.sup diff --git a/compiler/ztests/sql/select-star.yaml b/compiler/ztests/sql/select-star.yaml index b71615f0b1..3a64481177 100644 --- a/compiler/ztests/sql/select-star.yaml +++ b/compiler/ztests/sql/select-star.yaml @@ -1,7 +1,7 @@ script: | - super -s -c 'select * from "a.sup"' + super -static -s -c 'select * from "a.sup"' echo === - super -s -c 'select *,c+a as x from "a.sup"' + super -static -s -c 'select *,c+a as x from "a.sup"' inputs: - name: a.sup diff --git a/compiler/ztests/sql/select.yaml b/compiler/ztests/sql/select.yaml index b9beaec9b1..e13a7ffdf3 100644 --- a/compiler/ztests/sql/select.yaml +++ b/compiler/ztests/sql/select.yaml @@ -1,9 +1,9 @@ script: | super -s -c 'select a from "a.sup"' echo === - super -s -c 'select l.a,r.b from "a.sup" l join "b.sup" r on l.c==r.c' + super -static -s -c 'select l.a,r.b from "a.sup" l join "b.sup" r on l.c==r.c' echo === - super -s -c 'select l.a,m.s from "a.sup" l join "messy.sup" m on l.c==m.s' + super -static -s -c 'select l.a,m.s from "a.sup" l join "messy.sup" m on l.c==m.s' inputs: - name: a.sup diff --git a/compiler/ztests/sql/subqueries-select-star.yaml b/compiler/ztests/sql/subqueries-select-star.yaml index 3d4925e1bb..5364c481cb 100644 --- a/compiler/ztests/sql/subqueries-select-star.yaml +++ b/compiler/ztests/sql/subqueries-select-star.yaml @@ -2,7 +2,7 @@ script: | super -s -c 'select id from ( select * from (values (1)) x(id) )' echo // === # Ensure id can be selected when subquery has select * on a dynamic input. - super -s -c 'select id from ( select * from in.sup )' + super -static -s -c 'select id from ( select * from in.sup )' inputs: - name: in.sup diff --git a/compiler/ztests/sql/where.yaml b/compiler/ztests/sql/where.yaml index 3882693d10..c025e02732 100644 --- a/compiler/ztests/sql/where.yaml +++ b/compiler/ztests/sql/where.yaml @@ -1,9 +1,9 @@ script: | super -s -c 'select a from "a.sup" where a < 13 or c==4' echo === - super -s -c 'select l.a,r.b from "a.sup" l join "b.sup" r on l.c==r.c where len(r.b) >= 3' + super -static -s -c 'select l.a,r.b from "a.sup" l join "b.sup" r on l.c==r.c where len(r.b) >= 3' echo === - super -s -c 'select l.a,m.s from "a.sup" l join "messy.sup" m on l.c==m.s where m.s==4' + super -static -s -c 'select l.a,m.s from "a.sup" l join "messy.sup" m on l.c==m.s where m.s==4' inputs: - name: a.sup diff --git a/runtime/exec/environment.go b/runtime/exec/environment.go index 85e111cb62..b136966e13 100644 --- a/runtime/exec/environment.go +++ b/runtime/exec/environment.go @@ -32,7 +32,7 @@ type Environment struct { Dynamic bool IgnoreOpenErrors bool ReaderOpts anyio.ReaderOpts - SampleSize int + Static bool Stdin vio.Puller } diff --git a/sio/anyio/file.go b/sio/anyio/file.go index 1edffa577c..f2ce216c0f 100644 --- a/sio/anyio/file.go +++ b/sio/anyio/file.go @@ -2,13 +2,14 @@ package anyio import ( "context" + "errors" "io" - "math" "github.com/brimdata/super" "github.com/brimdata/super/pkg/storage" "github.com/brimdata/super/sbuf" "github.com/brimdata/super/sio" + "github.com/brimdata/super/vector" ) // Open uses engine to open path for reading. path is a local file path or a @@ -55,11 +56,8 @@ func NewFile(ctx context.Context, sctx *super.Context, rc io.ReadCloser, path st } // FileType returns a type for the values in the file at path. If the file -// contains values with differing types, FileType returns a fused type. If -// FileType must read values to compute a fused type, it reads at most -// sampleSize values or the entire file if sampleSize is less than 1, and it -// returns a nil type if the file is empty. -func FileType(ctx context.Context, sctx *super.Context, engine storage.Engine, path string, opts ReaderOpts, sampleSize int) (super.Type, error) { +// contains values with differing types, FileType returns a fused type. +func FileType(ctx context.Context, sctx *super.Context, engine storage.Engine, path string, opts ReaderOpts, static bool) (super.Type, error) { u, err := storage.ParseURI(path) if err != nil { return nil, err @@ -69,11 +67,11 @@ func FileType(ctx context.Context, sctx *super.Context, engine storage.Engine, p return nil, err } defer r.Close() - rs, ok := r.(io.ReadSeekCloser) + rs, ok := isReadSeeker(r) if !ok { - return nil, nil - } - if _, err := rs.Seek(0, io.SeekCurrent); err != nil { + if static { + return nil, errors.New("cannot get file type of non-seekable input") + } return nil, nil } f, err := NewFile(ctx, sctx, r, path, opts) @@ -88,18 +86,27 @@ func FileType(ctx context.Context, sctx *super.Context, engine storage.Engine, p if typed, ok := f.Puller.(sio.Typer); ok { return typed.Type() } - if sampleSize < 1 { - sampleSize = math.MaxInt + if !static { + return nil, nil } - // XXX this should pass super true when type checker can handle it - rr := sbuf.PullerReader(sbuf.NewMaterializer(f)) fuser := super.NewFuser(sctx, false) - for range sampleSize { - val, err := rr.Read() - if val == nil || err != nil { + for { + vec, err := f.Pull(false) + if vec == nil || err != nil { return fuser.Type(), err } - fuser.Fuse(val.Type()) + vector.Apply(vector.ApplyNone, func(vecs ...vector.Any) vector.Any { + fuser.Fuse(vecs[0].Type()) + return vecs[0] + }, vec) + } +} + +func isReadSeeker(r io.Reader) (io.ReadSeekCloser, bool) { + rs, ok := r.(io.ReadSeekCloser) + if !ok { + return nil, false } - return fuser.Type(), err + _, err := rs.Seek(0, io.SeekCurrent) + return rs, err == nil } diff --git a/ztest/ztest.go b/ztest/ztest.go index b3b6218bd2..5823c5f843 100644 --- a/ztest/ztest.go +++ b/ztest/ztest.go @@ -483,7 +483,7 @@ func (z *ZTest) runInternal(ctx context.Context) (string, error) { env := exec.NewEnvironment(eng, nil) env.Dynamic = inflags.Dynamic env.ReaderOpts = inflags.ReaderOpts - env.SampleSize = inflags.SampleSize + env.Static = inflags.Static q, err := runtime.CompileQuery(ctx, super.NewContext(), compiler.NewCompilerWithEnv(env), ast, nil) if err != nil { return "", err From 8f3eb07fc224bdf083834f7fd8464544977ef629 Mon Sep 17 00:00:00 2001 From: Matthew Nibecker Date: Wed, 9 Sep 2026 13:19:14 -0700 Subject: [PATCH 2/3] use expr instead of super.Type --- compiler/dag/op.go | 11 +++++------ compiler/semantic/dagen.go | 7 ++++++- compiler/sfmt/dag.go | 14 +++----------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/compiler/dag/op.go b/compiler/dag/op.go index 776241cb6e..56b0e8dffe 100644 --- a/compiler/dag/op.go +++ b/compiler/dag/op.go @@ -12,7 +12,6 @@ import ( "reflect" "slices" - "github.com/brimdata/super" "github.com/brimdata/super/pkg/field" "github.com/segmentio/ksuid" ) @@ -263,11 +262,11 @@ type ( Commit ksuid.KSUID `json:"commit"` } FileScan struct { - Kind string `json:"kind" unpack:""` - Paths []string `json:"paths"` - Type super.Type `json:"-"` - Format string `json:"format"` - Pushdown Pushdown `json:"pushdown"` + Kind string `json:"kind" unpack:""` + Paths []string `json:"paths"` + Type Expr `json:"type"` + Format string `json:"format"` + Pushdown Pushdown `json:"pushdown"` } ListerScan struct { Kind string `json:"kind" unpack:""` diff --git a/compiler/semantic/dagen.go b/compiler/semantic/dagen.go index 82673737d9..fe0a70b1ba 100644 --- a/compiler/semantic/dagen.go +++ b/compiler/semantic/dagen.go @@ -9,6 +9,7 @@ import ( "github.com/brimdata/super/compiler/dag" "github.com/brimdata/super/compiler/semantic/sem" "github.com/brimdata/super/pkg/field" + "github.com/brimdata/super/sup" ) type dagen struct { @@ -81,11 +82,15 @@ func (d *dagen) op(op sem.Op) dag.Op { Commit: op.Commit, } case *sem.FileScan: + var typ dag.Expr + if op.Type != nil && !isUnknown(op.Type) { + typ = &dag.PrimitiveExpr{Kind: "PrimitiveExpr", Value: sup.FormatType(op.Type)} + } return &dag.FileScan{ Kind: "FileScan", Paths: op.Paths, Format: op.Format, - Type: op.Type, + Type: typ, } case *sem.HTTPScan: return &dag.HTTPScan{ diff --git a/compiler/sfmt/dag.go b/compiler/sfmt/dag.go index 99b74d4c60..6b9f9dff8d 100644 --- a/compiler/sfmt/dag.go +++ b/compiler/sfmt/dag.go @@ -307,8 +307,9 @@ func (c *canonDAG) op(p dag.Op) { if p.Pushdown.Unordered { c.write(" unordered") } - if !isUnknown(p.Type) { - c.write(" type %s", sup.FormatType(p.Type)) + if p.Type != nil { + c.write(" type ") + c.expr(p.Type, "") } if len(p.Pushdown.Projection) > 0 { c.fields(p.Pushdown.Projection) @@ -643,12 +644,3 @@ func (c *canonDAG) sortExprs(sortExprs []dag.SortExpr) { c.write(" %s nulls %s", s.Order, s.Nulls) } } - -func isUnknown(typ super.Type) bool { - if err, ok := super.TypeUnder(typ).(*super.TypeError); ok { - if rec, ok := err.Type.(*super.TypeRecord); ok { - return len(rec.Fields) == 0 - } - } - return false -} From 17ee62df081fd66028dc429e36ad5ed43a8a99bc Mon Sep 17 00:00:00 2001 From: Matthew Nibecker Date: Thu, 10 Sep 2026 11:32:28 -0700 Subject: [PATCH 3/3] fixes --- cli/inputflags/flags.go | 4 +- cmd/super/compile/shared.go | 2 +- compiler/dag/op.go | 2 +- compiler/semantic/dagen.go | 6 +-- compiler/semantic/op.go | 6 +-- compiler/semantic/ztests/from-type.yaml | 50 ++++++++++++------------- compiler/sfmt/dag.go | 7 ++-- compiler/sfmt/ztests/input-files.yaml | 4 +- sio/anyio/file.go | 6 +-- 9 files changed, 43 insertions(+), 44 deletions(-) diff --git a/cli/inputflags/flags.go b/cli/inputflags/flags.go index 1c229ecc3e..a0188c5dc7 100644 --- a/cli/inputflags/flags.go +++ b/cli/inputflags/flags.go @@ -12,9 +12,9 @@ import ( type Flags struct { Dynamic bool ReaderOpts anyio.ReaderOpts + Static bool bsupReadMax auto.Bytes bsupReadSize auto.Bytes - Static bool } func (f *Flags) SetFlags(fs *flag.FlagSet, validate bool) { @@ -36,7 +36,7 @@ func (f *Flags) SetFlags(fs *flag.FlagSet, validate bool) { }) fs.BoolVar(&f.Dynamic, "dynamic", false, "disable static type checking of inputs") fs.StringVar(&opts.Format, "i", "auto", "format of input data [auto,arrows,bsup,csup,csv,json,line,parquet,sup,tsv,zeek]") - fs.BoolVar(&f.Static, "static", false, "force static type checking of query inputs") + fs.BoolVar(&f.Static, "static", false, "force static type checking of inputs") } // Init is called after flags have been parsed. diff --git a/cmd/super/compile/shared.go b/cmd/super/compile/shared.go index 6ac744fdea..b5516a0d5d 100644 --- a/cmd/super/compile/shared.go +++ b/cmd/super/compile/shared.go @@ -42,7 +42,7 @@ func (s *Shared) SetFlags(fs *flag.FlagSet) { fs.BoolVar(&s.optimize, "O", false, "display optimized DAG") fs.IntVar(&s.parallel, "P", 0, "display parallelized DAG") fs.BoolVar(&s.query, "C", false, "display DAG or AST as query text") - fs.BoolVar(&s.static, "static", false, "force static type checking on query inputs") + fs.BoolVar(&s.static, "static", false, "force static type checking of inputs on DAG") s.OutputFlags.SetFlags(fs) s.queryFlags.SetFlags(fs) } diff --git a/compiler/dag/op.go b/compiler/dag/op.go index 56b0e8dffe..278d007e0e 100644 --- a/compiler/dag/op.go +++ b/compiler/dag/op.go @@ -264,9 +264,9 @@ type ( FileScan struct { Kind string `json:"kind" unpack:""` Paths []string `json:"paths"` - Type Expr `json:"type"` Format string `json:"format"` Pushdown Pushdown `json:"pushdown"` + Type string `json:"type"` } ListerScan struct { Kind string `json:"kind" unpack:""` diff --git a/compiler/semantic/dagen.go b/compiler/semantic/dagen.go index fe0a70b1ba..403b15e9c4 100644 --- a/compiler/semantic/dagen.go +++ b/compiler/semantic/dagen.go @@ -82,9 +82,9 @@ func (d *dagen) op(op sem.Op) dag.Op { Commit: op.Commit, } case *sem.FileScan: - var typ dag.Expr - if op.Type != nil && !isUnknown(op.Type) { - typ = &dag.PrimitiveExpr{Kind: "PrimitiveExpr", Value: sup.FormatType(op.Type)} + var typ string + if !isUnknown(op.Type) { + typ = sup.FormatType(op.Type) } return &dag.FileScan{ Kind: "FileScan", diff --git a/compiler/semantic/op.go b/compiler/semantic/op.go index 7bbb8b9b56..0eb2dddfbe 100644 --- a/compiler/semantic/op.go +++ b/compiler/semantic/op.go @@ -309,10 +309,10 @@ func (t *translator) fileType(path, format string) (super.Type, error) { opts := t.env.ReaderOpts opts.Format = format typ, err := anyio.FileType(t.ctx, t.sctx, engine, path, opts, t.env.Static) - if typ != nil || err != nil { - return typ, err + if typ == nil { + typ = t.checker.unknown } - return t.checker.unknown, nil + return typ, err } func (t *translator) fromFileGlob(globLoc ast.Node, pattern string, args []ast.OpArg) sem.Op { diff --git a/compiler/semantic/ztests/from-type.yaml b/compiler/semantic/ztests/from-type.yaml index 6deb815255..41dcbe653a 100644 --- a/compiler/semantic/ztests/from-type.yaml +++ b/compiler/semantic/ztests/from-type.yaml @@ -2,15 +2,15 @@ script: | super -o test.csup test.sup super -f parquet -o test.parquet -c blend test.sup super -f json -o test.json test.sup - for format in "sup" "json" "csup" "parquet"; do + for format in csup json parquet sup; do echo "=== $format (default)" super compile -dag -C "from test.$format" echo "=== $format (static)" super compile -static -dag -C "from test.$format" - echo "=== $format (default /dev/stdin)" - cat "test.$format" | super compile -dag -C "from /dev/stdin" 2>&1 - echo "=== $format (static /dev/stdin)" - cat "test.$format" | super compile -static -dag -C "from /dev/stdin" 2>&1 || true + echo "=== $format (default pipe)" + cat "test.$format" | super compile -dag -C "from /dev/stdin" + echo "=== $format (static pipe)" + ! cat "test.$format" | super compile -static -dag -C "from /dev/stdin" 2>&1 done inputs: @@ -22,16 +22,16 @@ inputs: outputs: - name: stdout data: | - === sup (default) - file test.sup format sup + === csup (default) + file test.csup format csup type {x?:int64,y?:int64} | output main - === sup (static) - file test.sup format sup type {x?:int64,y?:int64} + === csup (static) + file test.csup format csup type {x?:int64,y?:int64} | output main - === sup (default /dev/stdin) + === csup (default pipe) file /dev/stdin | output main - === sup (static /dev/stdin) + === csup (static pipe) cannot get file type of non-seekable input at line 1, column 6: from /dev/stdin ~~~~~~~~~~ @@ -41,36 +41,36 @@ outputs: === json (static) file test.json format json type {x?:int64,y?:int64} | output main - === json (default /dev/stdin) + === json (default pipe) file /dev/stdin | output main - === json (static /dev/stdin) + === json (static pipe) cannot get file type of non-seekable input at line 1, column 6: from /dev/stdin ~~~~~~~~~~ - === csup (default) - file test.csup format csup type {x?:int64,y?:int64} + === parquet (default) + file test.parquet format parquet type {x:int64|null,y:int64|null} | output main - === csup (static) - file test.csup format csup type {x?:int64,y?:int64} + === parquet (static) + file test.parquet format parquet type {x:int64|null,y:int64|null} | output main - === csup (default /dev/stdin) + === parquet (default pipe) file /dev/stdin | output main - === csup (static /dev/stdin) + === parquet (static pipe) cannot get file type of non-seekable input at line 1, column 6: from /dev/stdin ~~~~~~~~~~ - === parquet (default) - file test.parquet format parquet type {x:int64|null,y:int64|null} + === sup (default) + file test.sup format sup | output main - === parquet (static) - file test.parquet format parquet type {x:int64|null,y:int64|null} + === sup (static) + file test.sup format sup type {x?:int64,y?:int64} | output main - === parquet (default /dev/stdin) + === sup (default pipe) file /dev/stdin | output main - === parquet (static /dev/stdin) + === sup (static pipe) cannot get file type of non-seekable input at line 1, column 6: from /dev/stdin ~~~~~~~~~~ diff --git a/compiler/sfmt/dag.go b/compiler/sfmt/dag.go index 6b9f9dff8d..5af4bd7a54 100644 --- a/compiler/sfmt/dag.go +++ b/compiler/sfmt/dag.go @@ -304,13 +304,12 @@ func (c *canonDAG) op(p dag.Op) { if p.Format != "" { c.write(" format %s", p.Format) } + if p.Type != "" { + c.write(" type %s", p.Type) + } if p.Pushdown.Unordered { c.write(" unordered") } - if p.Type != nil { - c.write(" type ") - c.expr(p.Type, "") - } if len(p.Pushdown.Projection) > 0 { c.fields(p.Pushdown.Projection) } diff --git a/compiler/sfmt/ztests/input-files.yaml b/compiler/sfmt/ztests/input-files.yaml index 35a93908bb..6bd2b6f520 100644 --- a/compiler/sfmt/ztests/input-files.yaml +++ b/compiler/sfmt/ztests/input-files.yaml @@ -2,13 +2,13 @@ script: | # Command line input files do not appear in the AST. super compile -C pass /dev/null /dev/zero echo === - super compile -C -dag pass /dev/null /dev/null + super compile -C -dynamic -dag pass /dev/null /dev/zero outputs: - name: stdout data: | pass === - file /dev/null,/dev/null + file /dev/null,/dev/zero | pass | output main diff --git a/sio/anyio/file.go b/sio/anyio/file.go index f2ce216c0f..74c7391ec3 100644 --- a/sio/anyio/file.go +++ b/sio/anyio/file.go @@ -67,7 +67,7 @@ func FileType(ctx context.Context, sctx *super.Context, engine storage.Engine, p return nil, err } defer r.Close() - rs, ok := isReadSeeker(r) + rs, ok := asReadSeeker(r) if !ok { if static { return nil, errors.New("cannot get file type of non-seekable input") @@ -102,8 +102,8 @@ func FileType(ctx context.Context, sctx *super.Context, engine storage.Engine, p } } -func isReadSeeker(r io.Reader) (io.ReadSeekCloser, bool) { - rs, ok := r.(io.ReadSeekCloser) +func asReadSeeker(r io.Reader) (io.ReadSeeker, bool) { + rs, ok := r.(io.ReadSeeker) if !ok { return nil, false }