Add -static flag

cli/inputflags/flags.go

@@ -12,7 +12,7 @@ import (
 type Flags struct {
 	Dynamic      bool
 	ReaderOpts   anyio.ReaderOpts
-	SampleSize   int
+	Static       bool
 	bsupReadMax  auto.Bytes
 	bsupReadSize auto.Bytes
 }
@@ -36,7 +36,7 @@ func (f *Flags) SetFlags(fs *flag.FlagSet, validate bool) {
 	})
 	fs.BoolVar(&f.Dynamic, "dynamic", false, "disable static type checking of inputs")
 	fs.StringVar(&opts.Format, "i", "auto", "format of input data [auto,arrows,bsup,csup,csv,json,line,parquet,sup,tsv,zeek]")
-	fs.IntVar(&f.SampleSize, "samplesize", 1000, "values to read per input file to determine type (<1 for all)")
+	fs.BoolVar(&f.Static, "static", false, "force static type checking of inputs")
 }
 
 // Init is called after flags have been parsed.
@@ -50,5 +50,8 @@ func (f *Flags) Init() error {
 	if bsup.Size < 0 {
 		return errors.New("target read buffer size must be greater than zero")
 	}
+	if f.Dynamic && f.Static {
+		return errors.New("-static and -dynamic flags cannot both be enabled")
+	}
 	return nil
 }

cmd/super/compile/shared.go

@@ -31,7 +31,7 @@ type Shared struct {
 	parallel    int
 	query       bool
 	runtime     bool
-	sampleSize  int
+	static      bool
 	queryFlags  queryflags.QueryTextFlags
 	OutputFlags outputflags.Flags
 }
@@ -42,7 +42,7 @@ func (s *Shared) SetFlags(fs *flag.FlagSet) {
 	fs.BoolVar(&s.optimize, "O", false, "display optimized DAG")
 	fs.IntVar(&s.parallel, "P", 0, "display parallelized DAG")
 	fs.BoolVar(&s.query, "C", false, "display DAG or AST as query text")
-	fs.IntVar(&s.sampleSize, "samplesize", 1000, "values to read per input file to determine type (<1 for all)")
+	fs.BoolVar(&s.static, "static", false, "force static type checking of inputs on DAG")
 	s.OutputFlags.SetFlags(fs)
 	s.queryFlags.SetFlags(fs)
 }
@@ -51,6 +51,9 @@ func (s *Shared) Run(ctx context.Context, args []string, dbFlags *dbflags.Flags,
 	if len(s.queryFlags.Query) == 0 && len(args) == 0 {
 		return errors.New("no query specified")
 	}
+	if s.dynamic && s.static {
+		return errors.New("-static and -dynamic flags cannot both be enabled")
+	}
 	var inputs []string
 	if len(args) > 0 {
 		s.queryFlags.Query = append(s.queryFlags.Query, &srcfiles.PlainInput{Text: args[0]})
@@ -87,7 +90,7 @@ func (s *Shared) Run(ctx context.Context, args []string, dbFlags *dbflags.Flags,
 	rctx := runtime.DefaultContext()
 	env := exec.NewEnvironment(storage.NewLocalEngine(), root)
 	env.Dynamic = s.dynamic
-	env.SampleSize = s.sampleSize
+	env.Static = s.static
 	dag, err := compiler.Analyze(rctx, ast, env, false)
 	if err != nil {
 		return err

cmd/super/root/command.go

@@ -90,7 +90,7 @@ func (c *Command) Run(args []string) error {
 	env.Dynamic = c.inputFlags.Dynamic
 	env.IgnoreOpenErrors = !c.stopErr
 	env.ReaderOpts = c.inputFlags.ReaderOpts
-	env.SampleSize = c.inputFlags.SampleSize
+	env.Static = c.inputFlags.Static
 	comp := compiler.NewCompilerWithEnv(env)
 	query, err := runtime.CompileQuery(ctx, super.NewContext(), comp, ast, nil)
 	if err != nil {

compiler/dag/op.go

@@ -266,6 +266,7 @@ type (
 		Paths    []string `json:"paths"`
 		Format   string   `json:"format"`
 		Pushdown Pushdown `json:"pushdown"`
+		Type     string   `json:"type"`
 	}
 	ListerScan struct {
 		Kind      string      `json:"kind" unpack:""`

compiler/semantic/dagen.go

@@ -9,6 +9,7 @@ import (
 	"github.com/brimdata/super/compiler/dag"
 	"github.com/brimdata/super/compiler/semantic/sem"
 	"github.com/brimdata/super/pkg/field"
+	"github.com/brimdata/super/sup"
 )
 
 type dagen struct {
@@ -81,10 +82,15 @@ func (d *dagen) op(op sem.Op) dag.Op {
 			Commit: op.Commit,
 		}
 	case *sem.FileScan:
+		var typ string
+		if !isUnknown(op.Type) {
+			typ = sup.FormatType(op.Type)
+		}
 		return &dag.FileScan{
 			Kind:   "FileScan",
 			Paths:  op.Paths,
 			Format: op.Format,
+			Type:   typ,
 		}
 	case *sem.HTTPScan:
 		return &dag.HTTPScan{

compiler/semantic/op.go

@@ -308,7 +308,11 @@ func (t *translator) fileType(path, format string) (super.Type, error) {
 	}
 	opts := t.env.ReaderOpts
 	opts.Format = format
-	return anyio.FileType(t.ctx, t.sctx, engine, path, opts, t.env.SampleSize)
+	typ, err := anyio.FileType(t.ctx, t.sctx, engine, path, opts, t.env.Static)
+	if typ == nil {
+		typ = t.checker.unknown
+	}
+	return typ, err
 }
 
 func (t *translator) fromFileGlob(globLoc ast.Node, pattern string, args []ast.OpArg) sem.Op {

compiler/semantic/ztests/checker-case.yaml

@@ -1,6 +1,6 @@
 script: |
-  super -s -I walk.spq in.sup
-  ! super -s -I fail.spq in.sup
+  super -static -s -I walk.spq in.sup
+  ! super -static -s -I fail.spq in.sup
 
 inputs:
   - name: walk.spq

compiler/semantic/ztests/checker-cond-fail.yaml

@@ -1,5 +1,5 @@
 script: |
-  ! super -s -c "values is_ok(n) ? this+1 : this+2" in.sup
+  ! super -static -s -c "values is_ok(n) ? this+1 : this+2" in.sup
 
 inputs:
   - name: in.sup

compiler/semantic/ztests/from-json.yaml

@@ -1,5 +1,5 @@
 script: |
-  super -s -c 'select a.x from a.json a'
+  super -static -s -c 'select a.x from a.json a'
 
 inputs:
   - name: a.json

compiler/semantic/ztests/from-type.yaml

@@ -0,0 +1,76 @@
+script: |
+  super -o test.csup test.sup
+  super -f parquet -o test.parquet -c blend test.sup
+  super -f json -o test.json test.sup
+  for format in csup json parquet sup; do
+    echo "=== $format (default)"
+    super compile -dag -C "from test.$format"
+    echo "=== $format (static)"
+    super compile -static -dag -C "from test.$format"
+    echo "=== $format (default pipe)"
+    cat "test.$format" | super compile -dag -C "from /dev/stdin"
+    echo "=== $format (static pipe)"
+    ! cat "test.$format" | super compile -static -dag -C "from /dev/stdin" 2>&1
+  done
+
+inputs:
+  - name: test.sup
+    data: |
+      {x:1}
+      {y:1}
+
+outputs:
+  - name: stdout
+    data: |
+      === csup (default)
+      file test.csup format csup type {x?:int64,y?:int64}
+      | output main
+      === csup (static)
+      file test.csup format csup type {x?:int64,y?:int64}
+      | output main
+      === csup (default pipe)
+      file /dev/stdin
+      | output main
+      === csup (static pipe)
+      cannot get file type of non-seekable input at line 1, column 6:
+      from /dev/stdin
+           ~~~~~~~~~~
+      === json (default)
+      file test.json format json
+      | output main
+      === json (static)
+      file test.json format json type {x?:int64,y?:int64}
+      | output main
+      === json (default pipe)
+      file /dev/stdin
+      | output main
+      === json (static pipe)
+      cannot get file type of non-seekable input at line 1, column 6:
+      from /dev/stdin
+           ~~~~~~~~~~
+      === parquet (default)
+      file test.parquet format parquet type {x:int64|null,y:int64|null}
+      | output main
+      === parquet (static)
+      file test.parquet format parquet type {x:int64|null,y:int64|null}
+      | output main
+      === parquet (default pipe)
+      file /dev/stdin
+      | output main
+      === parquet (static pipe)
+      cannot get file type of non-seekable input at line 1, column 6:
+      from /dev/stdin
+           ~~~~~~~~~~
+      === sup (default)
+      file test.sup format sup
+      | output main
+      === sup (static)
+      file test.sup format sup type {x?:int64,y?:int64}
+      | output main
+      === sup (default pipe)
+      file /dev/stdin
+      | output main
+      === sup (static pipe)
+      cannot get file type of non-seekable input at line 1, column 6:
+      from /dev/stdin
+           ~~~~~~~~~~

compiler/sfmt/dag.go

@@ -304,6 +304,9 @@ func (c *canonDAG) op(p dag.Op) {
 		if p.Format != "" {
 			c.write(" format %s", p.Format)
 		}
+		if p.Type != "" {
+			c.write(" type %s", p.Type)
+		}
 		if p.Pushdown.Unordered {
 			c.write(" unordered")
 		}

compiler/sfmt/ztests/input-files.yaml

@@ -2,7 +2,7 @@ script: |
   # Command line input files do not appear in the AST.
   super compile -C pass /dev/null /dev/zero
   echo ===
-  super compile -C -dag pass /dev/null /dev/zero
+  super compile -C -dynamic -dag pass /dev/null /dev/zero
 
 outputs:
   - name: stdout

compiler/ztests/pruner.yaml

@@ -10,7 +10,7 @@ outputs:
   - name: stdout
     data: |
       === Test pruner flips comparator with literal/path
-      file test.parquet format parquet filter (1<=v and 1>=v)
+      file test.parquet format parquet type {v:int64} filter (1<=v and 1>=v)
          pruner (
            expr compare(v.max, 1, true)>=0 and compare(v.min, 1, true)<=0
            fields v.max,v.min
@@ -20,7 +20,7 @@ outputs:
       === Test pruner optimization works with in subquery expressions
       null
       | values {x:(
-        file test.parquet format parquet filter (1<=v and 1>=v)
+        file test.parquet format parquet type {v:int64} filter (1<=v and 1>=v)
            pruner (
              expr compare(v.max, 1, true)>=0 and compare(v.min, 1, true)<=0
              fields v.max,v.min

compiler/ztests/quoted-type.yaml

@@ -1,5 +1,5 @@
 script: |
-  super -s -c 'is(this, <"@foo">)' in.sup
+  super -static -s -c 'is(this, <"@foo">)' in.sup
   echo ===
   super -s -c 'type `@foo`={x:int64} const foo = <"@foo"> type `Y Z`={y:"@foo"} const yz = <"Y Z"> is(this, <"Y Z">)' in.sup
 

compiler/ztests/sql/case.yaml

@@ -1,5 +1,5 @@
 script: |
-  super -s -c 'Select * FROM "a.sup" | droP c'
+  super -static -s -c 'Select * FROM "a.sup" | droP c'
 
 inputs:
   - name: a.sup

compiler/ztests/sql/cte.yaml

@@ -1,5 +1,5 @@
 script: |
-  super -s -I sales.spq
+  super -static -s -I sales.spq
   echo // ===
   super -s -c 'with x as ( select 1 as y ) select z.y from x as z'
   ! super -c 'with x as ( select 1 as y ), x as ( select 2 as y ) select * from x'

compiler/ztests/sql/drop.yaml

@@ -1,9 +1,9 @@
 script: |
-  super -s -c 'select * from "a.sup" | drop c'
+  super -static -s -c 'select * from "a.sup" | drop c'
   echo ===
-  super -dynamic -s -c 'select this from "messy.sup" | values that | drop s,t'
+  super -s -c 'select this from "messy.sup" | values that | drop s,t'
   echo ===
-  super -s -c 'select * from "b.sup" | drop b'
+  super -static -s -c 'select * from "b.sup" | drop b'
 
 inputs:
   - name: a.sup

compiler/ztests/sql/embedded-pipe.yaml

@@ -1,5 +1,5 @@
 script: |
-  super -s -I query.sql
+  super -static -s -I query.sql
 
 inputs:
   - name: query.sql

compiler/ztests/sql/join-using-table-star.yaml

@@ -1,9 +1,9 @@
 script: |
-  super -s -c "select * from j0.json join j1.json using (a)"
+  super -static -s -c "select * from j0.json join j1.json using (a)"
   echo ===
-  super -s -c "select j0.* from j0.json join j1.json using (a)"
+  super -static -s -c "select j0.* from j0.json join j1.json using (a)"
   echo ===
-  super -s -c "select j1.* from j0.json join j1.json using (a)"
+  super -static -s -c "select j1.* from j0.json join j1.json using (a)"
 
 inputs:
   - name: j0.json

compiler/ztests/sql/like-newline.yaml

@@ -1,5 +1,5 @@
 script: |
-  super -s -c "select * from a.json where a like '%bar%'"
+  super -static -s -c "select * from a.json where a like '%bar%'"
 
 inputs:
   - name: a.json

compiler/ztests/sql/precedence.yaml

@@ -1,7 +1,7 @@
 script: |
   super -s -I query.sql
   echo ===
-  super -s -c "SELECT * FROM 'data.csv' WHERE NOT Country = 'Spain';"
+  super -static -s -c "SELECT * FROM 'data.csv' WHERE NOT Country = 'Spain';"
   echo ===
   super -s -c "select 'a' || 'b' | count()"
   echo ===

compiler/ztests/sql/search.yaml

@@ -1,9 +1,9 @@
 script: |
-  super -s -c 'select * from "a.sup" | search 13'
+  super -static -s -c 'select * from "a.sup" | search 13'
   echo ===
-  super -s -c 'select * from "messy.sup" | search bar or s==4'
+  super -static -s -c 'select * from "messy.sup" | search bar or s==4'
   echo ===
-  super -s -c 'select * from "b.sup" | search len(b) >= 3'
+  super -static -s -c 'select * from "b.sup" | search len(b) >= 3'
 
 inputs:
   - name: a.sup

compiler/ztests/sql/select-record.yaml

@@ -1,5 +1,5 @@
 script: |
-  super -s -c 'select {x:T.a,y:c,c} as outer from "a.sup" T'
+  super -static -s -c 'select {x:T.a,y:c,c} as outer from "a.sup" T'
 
 inputs:
   - name: a.sup

compiler/ztests/sql/select-star.yaml

@@ -1,7 +1,7 @@
 script: |
-  super -s -c 'select * from "a.sup"'
+  super -static -s -c 'select * from "a.sup"'
   echo ===
-  super -s -c 'select *,c+a as x from "a.sup"'
+  super -static -s -c 'select *,c+a as x from "a.sup"'
 
 inputs:
   - name: a.sup