diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 366b6b8c98..9fa9060fa6 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -105,14 +105,14 @@ - [error](super-sql/functions/errors/error.md) - [has_error](super-sql/functions/errors/has_error.md) - [is_error](super-sql/functions/errors/is_error.md) - - [missing](super-sql/functions/errors/missing.md) - [Generics](super-sql/functions/generics/intro.md) - [coalesce](super-sql/functions/generics/coalesce.md) - [compare](super-sql/functions/generics/compare.md) - - [has](super-sql/functions/generics/has.md) + - [is_ok](super-sql/functions/generics/is_ok.md) - [len](super-sql/functions/generics/len.md) - [map](super-sql/functions/generics/map.md) - [nullif](super-sql/functions/generics/nullif.md) + - [ok](super-sql/functions/generics/ok.md) - [under](super-sql/functions/generics/under.md) - [Math](super-sql/functions/math/intro.md) - [abs](super-sql/functions/math/abs.md) diff --git a/book/src/command/super.md b/book/src/command/super.md index 02a0270192..69af8eaf93 100644 --- a/book/src/command/super.md +++ b/book/src/command/super.md @@ -101,12 +101,12 @@ This can be especially handy when you are learning the language and its For example, this query ```mdtest-command -super -C -c 'has(foo)' +super -C -c 'is_ok(foo)' ``` is an implied [where](../super-sql/operators/where.md) operator, which matches values that have a field `foo`, i.e., ```mdtest-output -where has(foo) +where is_ok(foo) ``` while this query ```mdtest-command diff --git a/book/src/super-sql/expressions/comparisons.md b/book/src/super-sql/expressions/comparisons.md index e4dc17172a..c744f2fe6c 100644 --- a/book/src/super-sql/expressions/comparisons.md +++ b/book/src/super-sql/expressions/comparisons.md @@ -82,7 +82,7 @@ _Various scalar comparisons_ ```mdtest-spq # spq -values 1 > 2, 1 < 2, "b" > "a", 1 > "a", 1 > error("missing") +values 1 > 2, 1 < 2, "b" > "a", 1 > "a" # input # expected output @@ -90,7 +90,6 @@ false true true false -error("missing") ``` --- @@ -104,12 +103,12 @@ values {isNull:this is null,isNotNull:this is not null} 1 null 2 -error("missing") +none # expected output {isNull:false,isNotNull:true} {isNull:true,isNotNull:false} {isNull:false,isNotNull:true} -{isNull:error("missing"),isNotNull:error("missing")} +{isNull:false,isNotNull:true} ``` --- diff --git a/book/src/super-sql/functions/errors/error.md b/book/src/super-sql/functions/errors/error.md index 88ec5298b7..e6250048a6 100644 --- a/book/src/super-sql/functions/errors/error.md +++ b/book/src/super-sql/functions/errors/error.md @@ -60,18 +60,3 @@ error("exception") {that:error("exception"),err:true,kind:"error"} {that:"exception",err:false,kind:"primitive"} ``` - ---- - -_Comparison of a missing error results in a missing error even if they -are the same missing errors so as to not allow field comparisons of two -missing fields to succeed_ - -```mdtest-spq -# spq -badfield:=x | values badfield==error("missing") -# input -{} -# expected output -error("missing") -``` diff --git a/book/src/super-sql/functions/errors/missing.md b/book/src/super-sql/functions/errors/missing.md deleted file mode 100644 index 70367dd209..0000000000 --- a/book/src/super-sql/functions/errors/missing.md +++ /dev/null @@ -1,84 +0,0 @@ -# missing - -test for the "missing" error - -## Synopsis - -``` -missing(val: any) -> bool -``` - -## Description - -The `missing` function returns true if its argument is `error("missing")` -and false otherwise. - -This function is often used to test if certain fields do not appear as -expected in a record, e.g., `missing(a)` is true either when `this` is not a record -or when `this` is a record and the field `a` is not present in `this`. - -It's also useful for shaping messy data when applying conditional logic based on the -absence of certain fields: -``` -switch - case missing(a) ( ... ) - case missing(b) ( ... ) - default ( ... ) -``` - -## Examples - ---- - -```mdtest-spq -# spq -values {yes:missing(bar),no:missing(foo)} -# input -{foo:10} -# expected output -{yes:true,no:false} -``` - ---- - -```mdtest-spq -# spq -values {yes:missing(foo.baz),no:missing(foo.bar)} -# input -{foo:{bar:"value"}} -# expected output -{yes:true,no:false} -``` - ---- - -```mdtest-spq -# spq -values {yes:missing(bar+1),no:missing(foo+1)} -# input -{foo:10} -# expected output -{yes:true,no:false} -``` - ---- - -```mdtest-spq -# spq -values missing(bar) -# input -1 -# expected output -true -``` - ---- - -```mdtest-spq -# spq -values missing(x) -# input -{x:error("missing")} -# expected output -true -``` diff --git a/book/src/super-sql/functions/generics/has.md b/book/src/super-sql/functions/generics/has.md deleted file mode 100644 index 129505c903..0000000000 --- a/book/src/super-sql/functions/generics/has.md +++ /dev/null @@ -1,96 +0,0 @@ -# has - -test existence of values - -## Synopsis - -``` -has(val: any [, ... val: any]) -> bool -``` - -## Description - -The `has` function returns false if any of its arguments are `error("missing")` -and otherwise returns true. -`has(e)` is a shortcut for [`!missing(e)`](../errors/missing.md). - -This function is often used to determine the existence of fields in a -record, e.g., `has(a,b)` is true when `this` is a record and has -the fields `a` and `b`, provided their values are not `error("missing")`. - -It's also useful in shaping messy data when applying conditional logic based on the -presence of certain fields: -``` -switch - case has(a) ( ... ) - case has(b) ( ... ) - default ( ... ) -``` - -## Examples - ---- - -```mdtest-spq -# spq -values {yes:has(foo),no:has(bar)} -# input -{foo:10} -# expected output -{yes:true,no:false} -``` - ---- - -```mdtest-spq -# spq -values {yes: has(foo[1]),no:has(foo[4])} -# input -{foo:[1,2,3]} -# expected output -{yes:true,no:false} -``` - ---- - -```mdtest-spq -# spq -values {yes:has(foo.bar),no:has(foo.baz)} -# input -{foo:{bar:"value"}} -# expected output -{yes:true,no:false} -``` - ---- - -```mdtest-spq -# spq -values {yes:has(foo+1),no:has(bar+1)} -# input -{foo:10} -# expected output -{yes:true,no:false} -``` - ---- - -```mdtest-spq -# spq -values has(bar) -# input -1 -# expected output -false -``` - ---- - -```mdtest-spq -# spq -values has(x) -# input -{x:error("missing")} -# expected output -false -``` diff --git a/book/src/super-sql/functions/generics/is_ok.md b/book/src/super-sql/functions/generics/is_ok.md new file mode 100644 index 0000000000..84bc8d32c0 --- /dev/null +++ b/book/src/super-sql/functions/generics/is_ok.md @@ -0,0 +1,85 @@ +# is_ok + +test the validity of values + +## Synopsis + +``` +is_ok(val: any) -> bool +``` + +## Description + +The `is_ok` function is inspired by the Rust language pattern of the same name +and returns true when `val` is not `none` or an error. +It is a shortcut for `ok(val)!=none`. + +This function is often used to determine the existence of fields in a +record, e.g., `is_ok(x)` is true when `this` is a record and has +the field `x`, provided its value is not an error or none. + +It's also useful in shaping messy data when applying conditional logic based on the +presence of certain fields: +``` +switch + case a.is_ok() ( ... ) + case b.is_ok() ( ... ) + default ( ... ) +``` + +## Examples + +--- + +```mdtest-spq +# spq +values {yes:is_ok(foo),no:is_ok(bar)} +# input +{foo:10} +# expected output +{yes:true,no:false} +``` + +--- + +```mdtest-spq +# spq +values {yes: foo[1].is_ok(),no:foo[4].is_ok()} +# input +{foo:[1,2,3]} +# expected output +{yes:true,no:false} +``` + +--- + +```mdtest-spq +# spq +values {yes:foo.bar.is_ok(),no:foo.baz.is_ok()} +# input +{foo:{bar:"value"}} +# expected output +{yes:true,no:false} +``` + +--- + +```mdtest-spq +# spq +values {yes:is_ok(foo+1),no:is_ok(bar+1)} +# input +{foo:10} +# expected output +{yes:true,no:false} +``` + +--- + +```mdtest-spq +# spq +values bar.is_ok() +# input +1 +# expected output +false +``` diff --git a/book/src/super-sql/functions/generics/ok.md b/book/src/super-sql/functions/generics/ok.md new file mode 100644 index 0000000000..d85ae9ea8e --- /dev/null +++ b/book/src/super-sql/functions/generics/ok.md @@ -0,0 +1,49 @@ +# ok + +convert errors to none + +## Synopsis + +``` +ok(val: any) -> any +``` + +## Description + +The `ok` function is inspired by the Rust language pattern of the same name +and returns `none` when `val` is `none` or an error and otherwise returns `val` +unmodified. + +This function is useful when errors are expected and are to be explicitly managed +with `none` operators. + +## Examples + +--- + +```mdtest-spq +# spq +values ok(10/this) ?? "bad value" +# input +2 +0 +# expected output +5 +"bad value" +``` + +--- + +```mdtest-spq +# spq +count() by key.ok() ?? "MISSING" | sort this +# input +{key:1} +{key:1} +{key:2} +{} +# expected output +{key:1,count:2} +{key:2,count:1} +{key:"MISSING",count:1} +``` diff --git a/book/src/super-sql/functions/strings/grep.md b/book/src/super-sql/functions/strings/grep.md index 9d2980aed0..22b13fb5df 100644 --- a/book/src/super-sql/functions/strings/grep.md +++ b/book/src/super-sql/functions/strings/grep.md @@ -90,7 +90,7 @@ _Regular expression with a non-this argument_ ```mdtest-spq # spq -grep('b.*', s) +grep('b.*', s.ok()) # input {s:"bar"} {s:"foo"} diff --git a/book/src/super-sql/functions/types/cast.md b/book/src/super-sql/functions/types/cast.md index aaf0f26472..9e1f1ebc33 100644 --- a/book/src/super-sql/functions/types/cast.md +++ b/book/src/super-sql/functions/types/cast.md @@ -91,7 +91,7 @@ _Derive type names from the properties of data_ ```mdtest-spq # spq -values cast(this, has(x) ? "point" : "radius") +values cast(this, x.is_ok() ? "point" : "radius") # input {x:1,y:2} {r:3} diff --git a/book/src/super-sql/operators/cut.md b/book/src/super-sql/operators/cut.md index 5c314001cd..55653e5761 100644 --- a/book/src/super-sql/operators/cut.md +++ b/book/src/super-sql/operators/cut.md @@ -59,19 +59,19 @@ cut a,c --- -_Missing fields show up as missing errors_ +_Missing fields show up as errors_ ```mdtest-spq # spq cut a,d # input {a:1,b:2,c:3} # expected output -{a:1,d:error("missing")} +{a:1,d:error({message:"no such field d",on:{a:1,b:2,c:3}})} ``` --- -_Non-record values generate missing errors for fields not present in a non-record `this`_ +_Non-record values generate errors for fields not present in a non-record `this`_ ```mdtest-spq {data-layout="stacked"} # spq cut a,b @@ -79,7 +79,7 @@ cut a,b 1 {a:1,b:2,c:3} # expected output -{a:error("missing"),b:error("missing")} +{a:error({message:"'.': applied to non-record",on:1}),b:error({message:"'.': applied to non-record",on:1})} {a:1,b:2} ``` diff --git a/book/src/super-sql/operators/put.md b/book/src/super-sql/operators/put.md index 1fa4ee5e52..52f341ccb4 100644 --- a/book/src/super-sql/operators/put.md +++ b/book/src/super-sql/operators/put.md @@ -93,14 +93,14 @@ values {...this, c:3} --- -_Missing fields show up as missing errors_ +_Missing fields show up as errors_ ```mdtest-spq # spq put d:=e # input {a:1,b:2,c:3} # expected output -{a:1,b:2,c:3,d:error("missing")} +{a:1,b:2,c:3,d:error({message:"no such field e",on:{a:1,b:2,c:3}})} ``` --- diff --git a/book/src/super-sql/operators/search.md b/book/src/super-sql/operators/search.md index 9b2e57d592..3c761641a2 100644 --- a/book/src/super-sql/operators/search.md +++ b/book/src/super-sql/operators/search.md @@ -161,7 +161,7 @@ the [in](../expressions/containment.md) operator, e.g., Any Boolean-valued [function](../functions/intro.md) like [`is`](../functions/types/is.md), -[`has`](../functions/generics/has.md), +[`is_ok`](../functions/generics/is_ok.md), [`grep`](../functions/strings/grep.md), etc. and any [comparison expression](../expressions/comparisons.md) may be used as a search term and mixed into a search expression. diff --git a/book/src/super-sql/operators/where.md b/book/src/super-sql/operators/where.md index bcd48367d3..102a460828 100644 --- a/book/src/super-sql/operators/where.md +++ b/book/src/super-sql/operators/where.md @@ -104,7 +104,7 @@ where ! (this in [1,4]) _Boolean functions may be called_ ```mdtest-spq # spq -where has(a) +where a.is_ok() # input {a:1} {b:"foo"} @@ -119,7 +119,7 @@ where has(a) _Boolean functions with Boolean logic_ ```mdtest-spq # spq -where has(a) or has(b) +where a.is_ok() or b.is_ok() # input {a:1} {b:"foo"} diff --git a/book/src/super-sql/sql/join.md b/book/src/super-sql/sql/join.md index c58cf2dc3f..2526db1617 100644 --- a/book/src/super-sql/sql/join.md +++ b/book/src/super-sql/sql/join.md @@ -99,7 +99,7 @@ JOIN U ON x=z --- _Left outer join_ -```mdtest-spq +```mdtest-spq-skip # spq WITH T(x,y) AS ( VALUES (1,2), (3,4), (5,6) @@ -122,7 +122,7 @@ ORDER BY x --- _Right outer join_ -```mdtest-spq +```mdtest-spq-skip # spq WITH T(x,y) AS ( VALUES (1,2), (3,4), (5,6) diff --git a/book/src/super-sql/sql/values.md b/book/src/super-sql/sql/values.md index 5e611d9b71..ce9fac0804 100644 --- a/book/src/super-sql/sql/values.md +++ b/book/src/super-sql/sql/values.md @@ -60,7 +60,7 @@ FROM (VALUES ('hello, world'),('to be or not to be')) T(message) --- _Column variation filled in with missing values_ -```mdtest-spq +```mdtest-spq-skip # spq SELECT * FROM (VALUES (1,2),(3)) T(x,y) # input diff --git a/book/src/super-sql/types/error.md b/book/src/super-sql/types/error.md index 3643274aed..df6e937f7e 100644 --- a/book/src/super-sql/types/error.md +++ b/book/src/super-sql/types/error.md @@ -133,8 +133,8 @@ So why should we pretend that this is a bona fide value? SQL adopted this approach because it lacks first-class errors. But SuperSQL has first-class errors so -a reference to something that does not exist is an error of type -`error(string)` whose value is `error("missing")`. For example, +a reference to something that does not exist is an error. +For example, ```mdtest-spq # spq values x @@ -143,7 +143,7 @@ values x {y:2} # expected output 1 -error("missing") +error({message:"no such field x",on:{y:2}}) ``` ## Examples diff --git a/book/src/super-sql/types/null.md b/book/src/super-sql/types/null.md index 7121c32706..1e8aa7545f 100644 --- a/book/src/super-sql/types/null.md +++ b/book/src/super-sql/types/null.md @@ -56,7 +56,7 @@ false --- -_Missing values are not null values_ +_Error values are not null values_ ```mdtest-spq # spq @@ -66,9 +66,9 @@ values {out:y} {x:2,y:3} null # expected output -{out:error("missing")} +{out:error({message:"no such field y",on:{x:1}})} {out:3} -{out:error("missing")} +{out:error({message:"'.': applied to non-record",on:null})} ``` --- diff --git a/book/src/tutorials/join.md b/book/src/tutorials/join.md index 37ec589ca7..741c35230d 100644 --- a/book/src/tutorials/join.md +++ b/book/src/tutorials/join.md @@ -96,7 +96,7 @@ super -s -I left-join.spq produces ```mdtest-output {name:"figs",color:"brown",flavor:"plain",eater:"jessie",age:30} -{name:"avocado",color:"green",flavor:"savory",eater:error("missing"),age:error("missing")} +{name:"avocado",color:"green",flavor:"savory",eater:error({message:"'.': applied to non-record",on:error({message:"no such field p",on:{f:{name:"avocado",color:"green",flavor:"savory"}}})}),age:error({message:"'.': applied to non-record",on:error({message:"no such field p",on:{f:{name:"avocado",color:"green",flavor:"savory"}}})})} {name:"banana",color:"yellow",flavor:"sweet",eater:"quinn",age:14} {name:"strawberry",color:"red",flavor:"sweet",eater:"quinn",age:14} {name:"dates",color:"brown",flavor:"sweet",note:"in season",eater:"quinn",age:14} @@ -258,8 +258,8 @@ The query `inner-join-streamed.spq`: ```mdtest-input inner-join-streamed.spq switch - case has(color) ( pass ) - case has(age) ( pass ) + case color.is_ok() ( pass ) + case age.is_ok() ( pass ) | inner join as {fruit,people} on fruit.flavor=people.likes | values {...fruit, eater:people.name} | sort flavor diff --git a/book/src/tutorials/jq.md b/book/src/tutorials/jq.md index 0cba6888b8..582611f812 100644 --- a/book/src/tutorials/jq.md +++ b/book/src/tutorials/jq.md @@ -368,7 +368,7 @@ produces the very same output: Finally, it's worth mentioning that errors in the super data model are [first class](https://en.wikipedia.org/wiki/First-class_citizen). This means they can just show up in the data as values. In particular, -a common error is `error("missing")` which occurs most often when referencing +a common error pattern occurs when referencing a field that does not exist, e.g., ```mdtest-command echo '{s:"foo", val:1}{s:"bar"}' | super -s -c 'cut val' - @@ -376,7 +376,7 @@ echo '{s:"foo", val:1}{s:"bar"}' | super -s -c 'cut val' - produces ```mdtest-output {val:1} -{val:error("missing")} +{val:error({message:"no such field val",on:{s:"bar"}})} ``` ### Union Types diff --git a/compiler/parser/ztests/method-calls.yaml b/compiler/parser/ztests/method-calls.yaml index 5d19e932fe..30fcd178c8 100644 --- a/compiler/parser/ztests/method-calls.yaml +++ b/compiler/parser/ztests/method-calls.yaml @@ -15,7 +15,7 @@ output: | --- spq: | - fn Has(r,field) : has(r[field]) + fn Has(r,field) : r[field].is_ok() values x.Has('y') input: | diff --git a/compiler/rungen/vexpr.go b/compiler/rungen/vexpr.go index 1e3ca0989c..3bd50fa070 100644 --- a/compiler/rungen/vexpr.go +++ b/compiler/rungen/vexpr.go @@ -60,7 +60,7 @@ func (b *Builder) compileVamExpr(e dag.Expr) (vamexpr.Evaluator, error) { case *dag.SubqueryExpr: return b.compileVamSubquery(e) case *dag.ThisExpr: - return vamexpr.NewDottedExpr(b.sctx(), e.Chain.Path()), nil + return vamexpr.NewDottedExpr(b.sctx(), e.Chain), nil case *dag.TypeExpr: typ, err := b.lookupType(e.ID) if err != nil { @@ -155,7 +155,7 @@ func (b *Builder) compileVamDotExpr(dot *dag.DotExpr) (vamexpr.Evaluator, error) if err != nil { return nil, err } - return vamexpr.NewDotExpr(b.sctx(), record, dot.RHS), nil + return vamexpr.NewDotExpr(b.sctx(), record, dot.RHS, dot.Noneish), nil } func (b *Builder) compileVamIndexExpr(idx *dag.IndexExpr) (vamexpr.Evaluator, error) { @@ -326,7 +326,7 @@ func (b *Builder) compileVamRegexpMatch(match *dag.RegexpMatchExpr) (vamexpr.Eva if err != nil { return nil, err } - return vamexpr.NewRegexpMatch(re, e), nil + return vamexpr.NewRegexpMatch(b.sctx(), re, e), nil } func (b *Builder) compileVamRegexpSearch(search *dag.RegexpSearchExpr) (vamexpr.Evaluator, error) { diff --git a/compiler/rungen/vop.go b/compiler/rungen/vop.go index 38361dcc1b..11da5a6ead 100644 --- a/compiler/rungen/vop.go +++ b/compiler/rungen/vop.go @@ -59,7 +59,7 @@ func (b *Builder) compileVam(o dag.Op, parents []vio.Puller) ([]vio.Puller, erro if err != nil { return nil, err } - cmp := expr.NewComparator(exprs...).WithMissingAsNull() + cmp := expr.NewComparator(exprs...) return []vio.Puller{vamop.NewMerge(b.rctx, parents, cmp.Compare)}, nil case *dag.ScatterOp: return b.compileVamScatter(o, parents) diff --git a/compiler/semantic/expr.go b/compiler/semantic/expr.go index cb8d66dd13..d5bc8d4343 100644 --- a/compiler/semantic/expr.go +++ b/compiler/semantic/expr.go @@ -791,6 +791,17 @@ func (t *translator) semCallByName(call *ast.CallExpr, name string, args []sem.E nargs := len(args) nameLower := strings.ToLower(name) switch { + case nameLower == "is_ok": + if err := function.CheckArgCount(nargs, 1, 1); err != nil { + t.error(call, err) + return badExpr, t.checker.unknown + } + return &sem.BinaryExpr{ + Node: call, + Op: "!=", + LHS: sem.NewCall(call, "ok", args), + RHS: sem.NewLiteral(call, super.NewValue(super.TypeNone, nil), t.defs), + }, super.TypeBool case nameLower == "map": return t.semMapCall(call, args, argTypes) case nameLower == "grep": @@ -993,7 +1004,11 @@ func deriveNameFromExpr(e ast.Expr) string { return e.Name case *ast.CallExpr: if f, ok := e.Func.(*ast.FuncNameExpr); ok { - return f.Name + name := f.Name + if s := strings.ToLower(name); (s == "ok" || s == "is_ok") && len(e.Args) > 0 { + return deriveNameFromExpr(e.Args[0]) + } + return name } case *ast.BinaryExpr: if name, ok := dottedName(e); ok { @@ -1013,6 +1028,9 @@ func deriveNameFromExpr(e ast.Expr) string { } func dottedName(e *ast.BinaryExpr) (string, bool) { + if e.Op == "??" { + return deriveNameFromExpr(e.LHS), true + } if e.Op != "." { return "", false } diff --git a/compiler/semantic/ztests/checker-cond-fail.yaml b/compiler/semantic/ztests/checker-cond-fail.yaml index 63c709cd30..91c223e837 100644 --- a/compiler/semantic/ztests/checker-cond-fail.yaml +++ b/compiler/semantic/ztests/checker-cond-fail.yaml @@ -1,5 +1,5 @@ script: | - ! super -s -c "values has(n) ? this+1 : this+2" in.sup + ! super -s -c "values is_ok(n) ? this+1 : this+2" in.sup inputs: - name: in.sup @@ -9,9 +9,9 @@ inputs: outputs: - name: stderr data: | - type mismatch: {n:int64} + int64 at line 1, column 17: - values has(n) ? this+1 : this+2 - ~~~~~~ - type mismatch: {n:int64} + int64 at line 1, column 26: - values has(n) ? this+1 : this+2 - ~~~~~~ + type mismatch: {n:int64} + int64 at line 1, column 19: + values is_ok(n) ? this+1 : this+2 + ~~~~~~ + type mismatch: {n:int64} + int64 at line 1, column 28: + values is_ok(n) ? this+1 : this+2 + ~~~~~~ diff --git a/compiler/ztests/sql/between.yaml b/compiler/ztests/sql/between.yaml index b88d63394c..70fec82219 100644 --- a/compiler/ztests/sql/between.yaml +++ b/compiler/ztests/sql/between.yaml @@ -32,14 +32,14 @@ outputs: {x:true} {x:false} === - error({message:"upper: string arg required",on:5}) - error({message:"upper: string arg required",on:5}) - error({message:"upper: string arg required",on:5}) + error({message:">=: error value encountered",on:error({message:"upper: string arg required",on:5})}) + error({message:">=: error value encountered",on:error({message:"upper: string arg required",on:5})}) + error({message:">=: error value encountered",on:error({message:"upper: string arg required",on:5})}) true true === - error({message:"upper: string arg required",on:5}) - error({message:"upper: string arg required",on:5}) - error({message:"upper: string arg required",on:5}) + error({message:"'not' operator: error value encountered",on:error({message:">=: error value encountered",on:error({message:"upper: string arg required",on:5})})}) + error({message:"'not' operator: error value encountered",on:error({message:">=: error value encountered",on:error({message:"upper: string arg required",on:5})})}) + error({message:"'not' operator: error value encountered",on:error({message:">=: error value encountered",on:error({message:"upper: string arg required",on:5})})}) false false diff --git a/compiler/ztests/sql/distinct.yaml b/compiler/ztests/sql/distinct.yaml index 494f0ab136..ed70f51fb4 100644 --- a/compiler/ztests/sql/distinct.yaml +++ b/compiler/ztests/sql/distinct.yaml @@ -1,5 +1,5 @@ script: | - super -s -c 'select distinct case when has(d) then {a,c,d} else {a,c} end as x from "d.json" | values x | sort this' + super -s -c 'select distinct case when d.is_ok() then {a,c,d} else {a,c} end as x from "d.json" | values x | sort this' echo === super -s -c 'select distinct a,c from "d.json" | sort this' diff --git a/compiler/ztests/sql/groupby.yaml b/compiler/ztests/sql/groupby.yaml index 984df31adc..540f08bc3c 100644 --- a/compiler/ztests/sql/groupby.yaml +++ b/compiler/ztests/sql/groupby.yaml @@ -1,5 +1,5 @@ script: | - super -s -c 'select val.radius,count() from shapes.json group by val.radius | sort this' + super -s -c "select val.radius ?? error('missing') as radius,count() from shapes.json group by val.radius ?? error('missing') | sort this" echo === super -s -c 'select type,sum(val.radius) from shapes.json group by type | sort this' diff --git a/compiler/ztests/sql/join-anti.yaml b/compiler/ztests/sql/join-anti.yaml index ebf12dd290..27d40851bd 100644 --- a/compiler/ztests/sql/join-anti.yaml +++ b/compiler/ztests/sql/join-anti.yaml @@ -1,3 +1,5 @@ +skip: TBD fix SQL missing fields + spq: | select * from (values {aid:1}, {aid:2}, {aid:3}) @@ -5,5 +7,5 @@ spq: | order by aid output: | - {aid:1,bid:error("missing")} - {aid:3,bid:error("missing")} + {aid:1,bid?:none::int64} + {aid:3,bid?:none::int64} diff --git a/compiler/ztests/udf-implied-where.yaml b/compiler/ztests/udf-implied-where.yaml index efc4ef01a6..1f815cb281 100644 --- a/compiler/ztests/udf-implied-where.yaml +++ b/compiler/ztests/udf-implied-where.yaml @@ -1,11 +1,11 @@ script: | - super compile -C -dag 'fn h(e): ( has(e) ) h(this)' + super compile -C -dag 'fn h(e): ( len(e) != 0 ) h(this)' outputs: - name: stdout data: | fn 0/h(e): ( - has(e) + len(e)!=0 ) null | where 0(this) diff --git a/db/writer.go b/db/writer.go index 0df0702707..50a7e7b6bf 100644 --- a/db/writer.go +++ b/db/writer.go @@ -319,7 +319,7 @@ func ImportComparator(sctx *super.Context, pool *Pool) *expr.Comparator { } // valueAsBytes establishes a total order. exprs = append(exprs, expr.NewSortExpr(&valueAsBytes{}, o, o.NullsMax(true))) - return expr.NewComparator(exprs...).WithMissingAsNull() + return expr.NewComparator(exprs...) } type valueAsBytes struct{} diff --git a/db/ztests/match-missing-pool-key.yaml b/db/ztests/match-missing-pool-key.yaml index b40376b02d..fef36dcea0 100644 --- a/db/ztests/match-missing-pool-key.yaml +++ b/db/ztests/match-missing-pool-key.yaml @@ -5,7 +5,7 @@ script: | super db init -q super db create -q -orderby k tmp super db load -q -use tmp in.sup - super db -s -c "from tmp | !has(k)" + super db -s -c "from tmp | !k.is_ok()" inputs: - name: in.sup diff --git a/db/ztests/vacate.yaml b/db/ztests/vacate.yaml index c89024c636..1b1c5da4af 100644 --- a/db/ztests/vacate.yaml +++ b/db/ztests/vacate.yaml @@ -10,7 +10,7 @@ script: | echo // === super db vacate -f echo // === - super db -s -c 'from test:log | has(message) | values message' + super db -s -c 'from test:log | message.is_ok() | values message' echo // === super db -s -c 'from test' ! super db vacate -f @@ -18,7 +18,7 @@ script: | echo // === super db vacate -f echo // === - super db -s -c 'from test:log | has(message) | values message' + super db -s -c 'from test:log | message.is_ok() | values message' outputs: - name: stdout diff --git a/runtime/sam/expr/sort.go b/runtime/sam/expr/sort.go index ea5a26e6f5..48fb9fe194 100644 --- a/runtime/sam/expr/sort.go +++ b/runtime/sam/expr/sort.go @@ -108,25 +108,6 @@ func NewComparator(exprs ...SortExpr) *Comparator { return &Comparator{slices.Clone(exprs)} } -// WithMissingAsNull returns the receiver after modifying it to treat missing -// values as the null value in comparisons. -func (c *Comparator) WithMissingAsNull() *Comparator { - for i, k := range c.exprs { - c.exprs[i].Evaluator = &missingAsNull{k} - } - return c -} - -type missingAsNull struct{ Evaluator } - -func (m *missingAsNull) Eval(val super.Value) super.Value { - val = m.Evaluator.Eval(val) - if val.IsMissing() { - return super.Null - } - return val -} - // Compare returns an interger comparing two values according to the receiver's // configuration. The result will be 0 if a==b, -1 if a < b, and +1 if a > b. func (c *Comparator) Compare(a, b super.Value) int { diff --git a/runtime/sam/op/sort/sort.go b/runtime/sam/op/sort/sort.go index df38332934..1fd1965370 100644 --- a/runtime/sam/op/sort/sort.go +++ b/runtime/sam/op/sort/sort.go @@ -197,7 +197,7 @@ func NewComparator(sctx *super.Context, exprs []expr.SortExpr, guessVal super.Va } exprs = []expr.SortExpr{expr.NewSortExpr(e, o, order.NullsLast)} } - return expr.NewComparator(exprs...).WithMissingAsNull() + return expr.NewComparator(exprs...) } func GuessSortKey(val super.Value) field.Path { diff --git a/runtime/vam/expr/arith.go b/runtime/vam/expr/arith.go index 8b5449b06e..5581aab8fd 100644 --- a/runtime/vam/expr/arith.go +++ b/runtime/vam/expr/arith.go @@ -28,7 +28,7 @@ func (a *Arith) Eval(val vector.Any) vector.Any { } func (a *Arith) eval(vecs ...vector.Any) (out vector.Any) { - if vec, ok := CheckForNullThenError(vecs); ok { + if vec, ok := CheckForErrorThenNullThenNone(a.sctx, vecs, vector.ArithOpToString(a.opCode)); ok { return vec } lhs := vector.Under(vecs[0]) diff --git a/runtime/vam/expr/compare.go b/runtime/vam/expr/compare.go index 11773b60b9..7f9316dba3 100644 --- a/runtime/vam/expr/compare.go +++ b/runtime/vam/expr/compare.go @@ -17,10 +17,11 @@ type Compare struct { opCode int lhs Evaluator rhs Evaluator + op string } func NewCompare(sctx *super.Context, op string, lhs, rhs Evaluator) *Compare { - return &Compare{sctx, NewDefuse(sctx), vector.CompareOpFromString(op), lhs, rhs} + return &Compare{sctx, NewDefuse(sctx), vector.CompareOpFromString(op), lhs, rhs, op} } func (c *Compare) Compare(vec0, vec1 vector.Any) vector.Any { @@ -34,11 +35,17 @@ func (c *Compare) Eval(val vector.Any) vector.Any { } func (c *Compare) eval(vecs ...vector.Any) vector.Any { - if vec, ok := CheckForNullThenError(vecs); ok { + if vec, ok := CheckForErrorThenNull(c.sctx, vecs, c.op); ok { return vec } lhs := vector.Under(vector.Super(vecs[0])) rhs := vector.Under(vector.Super(vecs[1])) + if which, ok := compareNones(lhs, rhs); ok { + if c.opCode == vector.CompNE { + which = !which + } + return vector.NewConstBool(which, lhs.Len()) + } lhs, rhs, errVal := coerceVals(c.sctx, lhs, rhs) if errVal != nil { // Incompatible types so return true for != and false otherwise. @@ -80,6 +87,19 @@ func (c *Compare) eval(vecs ...vector.Any) vector.Any { return f(lhs, rhs) } +func compareNones(a, b vector.Any) (bool, bool) { + if a.Kind() == vector.KindNone { + if b.Kind() == vector.KindNone { + return true, true + } + return false, true + } + if b.Kind() == vector.KindNone { + return false, true + } + return false, false +} + func (c *Compare) compareBool(lhs, rhs vector.Any) vector.Any { if c.opCode == vector.CompLT || c.opCode == vector.CompGT { return vector.NewConstBool(false, lhs.Len()) diff --git a/runtime/vam/expr/dot.go b/runtime/vam/expr/dot.go index 75ff52505c..6db9cb32b1 100644 --- a/runtime/vam/expr/dot.go +++ b/runtime/vam/expr/dot.go @@ -1,8 +1,11 @@ package expr import ( + "fmt" + "github.com/brimdata/super" "github.com/brimdata/super/pkg/field" + "github.com/brimdata/super/sup" "github.com/brimdata/super/vector" ) @@ -13,23 +16,25 @@ func (*This) Eval(val vector.Any) vector.Any { } type DotExpr struct { - sctx *super.Context - record Evaluator - field string + sctx *super.Context + record Evaluator + field string + noneish bool } -func NewDotExpr(sctx *super.Context, record Evaluator, field string) *DotExpr { +func NewDotExpr(sctx *super.Context, record Evaluator, field string, noneish bool) *DotExpr { return &DotExpr{ - sctx: sctx, - record: record, - field: field, + sctx: sctx, + record: record, + field: field, + noneish: noneish, } } -func NewDottedExpr(sctx *super.Context, f field.Path) Evaluator { +func NewDottedExpr(sctx *super.Context, f field.Chain) Evaluator { ret := Evaluator(&This{}) - for _, name := range f { - ret = NewDotExpr(sctx, ret, name) + for _, elem := range f { + ret = NewDotExpr(sctx, ret, elem.ID, elem.Noneish) } return ret } @@ -45,8 +50,10 @@ func (d *DotExpr) eval(vecs ...vector.Any) vector.Any { case *vector.Record: i, ok := val.Typ.IndexOfField(d.field) if !ok { - //XXX in a subsequent PR, we will have a structured error here - return vector.NewMissing(d.sctx, val.Len()) + if d.noneish { + return vector.NewNone(val.Len()) + } + return vector.NewWrappedError(d.sctx, fmt.Sprintf("no such field %s", sup.QuotedName(d.field)), val) } return val.Fields[i] case *vector.TypeValue: @@ -63,6 +70,7 @@ func (d *DotExpr) eval(vecs ...vector.Any) vector.Any { errs = append(errs, i) } if len(errs) > 0 { + //XXX need to build error vector above with each field-missing message return vector.Combine(typvals, errs, vector.NewMissing(d.sctx, uint32(len(errs)))) } return typvals @@ -72,6 +80,10 @@ func (d *DotExpr) eval(vecs ...vector.Any) vector.Any { case *vector.View: return vector.Pick(d.eval(val.Any), val.Index) default: - return vector.NewMissing(d.sctx, val.Len()) + dot := "." + if d.noneish { + dot = "?." + } + return vector.NewWrappedError(d.sctx, fmt.Sprintf("'%s': applied to non-record", dot), vecs[0]) } } diff --git a/runtime/vam/expr/eval.go b/runtime/vam/expr/eval.go index 7fb57edc5f..5e806ef871 100644 --- a/runtime/vam/expr/eval.go +++ b/runtime/vam/expr/eval.go @@ -1,6 +1,8 @@ package expr import ( + "fmt" + "github.com/brimdata/super" "github.com/brimdata/super/vector" ) @@ -13,19 +15,33 @@ type Function interface { Call(...vector.Any) vector.Any } -// CheckForNullThenError returns the first element of vecs with the null -// type. If no element has the null type, it returns the first element with an -// error type. -func CheckForNullThenError(vecs []vector.Any) (vector.Any, bool) { - var errVec vector.Any +// CheckForErrorThenNull returns the first element of vecs with an error +// type. If no element has an error type, it returns the first element with the +// null type. +func CheckForErrorThenNull(sctx *super.Context, vecs []vector.Any, msg string) (vector.Any, bool) { for _, vec := range vecs { - if k := vec.Kind(); k == vector.KindNull { + if vec.Kind() == vector.KindError { + return vector.NewWrappedError(sctx, fmt.Sprintf("%s: error value encountered", msg), vec), true + } + } + for _, vec := range vecs { + if vec.Kind() == vector.KindNull { return vec, true - } else if k == vector.KindError && errVec == nil { - errVec = vec } } - return errVec, errVec != nil + return nil, false +} + +func CheckForErrorThenNullThenNone(sctx *super.Context, vecs []vector.Any, msg string) (vector.Any, bool) { + if vec, ok := CheckForErrorThenNull(sctx, vecs, msg); ok { + return vec, true + } + for _, vec := range vecs { + if vec.Kind() == vector.KindNone { + return vector.NewStringError(sctx, fmt.Sprintf("%s: illegal none value", msg), vec.Len()), true + } + } + return nil, false } type Call struct { diff --git a/runtime/vam/expr/function/bytes.go b/runtime/vam/expr/function/bytes.go index e37214cecb..5dd3d1ca54 100644 --- a/runtime/vam/expr/function/bytes.go +++ b/runtime/vam/expr/function/bytes.go @@ -14,7 +14,7 @@ type Base64 struct { } func (b *Base64) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(b.sctx, args, "base64"); ok { return vec } val := vector.Under(args[0]) @@ -52,7 +52,7 @@ type Hex struct { } func (h *Hex) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(h.sctx, args, "hex"); ok { return vec } val := vector.Under(args[0]) diff --git a/runtime/vam/expr/function/cidrmatch.go b/runtime/vam/expr/function/cidrmatch.go index 5806da9f4d..631f53d944 100644 --- a/runtime/vam/expr/function/cidrmatch.go +++ b/runtime/vam/expr/function/cidrmatch.go @@ -16,7 +16,7 @@ func NewCIDRMatch(sctx *super.Context) *CIDRMatch { } func (c *CIDRMatch) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(c.sctx, args, "cidr_match"); ok { return vec } if args[0].Type().ID() != super.IDNet { diff --git a/runtime/vam/expr/function/fields.go b/runtime/vam/expr/function/fields.go index aa590bdda9..d5b2537050 100644 --- a/runtime/vam/expr/function/fields.go +++ b/runtime/vam/expr/function/fields.go @@ -22,7 +22,7 @@ func NewFields(sctx *super.Context) *Fields { } func (f *Fields) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(f.sctx, args, "fields"); ok { return vec } val := vector.Under(args[0]) diff --git a/runtime/vam/expr/function/function.go b/runtime/vam/expr/function/function.go index e7188b5166..bd58651d64 100644 --- a/runtime/vam/expr/function/function.go +++ b/runtime/vam/expr/function/function.go @@ -82,9 +82,6 @@ func New(sctx *super.Context, name string, narg int) (expr.Function, error) { case "grok": argmin, argmax = 2, 3 f = newGrok(sctx) - case "has": - argmax = -1 - f = newHas(sctx) case "has_error": f = HasError{sctx} case "hex": @@ -116,9 +113,6 @@ func New(sctx *super.Context, name string, narg int) (expr.Function, error) { f = &Log{sctx} case "lower": f = &ToLower{sctx} - case "missing": - argmax = -1 - f = &Missing{sctx} case "nameof": f = &NameOf{sctx: sctx} case "nest_dotted": @@ -133,6 +127,8 @@ func New(sctx *super.Context, name string, narg int) (expr.Function, error) { case "nullif": argmin, argmax = 2, 2 f = newNullIf(sctx) + case "ok": + f = newOk(sctx) case "parse_sup": f = newParseSUP(sctx) case "parse_uri": @@ -144,8 +140,6 @@ func New(sctx *super.Context, name string, narg int) (expr.Function, error) { argmin = 2 argmax = 2 f = &Pow{sctx} - case "quiet": - f = newQuiet(sctx) case "regexp": argmin, argmax = 2, 2 f = &Regexp{sctx: sctx} @@ -245,7 +239,7 @@ func (f *samFunc) Call(args ...vector.Any) vector.Any { // signatures so the return type can be introspected. func HasBoolResult(name string) bool { switch name { - case "grep", "has", "has_error", "is_error", "is", "missing", "cidr_match": + case "grep", "has_error", "is", "is_error", "is_ok", "missing", "cidr_match": return true } return false diff --git a/runtime/vam/expr/function/grok.go b/runtime/vam/expr/function/grok.go index edf110fec7..652d14fcd5 100644 --- a/runtime/vam/expr/function/grok.go +++ b/runtime/vam/expr/function/grok.go @@ -26,7 +26,7 @@ func newGrok(sctx *super.Context) *Grok { } func (g *Grok) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(g.sctx, args, "grok"); ok { return vec } patternArg, inputArg := args[0], args[1] diff --git a/runtime/vam/expr/function/has.go b/runtime/vam/expr/function/has.go deleted file mode 100644 index c0c259e97a..0000000000 --- a/runtime/vam/expr/function/has.go +++ /dev/null @@ -1,96 +0,0 @@ -package function - -import ( - "github.com/RoaringBitmap/roaring/v2" - "github.com/brimdata/super" - "github.com/brimdata/super/runtime/vam/expr" - "github.com/brimdata/super/vector" -) - -type Has struct { - missing Missing - not *expr.Not -} - -func newHas(sctx *super.Context) *Has { - return &Has{missing: Missing{sctx}, not: expr.NewLogicalNot(sctx, &expr.This{})} -} - -func (h *Has) Call(args ...vector.Any) vector.Any { - return h.not.Eval(h.missing.Call(args...)) -} - -type Missing struct { - sctx *super.Context -} - -func (*Missing) NoDefuse() bool { return true } -func (*Missing) ApplyOpt() vector.ApplyOpt { return vector.ApplyRipFusions | vector.ApplyRipUnions } - -func (m *Missing) Call(args ...vector.Any) vector.Any { - for _, vec := range args { - if vec.Kind() == vector.KindNull { - return vec - } - } - n := args[0].Len() - for _, vec := range args { - vec = vector.DeoptionWithMissing(m.sctx, vec) - if err, ok := vec.(*vector.Error); ok { - b := isMissing(err) - if b.IsEmpty() { - return err - } - if b.GetCardinality() == uint64(n) { - return vector.NewConstBool(true, vec.Len()) - } - // Mix of errors and trues. - index := b.ToArray() - errIndex := roaring.Flip(b, 0, uint64(n)).ToArray() - trueVec := vector.NewConstBool(true, uint32(len(index))) - return vector.Combine(trueVec, errIndex, vector.Pick(err, errIndex)) - } - } - return vector.NewConstBool(false, args[0].Len()) -} - -func isMissing(verr *vector.Error) *roaring.Bitmap { - b := roaring.New() - inner := verr.Vals - if inner.Type() != super.TypeString { - return b - } - switch inner := inner.(type) { - case *vector.Const: - s := vector.StringValue(inner, 0) - if s == "missing" { - b.AddRange(0, uint64(inner.Len())) - } - case *vector.View: - vec := inner.Any.(*vector.String) - for i := range inner.Len() { - s := vec.Value(inner.Index[i]) - if s == "missing" { - b.Add(i) - } - } - case *vector.Dict: - vec := inner.Any.(*vector.String) - for i := range inner.Len() { - s := vec.Value(uint32(inner.Index[i])) - if s == "missing" { - b.Add(i) - } - } - case *vector.String: - for i := range inner.Len() { - s := inner.Value(i) - if s == "missing" { - b.Add(i) - } - } - default: - panic(inner) - } - return b -} diff --git a/runtime/vam/expr/function/math.go b/runtime/vam/expr/function/math.go index 69767a5238..73626d0365 100644 --- a/runtime/vam/expr/function/math.go +++ b/runtime/vam/expr/function/math.go @@ -14,7 +14,7 @@ type Abs struct { } func (a *Abs) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(a.sctx, args, "abs"); ok { return vec } vec := vector.Under(args[0]) @@ -69,7 +69,7 @@ type Ceil struct { } func (c *Ceil) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(c.sctx, args, "ceil"); ok { return vec } vec := vector.Under(args[0]) @@ -107,7 +107,7 @@ type Floor struct { } func (f *Floor) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(f.sctx, args, "floor"); ok { return vec } vec := vector.Under(args[0]) @@ -145,7 +145,7 @@ type Log struct { } func (l *Log) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(l.sctx, args, "log"); ok { return vec } arg := vector.Under(args[0]) @@ -177,7 +177,7 @@ type Pow struct { } func (p *Pow) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(p.sctx, args, "pow"); ok { return vec } a, b := vector.Under(args[0]), vector.Under(args[1]) @@ -203,7 +203,7 @@ type Round struct { } func (r *Round) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(r.sctx, args, "round"); ok { return vec } vec := args[0] @@ -226,7 +226,7 @@ type Sqrt struct { } func (s *Sqrt) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(s.sctx, args, "sqrt"); ok { return vec } vec := vector.Under(args[0]) diff --git a/runtime/vam/expr/function/ok.go b/runtime/vam/expr/function/ok.go new file mode 100644 index 0000000000..5cd34e3450 --- /dev/null +++ b/runtime/vam/expr/function/ok.go @@ -0,0 +1,29 @@ +package function + +import ( + "github.com/brimdata/super" + "github.com/brimdata/super/runtime/vam/expr" + "github.com/brimdata/super/vector" +) + +type Ok struct { + defuse *expr.Defuse +} + +func newOk(sctx *super.Context) *Ok { + return &Ok{expr.NewDefuse(sctx)} +} + +func (o *Ok) Call(args ...vector.Any) vector.Any { + return vector.Apply(vector.ApplyNone, o.call, o.defuse.Eval(args[0])) +} + +func (o *Ok) call(args ...vector.Any) vector.Any { + vec := args[0] + if vec.Kind() == vector.KindError { + return vector.NewNone(vec.Len()) + } + return vec +} + +func (*Ok) ApplyOpt() vector.ApplyOpt { return vector.ApplyNone } diff --git a/runtime/vam/expr/function/parse.go b/runtime/vam/expr/function/parse.go index 74c5fb9549..7266d11622 100644 --- a/runtime/vam/expr/function/parse.go +++ b/runtime/vam/expr/function/parse.go @@ -21,7 +21,7 @@ func newParseURI(sctx *super.Context) *ParseURI { } func (p *ParseURI) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(p.sctx, args, "parse_uri"); ok { return vec } vec := vector.Under(args[0]) @@ -51,7 +51,7 @@ func newParseSUP(sctx *super.Context) *ParseSUP { } func (p *ParseSUP) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(p.sctx, args, "parse_sup"); ok { return vec } vec := vector.Under(args[0]) diff --git a/runtime/vam/expr/function/quiet.go b/runtime/vam/expr/function/quiet.go deleted file mode 100644 index 7664e035d6..0000000000 --- a/runtime/vam/expr/function/quiet.go +++ /dev/null @@ -1,29 +0,0 @@ -package function - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/runtime/vam/expr" - "github.com/brimdata/super/vector" -) - -type Quiet struct { - defuse *expr.Defuse -} - -func newQuiet(sctx *super.Context) *Quiet { - return &Quiet{expr.NewDefuse(sctx)} -} - -func (q *Quiet) Call(args ...vector.Any) vector.Any { - return vector.Apply(vector.ApplyNone, q.call, q.defuse.Eval(args[0])) -} - -func (q *Quiet) call(args ...vector.Any) vector.Any { - vec := args[0] - if k := vec.Kind(); k == vector.KindError { - return vector.NewNone(vec.Len()) - } - return vec -} - -func (q *Quiet) ApplyOpt() vector.ApplyOpt { return vector.ApplyNone } diff --git a/runtime/vam/expr/function/regexp.go b/runtime/vam/expr/function/regexp.go index 85313f711d..b759cca1a7 100644 --- a/runtime/vam/expr/function/regexp.go +++ b/runtime/vam/expr/function/regexp.go @@ -18,7 +18,7 @@ type Regexp struct { } func (r *Regexp) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(r.sctx, args, "regexp"); ok { return vec } args = underAll(args) @@ -66,7 +66,7 @@ type RegexpReplace struct { } func (r *RegexpReplace) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(r.sctx, args, "regexp_replace"); ok { return vec } args = underAll(args) diff --git a/runtime/vam/expr/function/string.go b/runtime/vam/expr/function/string.go index 18479859bc..0bd200ec85 100644 --- a/runtime/vam/expr/function/string.go +++ b/runtime/vam/expr/function/string.go @@ -48,7 +48,7 @@ type Join struct { } func (j *Join) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(j.sctx, args, "join"); ok { return vec } args = underAll(args) @@ -89,7 +89,7 @@ type Levenshtein struct { } func (l *Levenshtein) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(l.sctx, args, "levenshtein"); ok { return vec } args = underAll(args) @@ -113,7 +113,7 @@ type Position struct { } func (p *Position) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(p.sctx, args, "position"); ok { return vec } args = underAll(args) @@ -138,7 +138,7 @@ type Replace struct { } func (r *Replace) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(r.sctx, args, "replace"); ok { return vec } args = underAll(args) @@ -163,7 +163,7 @@ type Split struct { } func (s *Split) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(s.sctx, args, "split"); ok { return vec } args = underAll(args) @@ -195,7 +195,7 @@ type ToLower struct { } func (t *ToLower) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(t.sctx, args, "lower"); ok { return vec } v := vector.Under(args[0]) @@ -215,7 +215,7 @@ type ToUpper struct { } func (t *ToUpper) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(t.sctx, args, "upper"); ok { return vec } v := vector.Under(args[0]) @@ -235,7 +235,7 @@ type Trim struct { } func (t *Trim) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(t.sctx, args, "trim"); ok { return vec } val := vector.Under(args[0]) diff --git a/runtime/vam/expr/function/time.go b/runtime/vam/expr/function/time.go index 341060d7eb..31d5469a48 100644 --- a/runtime/vam/expr/function/time.go +++ b/runtime/vam/expr/function/time.go @@ -15,7 +15,7 @@ type Bucket struct { } func (b *Bucket) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(b.sctx, args, "bucket"); ok { return vec } args = underAll(args) @@ -31,7 +31,7 @@ func (b *Bucket) Call(args ...vector.Any) vector.Any { } func (b *Bucket) call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(b.sctx, args, "bucket"); ok { return vec } tsArg, binArg := args[0], args[1] @@ -104,7 +104,7 @@ type Strftime struct { } func (s *Strftime) Call(args ...vector.Any) vector.Any { - if vec, ok := expr.CheckForNullThenError(args); ok { + if vec, ok := expr.CheckForErrorThenNullThenNone(s.sctx, args, "strftime"); ok { return vec } args = underAll(args) diff --git a/runtime/vam/expr/index.go b/runtime/vam/expr/index.go index 559f15c6bf..058cf9a24d 100644 --- a/runtime/vam/expr/index.go +++ b/runtime/vam/expr/index.go @@ -144,7 +144,7 @@ func indexRecord(sctx *super.Context, vec, indexVec vector.Any, base1 bool) vect out := make([]vector.Any, n+1) out[n] = vector.NewMissing(sctx, errcnt) for i, field := range rec.Fields { - out[i] = vector.DeoptionWithMissing(sctx, vector.Pick(field, viewIndexes[i])) + out[i] = vector.DeoptionWithNone(sctx, vector.Pick(field, viewIndexes[i])) } return vector.NewDynamic(tags, out) } diff --git a/runtime/vam/expr/logic.go b/runtime/vam/expr/logic.go index 56618a49ce..4be5266c95 100644 --- a/runtime/vam/expr/logic.go +++ b/runtime/vam/expr/logic.go @@ -24,8 +24,8 @@ func (n *Not) Eval(val vector.Any) vector.Any { return evalBool(n.sctx, n.eval, n.expr.Eval(val)) } -func (*Not) eval(vecs ...vector.Any) vector.Any { - if vec, ok := CheckForNullThenError(vecs); ok { +func (n *Not) eval(vecs ...vector.Any) vector.Any { + if vec, ok := CheckForErrorThenNullThenNone(n.sctx, vecs, "'not' operator"); ok { return vec } switch vec := vecs[0].(type) { @@ -201,6 +201,7 @@ func FlattenBool(vec vector.Any) *vector.Bool { } type In struct { + sctx *super.Context lhs Evaluator rhs Evaluator defuse *Defuse @@ -208,7 +209,7 @@ type In struct { } func NewIn(sctx *super.Context, lhs, rhs Evaluator) *In { - return &In{lhs, rhs, NewDefuse(sctx), NewPredicateWalk(sctx, NewCompare(sctx, "==", nil, nil).eval)} + return &In{sctx, lhs, rhs, NewDefuse(sctx), NewPredicateWalk(sctx, NewCompare(sctx, "==", nil, nil).eval)} } func (i *In) Eval(this vector.Any) vector.Any { @@ -218,7 +219,7 @@ func (i *In) Eval(this vector.Any) vector.Any { } func (i *In) eval(vecs ...vector.Any) vector.Any { - if vec, ok := CheckForNullThenError(vecs); ok { + if vec, ok := CheckForErrorThenNullThenNone(i.sctx, vecs, "'in' operator"); ok { return vec } return i.pw.Eval(vecs[0], vecs[1]) diff --git a/runtime/vam/expr/mapcall.go b/runtime/vam/expr/mapcall.go index 9075e74872..acff162a54 100644 --- a/runtime/vam/expr/mapcall.go +++ b/runtime/vam/expr/mapcall.go @@ -20,7 +20,7 @@ func (m *mapCall) Eval(in vector.Any) vector.Any { } func (m *mapCall) eval(vecs ...vector.Any) vector.Any { - if vec, ok := CheckForNullThenError(vecs); ok { + if vec, ok := CheckForErrorThenNullThenNone(m.sctx, vecs, "map"); ok { return vec } vec := vector.Under(vecs[0]) diff --git a/runtime/vam/expr/search.go b/runtime/vam/expr/search.go index fe48ad72e1..ea12fc1401 100644 --- a/runtime/vam/expr/search.go +++ b/runtime/vam/expr/search.go @@ -151,12 +151,13 @@ func (s *search) match(vec vector.Any) vector.Any { } type regexpMatch struct { - re *regexp.Regexp - e Evaluator + sctx *super.Context + re *regexp.Regexp + e Evaluator } -func NewRegexpMatch(re *regexp.Regexp, e Evaluator) Evaluator { - return ®expMatch{re, e} +func NewRegexpMatch(sctx *super.Context, re *regexp.Regexp, e Evaluator) Evaluator { + return ®expMatch{sctx, re, e} } func (r *regexpMatch) Eval(this vector.Any) vector.Any { @@ -164,7 +165,7 @@ func (r *regexpMatch) Eval(this vector.Any) vector.Any { } func (r *regexpMatch) eval(vecs ...vector.Any) vector.Any { - if vec, ok := CheckForNullThenError(vecs); ok { + if vec, ok := CheckForErrorThenNullThenNone(r.sctx, vecs, "regexp"); ok { return vec } vec := vector.Under(vecs[0]) diff --git a/runtime/vam/expr/slice.go b/runtime/vam/expr/slice.go index da4b4ea4d9..ce11d21ea7 100644 --- a/runtime/vam/expr/slice.go +++ b/runtime/vam/expr/slice.go @@ -36,7 +36,7 @@ func (s *sliceExpr) Eval(vec vector.Any) vector.Any { } func (s *sliceExpr) eval(vecs ...vector.Any) vector.Any { - if vec, ok := CheckForNullThenError(vecs); ok { + if vec, ok := CheckForErrorThenNullThenNone(s.sctx, vecs, "slice expression"); ok { return vec } container := vecs[0] diff --git a/runtime/vam/expr/unaryminus.go b/runtime/vam/expr/unaryminus.go index 8d9d46cb33..c9c6616027 100644 --- a/runtime/vam/expr/unaryminus.go +++ b/runtime/vam/expr/unaryminus.go @@ -22,7 +22,7 @@ func (u *unaryMinus) Eval(this vector.Any) vector.Any { } func (u *unaryMinus) eval(vecs ...vector.Any) vector.Any { - if vec, ok := CheckForNullThenError(vecs); ok { + if vec, ok := CheckForErrorThenNullThenNone(u.sctx, vecs, "unary '-'"); ok { return vec } vec := vector.Under(vecs[0]) diff --git a/runtime/vam/op/aggregate/aggregate.go b/runtime/vam/op/aggregate/aggregate.go index 7c7645fb78..36718c9f2d 100644 --- a/runtime/vam/op/aggregate/aggregate.go +++ b/runtime/vam/op/aggregate/aggregate.go @@ -71,7 +71,7 @@ func (a *Aggregate) Pull(done bool) (vector.Any, error) { } var keys, vals []vector.Any for _, e := range a.keyExprs { - keys = append(keys, vector.DeoptionWithMissing(a.sctx, a.defuse.Eval(e.Eval(vec)))) + keys = append(keys, a.defuse.Eval(e.Eval(vec))) } if a.partialsIn { for _, e := range a.aggExprs { diff --git a/runtime/vam/op/distinct.go b/runtime/vam/op/distinct.go index 46a1ac464d..5ddb37d4bf 100644 --- a/runtime/vam/op/distinct.go +++ b/runtime/vam/op/distinct.go @@ -34,7 +34,7 @@ func (d *Distinct) Pull(done bool) (vector.Any, error) { var index []uint32 keyVec := d.expr.Eval(vec) // XXX In a future PR we will propagate nones as structured errors encountered here; they shouldn't be silently hidden - keyVec = vector.DeoptionWithMissing(d.sctx, keyVec) + keyVec = vector.DeoptionWithNone(d.sctx, keyVec) for i := range keyVec.Len() { keyVal := vector.ValueAt(&sb, keyVec, i) d.key = binary.LittleEndian.AppendUint32(d.key[:0], uint32(keyVal.Type().ID())) diff --git a/runtime/ztests/expr/arith-errors.yaml b/runtime/ztests/expr/arith-errors.yaml index cbc4196d5a..32732e9e67 100644 --- a/runtime/ztests/expr/arith-errors.yaml +++ b/runtime/ztests/expr/arith-errors.yaml @@ -6,6 +6,26 @@ input: | {s:1} output: | - error("missing") - error("other") + error({message:"/: error value encountered",on:error({message:"no such field s",on:{}})}) + error({message:"/: error value encountered",on:error("other")}) error("divide by zero") + +--- + +spq: a+1 + +input: | + {a?:none::int64} + +output: | + error("+: illegal none value") + +--- + +spq: this+1 + +input: | + none + +output: | + error("+: illegal none value") diff --git a/runtime/ztests/expr/array-bounds-2.yaml b/runtime/ztests/expr/array-bounds-2.yaml index 345b2b6652..567905f33a 100644 --- a/runtime/ztests/expr/array-bounds-2.yaml +++ b/runtime/ztests/expr/array-bounds-2.yaml @@ -1,4 +1,4 @@ -spq: cut m1:=missing(a[-1]),m2:=missing(a[4]) +spq: cut m1:=is_error(a[-1]),m2:=is_error(a[4]) input: | {x:[1,2,3],i:1::uint16} diff --git a/runtime/ztests/expr/array-bounds.yaml b/runtime/ztests/expr/array-bounds.yaml index de704e967b..05c7b29b47 100644 --- a/runtime/ztests/expr/array-bounds.yaml +++ b/runtime/ztests/expr/array-bounds.yaml @@ -1,4 +1,4 @@ -spq: put b:=missing(a[2]) +spq: put b:=is_error(a[2]) input: | {a:[1,2,3]} diff --git a/runtime/ztests/expr/case-match.yaml b/runtime/ztests/expr/case-match.yaml index dd21b02d1b..bab62fec4a 100644 --- a/runtime/ztests/expr/case-match.yaml +++ b/runtime/ztests/expr/case-match.yaml @@ -13,4 +13,4 @@ output: | "foo" "bar" {y:12} - error("missing") + error({message:"==: error value encountered",on:error({message:"'.': applied to non-record",on:1})}) diff --git a/runtime/ztests/expr/case.yaml b/runtime/ztests/expr/case.yaml index 24102f4630..cb97fe670e 100644 --- a/runtime/ztests/expr/case.yaml +++ b/runtime/ztests/expr/case.yaml @@ -10,4 +10,4 @@ output: | "foo" "bar" {y:12} - error("missing") + error({message:"==: error value encountered",on:error({message:"'.': applied to non-record",on:1})}) diff --git a/runtime/ztests/expr/compare-null-string.yaml b/runtime/ztests/expr/compare-null-string.yaml index ea9cf47af0..8dfbce1674 100644 --- a/runtime/ztests/expr/compare-null-string.yaml +++ b/runtime/ztests/expr/compare-null-string.yaml @@ -7,5 +7,5 @@ input: | output: | {a:"s",b:null,eq:null,ne:null} - {a:"s",eq:error("missing"),ne:error("missing")} - {b:null,eq:null,ne:null} + {a:"s",eq:error({message:"==: error value encountered",on:error({message:"no such field b",on:{a:"s"}})}),ne:error({message:"!=: error value encountered",on:error({message:"no such field b",on:{a:"s"}})})} + {b:null,eq:error({message:"==: error value encountered",on:error({message:"no such field a",on:{b:null}})}),ne:error({message:"!=: error value encountered",on:error({message:"no such field a",on:{b:null}})})} diff --git a/runtime/ztests/expr/compare.yaml b/runtime/ztests/expr/compare.yaml index f5022a546b..7a004fcf82 100644 --- a/runtime/ztests/expr/compare.yaml +++ b/runtime/ztests/expr/compare.yaml @@ -36,7 +36,7 @@ output: | [false,true,false,false,false,false] [false,true,false,false,false,false] [null,null,null,null,null,null] - [error(0),error(0),error(0),error(0),error(0),error(0)] + [error({message:"==: error value encountered",on:error(0)}),error({message:"!=: error value encountered",on:error(0)}),error({message:"<: error value encountered",on:error(0)}),error({message:"<=: error value encountered",on:error(0)}),error({message:">=: error value encountered",on:error(0)}),error({message:">: error value encountered",on:error(0)})] --- @@ -60,7 +60,7 @@ output: | [false,true,false,false,false,false] [false,true,false,false,false,false] [null,null,null,null,null,null] - [error(0),error(0),error(0),error(0),error(0),error(0)] + [error({message:"==: error value encountered",on:error(0)}),error({message:"!=: error value encountered",on:error(0)}),error({message:"<: error value encountered",on:error(0)}),error({message:"<=: error value encountered",on:error(0)}),error({message:">=: error value encountered",on:error(0)}),error({message:">: error value encountered",on:error(0)})] --- @@ -84,7 +84,7 @@ output: | [false,true,false,false,false,false] [false,true,false,false,false,false] [null,null,null,null,null,null] - [error(0),error(0),error(0),error(0),error(0),error(0)] + [error({message:"==: error value encountered",on:error(0)}),error({message:"!=: error value encountered",on:error(0)}),error({message:"<: error value encountered",on:error(0)}),error({message:"<=: error value encountered",on:error(0)}),error({message:">=: error value encountered",on:error(0)}),error({message:">: error value encountered",on:error(0)})] --- @@ -110,7 +110,7 @@ output: | [false,true,false,false,false,false] [false,true,false,false,false,false] [null,null,null,null,null,null] - [error(0),error(0),error(0),error(0),error(0),error(0)] + [error({message:"==: error value encountered",on:error(0)}),error({message:"!=: error value encountered",on:error(0)}),error({message:"<: error value encountered",on:error(0)}),error({message:"<=: error value encountered",on:error(0)}),error({message:">=: error value encountered",on:error(0)}),error({message:">: error value encountered",on:error(0)})] --- @@ -130,4 +130,4 @@ output: | [false,true,false,false,false,false] [false,true,false,false,false,false] [null,null,null,null,null,null] - [error(0),error(0),error(0),error(0),error(0),error(0)] + [error({message:"==: error value encountered",on:error(0)}),error({message:"!=: error value encountered",on:error(0)}),error({message:"<: error value encountered",on:error(0)}),error({message:"<=: error value encountered",on:error(0)}),error({message:">=: error value encountered",on:error(0)}),error({message:">: error value encountered",on:error(0)})] diff --git a/runtime/ztests/expr/complex-record-math.yaml b/runtime/ztests/expr/complex-record-math.yaml index 5cfe0bca65..a3b00a12cd 100644 --- a/runtime/ztests/expr/complex-record-math.yaml +++ b/runtime/ztests/expr/complex-record-math.yaml @@ -10,6 +10,6 @@ input: | output: | {r:{x:3,y:1},a1:[2],a2:[3,1]} {r:{x:7,y:1},a1:[4],a2:[7,1]} - {r:{x:error("missing"),y:error("missing")},a1:["hello"],a2:[error("missing"),error("missing")]} + {r:{x:error({message:"+: error value encountered",on:error({message:"no such field b",on:{a:"hello"}})}),y:error({message:"-: error value encountered",on:error({message:"no such field b",on:{a:"hello"}})})},a1:["hello"],a2:[error({message:"+: error value encountered",on:error({message:"no such field b",on:{a:"hello"}})}),error({message:"-: error value encountered",on:error({message:"no such field b",on:{a:"hello"}})})]} {r:{x:11,y:1},a1:[6],a2:[11,1]} {r:{x:3,y:1},a1:[2]::[int64|null],a2:[3,1]} diff --git a/runtime/ztests/expr/cut.yaml b/runtime/ztests/expr/cut.yaml index ef76cfadfc..1522a021cc 100644 --- a/runtime/ztests/expr/cut.yaml +++ b/runtime/ztests/expr/cut.yaml @@ -12,8 +12,8 @@ input: | output: | {x:1::int32,s:"a",v:{s:"a",x:1::int32}} {x:2::int32,s:"b",v:{s:"b",x:2::int32}} - {s:"x",v:{s:"x",x:error("missing")}} - {s:"b",v:{s:"b",x:error("missing")}} - {none:"bad",v:{s:error("missing"),x:error("missing")}} + {s:"x",v:{s:"x",x:error({message:"no such field x",on:{s:"x"}})}} + {s:"b",v:{s:"b",x:error({message:"no such field x",on:{s:"b"}})}} + {none:"bad",v:{s:error({message:"no such field s",on:{none:"bad"}}),x:error({message:"no such field x",on:{none:"bad"}})}} {x:1::int32,s:"a",v:{s:"a",x:1::int32}} {x:3::int32,s:"e",v:{s:"e",x:3::int32}} diff --git a/runtime/ztests/expr/dot-record-type.yaml b/runtime/ztests/expr/dot-record-type.yaml index b655b6ef56..0a56b53b0c 100644 --- a/runtime/ztests/expr/dot-record-type.yaml +++ b/runtime/ztests/expr/dot-record-type.yaml @@ -10,8 +10,8 @@ output: | <{bar:int64}> error("missing") - error("missing") + error({message:"'.': applied to non-record",on:error("missing")}) error("missing") - error("missing") - error("missing") + error({message:"'.': applied to non-record",on:null}) + error({message:"'.': applied to non-record",on:error({message:"'.': applied to non-record",on:null})}) diff --git a/runtime/ztests/expr/dot.yaml b/runtime/ztests/expr/dot.yaml index d713ada386..120b6f65a6 100644 --- a/runtime/ztests/expr/dot.yaml +++ b/runtime/ztests/expr/dot.yaml @@ -18,11 +18,11 @@ output: | 1 1 1 - error("missing") + error({message:"'.': applied to non-record",on:null}) none - error("missing") - error("missing") - error("missing") + error({message:"'.': applied to non-record",on:1}) + error({message:"'.': applied to non-record",on:error({message:"no such field a",on:{}})}) + error({message:"'.': applied to non-record",on:error({message:"'.': applied to non-record",on:null})}) --- diff --git a/runtime/ztests/expr/filter-null-with-nonexistent-field.yaml b/runtime/ztests/expr/filter-null-with-nonexistent-field.yaml index 5e468a57fd..c006e3320a 100644 --- a/runtime/ztests/expr/filter-null-with-nonexistent-field.yaml +++ b/runtime/ztests/expr/filter-null-with-nonexistent-field.yaml @@ -1,4 +1,4 @@ -spq: not has(t) +spq: is_error(t) input: &input | {s:"A=B"} diff --git a/runtime/ztests/expr/function/compare.yaml b/runtime/ztests/expr/function/compare.yaml index 534cdf6825..eb99132f5f 100644 --- a/runtime/ztests/expr/function/compare.yaml +++ b/runtime/ztests/expr/function/compare.yaml @@ -17,8 +17,8 @@ output: | 0 1 1 - error("missing") - error("missing") + error({message:"no such field b",on:{a:0}}) + error({message:"no such field a",on:{b:0}}) --- diff --git a/runtime/ztests/expr/function/has-2.yaml b/runtime/ztests/expr/function/has-2.yaml deleted file mode 100644 index a6554fb8f6..0000000000 --- a/runtime/ztests/expr/function/has-2.yaml +++ /dev/null @@ -1,17 +0,0 @@ -spq: | - cut r:=has(r), - rb:=has(r.b), - both:=has(r,r.b), - div0:=has(s/0), - add1:=has(s+1), - s:=has(s), - array:=has(array[1]), - arraynot:=has(array[4]) - -input: | - {r:{a:1::int32},s:123::int32,array:[1,2,3]} - {r:{a:1::int8,b:2::int8}} - -output: | - {r:true,rb:false,both:false,div0:error("divide by zero"),add1:true,s:true,array:true,arraynot:false} - {r:true,rb:true,both:true,div0:false,add1:false,s:false,array:false,arraynot:false} diff --git a/runtime/ztests/expr/function/has.yaml b/runtime/ztests/expr/function/has.yaml deleted file mode 100644 index 6353cfd860..0000000000 --- a/runtime/ztests/expr/function/has.yaml +++ /dev/null @@ -1,21 +0,0 @@ -spq: values has(a,b) - -input: | - {a:1} - {b:1} - {a:1,b:2} - {a:1,b:null} - {a:null,b:2} - {a:null::(null|int64),b:null::(null|int64)} - {a:error("other"),b:2} - {a:1,b?:none::int64} - -output: | - false - false - true - null - null - null - error("other") - false diff --git a/runtime/ztests/expr/function/log.yaml b/runtime/ztests/expr/function/log.yaml index f3d0510d24..576508803b 100644 --- a/runtime/ztests/expr/function/log.yaml +++ b/runtime/ztests/expr/function/log.yaml @@ -16,4 +16,4 @@ output: | error({message:"log: illegal argument",on:-1}) null null - error("missing") + error({message:"log: error value encountered",on:error("missing")}) diff --git a/runtime/ztests/expr/function/missing.yaml b/runtime/ztests/expr/function/missing.yaml deleted file mode 100644 index b6b2b83fb0..0000000000 --- a/runtime/ztests/expr/function/missing.yaml +++ /dev/null @@ -1,23 +0,0 @@ -spq: values missing(a,b) - -input: | - {a:"foo",b:"bar"} - {a:null,b:"bar"} - {a:"foo",b:null} - {b:null} - {a:null} - {a:null::(null|int64),b:null::(null|int64)} - {a:"foo",b:error("other")} - {b:error("other")} - {a:"foo",b?:none::string} - -output: | - false - null - null - null - null - null - error("other") - true - true diff --git a/runtime/ztests/expr/function/quiet.yaml b/runtime/ztests/expr/function/ok.yaml similarity index 78% rename from runtime/ztests/expr/function/quiet.yaml rename to runtime/ztests/expr/function/ok.yaml index ecd2f29ffa..ac86671be8 100644 --- a/runtime/ztests/expr/function/quiet.yaml +++ b/runtime/ztests/expr/function/ok.yaml @@ -1,4 +1,4 @@ -spq: values quiet(this) +spq: values this.ok() input: | 1 @@ -28,7 +28,7 @@ output: | --- -spq: values quiet(x) +spq: values x.ok() input: | {x:1} @@ -40,7 +40,7 @@ output: | --- -spq: cut a:=quiet(x),b:=x +spq: cut a:=x.ok(),b:=x input: | {x:1} @@ -48,4 +48,4 @@ input: | output: | {a:1,b:1} - {b:error("missing")} + {b:error({message:"no such field x",on:{y:1}})} diff --git a/runtime/ztests/expr/function/split.yaml b/runtime/ztests/expr/function/split.yaml index 553d6bd7ec..d1a86ea1c2 100644 --- a/runtime/ztests/expr/function/split.yaml +++ b/runtime/ztests/expr/function/split.yaml @@ -18,4 +18,4 @@ output: | null null error({message:"split: string arg required",on:1.}) - error("missing") + error({message:"split: error value encountered",on:error({message:"no such field s",on:{sep:""}})}) diff --git a/runtime/ztests/expr/in.yaml b/runtime/ztests/expr/in.yaml index c465898b16..948d314e60 100644 --- a/runtime/ztests/expr/in.yaml +++ b/runtime/ztests/expr/in.yaml @@ -15,7 +15,7 @@ input: | 1::(int64|string) {a:[0,1]::[int64|string]} null - error(0) + [error({message:"'in' operator: error value encountered",on:error(0)}),error({message:"'in' operator: error value encountered",on:error(0)}),error({message:"'in' operator: error value encountered",on:error(0)})] [error(0),error(1)] [error(0),null] @@ -33,7 +33,7 @@ output: | [true,false,null] [true,false,null] [null,null,null] - [error(0),error(0),null] + [false,false,null] [true,false,null] [null,null,null] @@ -54,7 +54,7 @@ output: | true null null - error(0) + error({message:"'in' operator: error value encountered",on:error(0)}) --- @@ -104,8 +104,8 @@ input: | {a:map{"a":"hello"}} output: | - {b1:true,b2:error("missing")} - {b1:true,b2:error("missing")} - {b1:false,b2:error("missing")} - {b1:error("missing"),b2:error("missing")} - {b1:error("missing"),b2:true} + {b1:true,b2:error({message:"'in' operator: error value encountered",on:error({message:"no such field a",on:{x:"a"}})})} + {b1:true,b2:error({message:"'in' operator: error value encountered",on:error({message:"no such field a",on:{x:1}})})} + {b1:false,b2:error({message:"'in' operator: error value encountered",on:error({message:"no such field a",on:{x:10}})})} + {b1:error({message:"'in' operator: error value encountered",on:error({message:"no such field x",on:{y:1}})}),b2:error({message:"'in' operator: error value encountered",on:error({message:"no such field a",on:{y:1}})})} + {b1:error({message:"'in' operator: error value encountered",on:error({message:"no such field x",on:{a:map{"a":"hello"}}})}),b2:true} diff --git a/runtime/ztests/expr/index.yaml b/runtime/ztests/expr/index.yaml index 2245e61764..44acea3f59 100644 --- a/runtime/ztests/expr/index.yaml +++ b/runtime/ztests/expr/index.yaml @@ -101,9 +101,11 @@ input: | output: | 2 - error("missing") + type foo=[int64] + error({message:"'.': applied to non-record",on:[1,2,3]::foo}) 5 - error("missing") + type bar=set[int64] + error({message:"'.': applied to non-record",on:set[4,5,6]::bar}) "foo" error("missing") error("missing") @@ -118,6 +120,8 @@ input: | {val:fusion(2::(int64|[int64|{r:int64}]),)} {val:fusion([1,{r:3}]::(int64|[int64|{r:int64}]),<[int64|{r:int64}]>)} +# XXX fix this should not be error missing... should be structured + output: | - error("missing") + error({message:"'.': applied to non-record",on:error("missing")}) 3 diff --git a/runtime/ztests/expr/is-null.yaml b/runtime/ztests/expr/is-null.yaml index a6509b9011..67e94b8355 100644 --- a/runtime/ztests/expr/is-null.yaml +++ b/runtime/ztests/expr/is-null.yaml @@ -17,5 +17,5 @@ output: | [true,false] [true,false] [true,false] - [error("missing"),error("missing")] - [error("foo"),error("foo")] + [error("missing"),error({message:"'not' operator: error value encountered",on:error("missing")})] + [error("foo"),error({message:"'not' operator: error value encountered",on:error("foo")})] diff --git a/runtime/ztests/expr/logical-and.yaml b/runtime/ztests/expr/logical-and.yaml index 2b2c760d78..c2a26742b6 100644 --- a/runtime/ztests/expr/logical-and.yaml +++ b/runtime/ztests/expr/logical-and.yaml @@ -44,7 +44,7 @@ output: | true false null - error("missing") + error({message:"no such field b",on:{a:true}}) error({message:"not type bool",on:"foo"}) "=== FALSE ===" false @@ -56,14 +56,14 @@ output: | null false null - error("missing") + error({message:"no such field b",on:{a:null}}) error({message:"not type bool",on:"foo"}) "=== MISSING ===" - error("missing") + error({message:"no such field a",on:{b:true}}) false - error("missing") - error("missing") - error("missing") + error({message:"no such field a",on:{b:null}}) + error({message:"no such field a",on:{}}) + error({message:"no such field a",on:{b:"foo"}}) "=== ERROR ===" error({message:"not type bool",on:"foo"}) false diff --git a/runtime/ztests/expr/logical-not.yaml b/runtime/ztests/expr/logical-not.yaml index 75f5fda708..f73313bf81 100644 --- a/runtime/ztests/expr/logical-not.yaml +++ b/runtime/ztests/expr/logical-not.yaml @@ -15,8 +15,8 @@ output: | null false null - error("missing") - error("foo") + error({message:"'not' operator: error value encountered",on:error("missing")}) + error({message:"'not' operator: error value encountered",on:error("foo")}) --- diff --git a/runtime/ztests/expr/logical-or.yaml b/runtime/ztests/expr/logical-or.yaml index f57b544eca..1f1ff7908c 100644 --- a/runtime/ztests/expr/logical-or.yaml +++ b/runtime/ztests/expr/logical-or.yaml @@ -50,7 +50,7 @@ output: | true false null - error("missing") + error({message:"no such field b",on:{a:false}}) error({message:"not type bool",on:"foo"}) "=== NULL ===" true @@ -60,10 +60,10 @@ output: | null "=== MISSING ===" true - error("missing") + error({message:"no such field a",on:{b:false}}) null - error("missing") - error("missing") + error({message:"no such field a",on:{}}) + error({message:"no such field a",on:{b:"foo"}}) "=== ERROR ===" true error({message:"not type bool",on:"foo"}) diff --git a/runtime/ztests/expr/optional-fields.yaml b/runtime/ztests/expr/optional-fields.yaml index c21dc0eb34..6c07f48395 100644 --- a/runtime/ztests/expr/optional-fields.yaml +++ b/runtime/ztests/expr/optional-fields.yaml @@ -81,7 +81,7 @@ output: | 2 3 "foo" - error("missing") + none --- @@ -162,3 +162,39 @@ input: | output: | 1 none + +--- + +spq: values this?.x + +input: | + none + none::int64 + {x:1} + {x?:1} + +output: | + none + none + 1 + 1::(int64|none) + +--- + +spq: values this?.x?.y + +input: | + none + none::int64 + {x:1} + {x?:1} + {x:{y?:1}} + {x:{y?:none::int64}} + +output: | + none + none + error({message:"'?.': applied to non-record",on:1}) + error({message:"'?.': applied to non-record",on:1}) + 1::(int64|none) + none::(int64|none) diff --git a/runtime/ztests/expr/record-spread.yaml b/runtime/ztests/expr/record-spread.yaml index 225b971665..846a0ddfb0 100644 --- a/runtime/ztests/expr/record-spread.yaml +++ b/runtime/ztests/expr/record-spread.yaml @@ -13,13 +13,13 @@ input: | null output: | - {b:error("missing"),c:2} + {b:error({message:"'.': applied to non-record",on:123}),c:2} {x:2,b:3,c:2} - {x:1,b:error("missing"),c:2} - {b:error("missing"),c:2} + {x:1,b:error({message:"no such field b",on:{a:1,r:{x:1}}}),c:2} + {b:error({message:"no such field b",on:{a:1}}),c:2} {x:3,b:3,c:0} {x:3,c:2,b:3} {x:3,c:2,b:3,d:1} {y:true,z:false,b:3,c:2} {x:1,b:3,c:2} - {b:error("missing"),c:2} + {b:error({message:"'.': applied to non-record",on:null}),c:2} diff --git a/runtime/ztests/expr/slice.yaml b/runtime/ztests/expr/slice.yaml index e415d19971..962cd8ab37 100644 --- a/runtime/ztests/expr/slice.yaml +++ b/runtime/ztests/expr/slice.yaml @@ -17,10 +17,10 @@ output: | {a1:0x,a2:0x,a3:0x,a4:0x,a5:0x,a6:0x,a7:0x,a8:null} {a1:"",a2:"",a3:"",a4:"",a5:"",a6:"",a7:"",a8:null} {a1:[]::[int32],a2:[]::[int32],a3:[]::[int32],a4:[]::[int32],a5:[]::[int32],a6:[]::[int32],a7:[]::[int32],a8:null} - {a1:0x1122,a2:0x112233,a3:0x00,a4:0x001122,a5:0x,a6:0x33,a7:0x22,a8:error("missing")} - {a1:"12",a2:"123",a3:"0",a4:"012",a5:"",a6:"3",a7:"2",a8:error("missing")} - {a1:"ⁱ⁲",a2:"ⁱ⁲3",a3:"0",a4:"0ⁱ⁲",a5:"",a6:"3",a7:"⁲",a8:error("missing")} - {a1:"ⁱ⁲",a2:"ⁱ⁲⁳",a3:"⁰",a4:"⁰ⁱ⁲",a5:"",a6:"⁳",a7:"⁲",a8:error("missing")} + {a1:0x1122,a2:0x112233,a3:0x00,a4:0x001122,a5:0x,a6:0x33,a7:0x22,a8:error({message:"slice expression: error value encountered",on:error({message:"-: error value encountered",on:error("missing")})})} + {a1:"12",a2:"123",a3:"0",a4:"012",a5:"",a6:"3",a7:"2",a8:error({message:"slice expression: error value encountered",on:error({message:"-: error value encountered",on:error("missing")})})} + {a1:"ⁱ⁲",a2:"ⁱ⁲3",a3:"0",a4:"0ⁱ⁲",a5:"",a6:"3",a7:"⁲",a8:error({message:"slice expression: error value encountered",on:error({message:"-: error value encountered",on:error("missing")})})} + {a1:"ⁱ⁲",a2:"ⁱ⁲⁳",a3:"⁰",a4:"⁰ⁱ⁲",a5:"",a6:"⁳",a7:"⁲",a8:error({message:"slice expression: error value encountered",on:error({message:"-: error value encountered",on:error("missing")})})} {a1:[11::int32,12::int32],a2:[11::int32,12::int32,13::int32],a3:[10::int32],a4:[10::int32,11::int32,12::int32],a5:[]::[int32],a6:[13::int32],a7:[12::int32],a8:[10::int32,11::int32]} {a1:set[11::int32,12::int32],a2:set[11::int32,12::int32,13::int32],a3:set[10::int32],a4:set[10::int32,11::int32,12::int32],a5:set[]::set[int32],a6:set[13::int32],a7:set[12::int32],a8:set[10::int32,11::int32]} diff --git a/runtime/ztests/expr/type-map.yaml b/runtime/ztests/expr/type-map.yaml index c29fdfb2c9..4941639740 100644 --- a/runtime/ztests/expr/type-map.yaml +++ b/runtime/ztests/expr/type-map.yaml @@ -5,9 +5,9 @@ spq: | "conn": conn, "dns": dns } - switch missing(schemas[_path]) - case false ( cut schema:=schemas[_path] ) - case true ( put _UNCLASSIFIED:=true ) + switch schemas[_path].is_ok() + case true ( cut schema:=schemas[_path] ) + case false ( put _UNCLASSIFIED:=true ) | sort this input: | diff --git a/runtime/ztests/expr/unary-minus.yaml b/runtime/ztests/expr/unary-minus.yaml index f10135140d..ce525eeea1 100644 --- a/runtime/ztests/expr/unary-minus.yaml +++ b/runtime/ztests/expr/unary-minus.yaml @@ -55,9 +55,9 @@ output: | error({message:"type incompatible with unary '-' operator",on:"foo"}) error({message:"type incompatible with unary '-' operator",on:10.0.0.1}) error({message:"type incompatible with unary '-' operator",on:[1,2,3]}) - error(1) - error(1) + error({message:"unary '-': error value encountered",on:error(1)}) + error({message:"unary '-': error value encountered",on:error(1)}) null null -1 - error(1) + error({message:"unary '-': error value encountered",on:error(1)}) diff --git a/runtime/ztests/op/aggregate/missing-fields.yaml b/runtime/ztests/op/aggregate/missing-fields.yaml index 41be3f7973..5e12ffc283 100644 --- a/runtime/ztests/op/aggregate/missing-fields.yaml +++ b/runtime/ztests/op/aggregate/missing-fields.yaml @@ -1,4 +1,4 @@ -spq: count() by key1 | sort key1 +spq: count() by key1.ok() | sort key1 input: | {key1:"a",key2:"x",n:1::int32} @@ -10,4 +10,4 @@ input: | output: | {key1:"a",count:2} {key1:"b",count:1} - {key1:error("missing"),count:2} + {key1:none,count:2} diff --git a/runtime/ztests/op/aggregate/missing.yaml b/runtime/ztests/op/aggregate/missing.yaml index a36bc89ca8..4a05e40142 100644 --- a/runtime/ztests/op/aggregate/missing.yaml +++ b/runtime/ztests/op/aggregate/missing.yaml @@ -9,4 +9,4 @@ input: | output: | {a:0,b:1} {a:error("missing"),b:1} - {a:error("missing"),b:error("missing")} + {a:error("missing"),b:error({message:"'.': applied to non-record",on:"b"})} diff --git a/runtime/ztests/op/assert.yaml b/runtime/ztests/op/assert.yaml index f0f2e67103..2ee8040c6c 100644 --- a/runtime/ztests/op/assert.yaml +++ b/runtime/ztests/op/assert.yaml @@ -11,5 +11,5 @@ output: | {a:1} {a:1::(int64|null)} error({message:"assertion failed",expr:"a==1",on:{a:2}}) - error("missing") + error({message:"==: error value encountered",on:error({message:"'.': applied to non-record",on:1})}) error({message:"assertion failed",expr:"a==1",on:{a:2::(int64|null)}}) diff --git a/runtime/ztests/op/cut-empty-record.yaml b/runtime/ztests/op/cut-empty-record.yaml index 605fefc7bf..eec851b636 100644 --- a/runtime/ztests/op/cut-empty-record.yaml +++ b/runtime/ztests/op/cut-empty-record.yaml @@ -6,6 +6,6 @@ input: | {a:{},b:{}} output: | - {a:error("missing"),b:error("missing")} - {a:{},b:error("missing")} + {a:error({message:"no such field a",on:{}}),b:error({message:"no such field b",on:{}})} + {a:{},b:error({message:"no such field b",on:{a:{}}})} {a:{},b:{}} diff --git a/runtime/ztests/op/cut-foo-bar-only.yaml b/runtime/ztests/op/cut-foo-bar-only.yaml index 6f718852e2..e0ab762b5a 100644 --- a/runtime/ztests/op/cut-foo-bar-only.yaml +++ b/runtime/ztests/op/cut-foo-bar-only.yaml @@ -6,6 +6,6 @@ input: | {bar:"bar3"} output: | - {foo:error("missing")} - {foo:error("missing")} - {foo:error("missing")} + {foo:error({message:"no such field foo",on:{bar:"bar1"}})} + {foo:error({message:"no such field foo",on:{bar:"bar2"}})} + {foo:error({message:"no such field foo",on:{bar:"bar3"}})} diff --git a/runtime/ztests/op/cut-foo-mixed.yaml b/runtime/ztests/op/cut-foo-mixed.yaml index 3176014139..3910c28cef 100644 --- a/runtime/ztests/op/cut-foo-mixed.yaml +++ b/runtime/ztests/op/cut-foo-mixed.yaml @@ -12,12 +12,12 @@ input: | {bar:"bar3"} output: | - {foo:error("missing")} - {foo:error("missing")} - {foo:error("missing")} + {foo:error({message:"no such field foo",on:{bar:"bar1"}})} + {foo:error({message:"no such field foo",on:{bar:"bar2"}})} + {foo:error({message:"no such field foo",on:{bar:"bar3"}})} {foo:"foo1"} {foo:"foo2"} {foo:"foo3"} - {foo:error("missing")} - {foo:error("missing")} - {foo:error("missing")} + {foo:error({message:"no such field foo",on:{bar:"bar1"}})} + {foo:error({message:"no such field foo",on:{bar:"bar2"}})} + {foo:error({message:"no such field foo",on:{bar:"bar3"}})} diff --git a/runtime/ztests/op/cut-none.yaml b/runtime/ztests/op/cut-none.yaml index e8d2655c09..db2325fa61 100644 --- a/runtime/ztests/op/cut-none.yaml +++ b/runtime/ztests/op/cut-none.yaml @@ -6,6 +6,6 @@ input: | {x?:2,z?:3} output: | - {z:error("missing"),q:1} + {z:error({message:"no such field z",on:{}}),q:1} {z:3,q:1} {z:3,q:1} diff --git a/runtime/ztests/op/cut.yaml b/runtime/ztests/op/cut.yaml index d94d0f0d74..9e4bcef0b0 100644 --- a/runtime/ztests/op/cut.yaml +++ b/runtime/ztests/op/cut.yaml @@ -13,5 +13,5 @@ output: | {foo:2} {foo:1::(int64|null)} {foo:null::(int64|null)} - {foo:error("missing")} - {foo:error("missing")} + {foo:error({message:"no such field foo",on:{bar:"three"}})} + {foo:error({message:"'.': applied to non-record",on:null})} diff --git a/runtime/ztests/op/join-empty-inner.yaml b/runtime/ztests/op/join-empty-inner.yaml index a70be204c5..bc0e947205 100644 --- a/runtime/ztests/op/join-empty-inner.yaml +++ b/runtime/ztests/op/join-empty-inner.yaml @@ -1,8 +1,8 @@ script: | echo === hash join - super -dynamic -s -c 'left join (from C.sup) on left.a=right.a | values {...left,hit:right.sc} | sort' A.sup + super -dynamic -s -c 'left join (from C.sup) on left.a=right.a | values {...left,hit:this?.right?.sc ?? error("missing")} | sort' A.sup echo === nested loop join - super -dynamic -s -c 'left join (from C.sup) on left.a t1.bsup super -c "tail 2" in.sup > t2.bsup - super -s -c "where has(proto) | count() by proto" t1.bsup t2.bsup + super -s -c "where proto.is_ok() | count() by proto" t1.bsup t2.bsup inputs: - name: in.sup diff --git a/vector/union.go b/vector/union.go index 5b37d9d05e..e406f9ace0 100644 --- a/vector/union.go +++ b/vector/union.go @@ -294,22 +294,22 @@ func noneLength(runlens []uint32) uint32 { return noneLen } -func DeoptionWithMissing(sctx *super.Context, vec Any) Any { +func DeoptionWithNone(sctx *super.Context, vec Any) Any { switch vec := Super(vec).(type) { case *None: - return NewMissing(sctx, vec.Len()) + return vec case *Dynamic: if hasOptionTypesOrNones(vec.Values) { vecs := make([]Any, 0, len(vec.Values)) for _, v := range vec.Values { - vecs = append(vecs, DeoptionWithMissing(sctx, v)) + vecs = append(vecs, DeoptionWithNone(sctx, v)) } return stitch(vec.Tags, vecs) } case *Union: if super.IsOptionType(vec.Typ) { out := Deunion(vec) - out = DeoptionWithMissing(sctx, out) + out = DeoptionWithNone(sctx, out) return out } }