fix(spec): lower equality triples with a $field comparand to {$eq: ref} (#7597)

.changeset/equality-field-reference-lowering.md

@@ -0,0 +1,47 @@
+---
+"@objectstack/spec": minor
+"@objectstack/driver-sql": minor
+---
+
+fix(spec): lower equality triples with a `$field` comparand to `{ $eq: ref }` (#7597)
+
+`parseFilterAST` lowered one authored intent two different ways depending only on
+how the operator was spelled:
+
+| authored | lowered to | what it did |
+| :--- | :--- | :--- |
+| `['amount', '>', { $field: 'budget' }]` | `{ amount: { $gt: { $field: 'budget' } } }` | worked on both evaluation paths |
+| `['amount', '=', { $field: 'budget' }]` | `{ amount: { $field: 'budget' } }` | matched **nothing**, silently |
+
+The four equality spellings (`=`, `==`, `equals`, `eq`) dropped the operator,
+because a LITERAL comparand's implicit-equality form is `{ field: value }` —
+correct for a literal, and for a field reference it produces a field spec whose
+only key is `$field`. Every consumer reads an all-`$` key set as an OPERATOR
+SPEC, and nothing implements an operator named `$field`: the in-memory evaluator
+(`@objectstack/formula`) dispatches it to its operator switch, finds no arm, and
+returns the fail-closed `false` — so the filter matched no record on the very
+path that produced it, with no error anywhere. On SQL push-down the same shape
+arrived as an unknown operator and was refused.
+
+An equality triple whose comparand is a `FieldReferenceSchema` now lowers to the
+explicit `{ field: { $eq: ref } }` — the spelling both evaluation paths already
+implement (the memory evaluator resolves the reference; `driver-sql` compiles it
+to a column-to-column comparison, #5222). `['amount', '=', ref]` and
+`['amount', '>', ref]` are now the same kind of thing.
+
+Unchanged, deliberately:
+
+- **Literal comparands.** `['amount', '=', 5]` still lowers to `{ amount: 5 }`.
+  The fix branches on the comparand being a field reference, never on the
+  operator, and a `$field` carrying a non-string is not a field reference on any
+  path — it keeps the literal lowering too.
+- **The in-memory evaluator's unknown-operator posture.** #6520 examined it and
+  kept it; a hand-authored bare `{ amount: { $field: 'budget' } }`
+  `FilterCondition` keeps exactly its current fate on every backend — fail-closed
+  `false` in memory, and `driver-sql`'s actionable refusal naming `$eq` (#5222).
+  Only what the ARRAY sugar produces has changed.
+
+`@objectstack/driver-sql` gains `CROSS_FIELD_AUTHORED_CASES` — the conformance
+corpus's new AUTHORING arm, entering through the lowering sink instead of at the
+already-lowered object, run by both SQL drivers' cross-field suites. Its only
+other change is documentation.

packages/drivers/driver-sql/src/cross-field-conformance-cases.ts

@@ -236,6 +236,96 @@ export const CROSS_FIELD_CASES: readonly CrossFieldCase[] = [
   },
 ] as const;
 
+/**
+ * [#7597] The AUTHORING arm: the same conformance obligation, entered through
+ * the array sugar a caller actually writes rather than through the lowered
+ * `FilterCondition` object.
+ *
+ * ## Why the corpus needed a second entrance
+ *
+ * Every case above is a hand-written `FilterCondition`. That is the shape a
+ * DRIVER sees, and it is not the shape anyone AUTHORS: the ObjectUI client, the
+ * `FilterBuilder` and every stored view carry the array triple
+ * `['amount', '=', { $field: 'budget' }]`, which `parseFilterAST`
+ * (`@objectstack/spec`, the single lowering sink per #5158) turns into one of
+ * the objects above. A corpus that only enters at the object skips that sink —
+ * and the sink is exactly where #7597 was: the four EQUALITY spellings dropped
+ * the operator, because implicit equality (`{ field: comparand }`) is right for
+ * a literal and produces `{ amount: { $field: 'budget' } }` for a reference —
+ * a field spec whose only key is `$field`, which no backend reads as an
+ * equality. `['amount', '>', ref]` kept its operator and worked; `['amount',
+ * '=', ref]` silently matched nothing. One intent, two spellings, two fates.
+ *
+ * So these cases assert TWO things per row, and the pair is the point:
+ * `loweredTo` pins what the sink produces (a lowering regression fails here,
+ * in the conformance suite, rather than in a spec unit test nobody reads
+ * beside the driver), and `expected` holds the lowered filter to the same
+ * both-paths-same-rows rule as every case above.
+ *
+ * The `>` control rides along deliberately: it is the spelling that ALWAYS
+ * worked, so a run where the equality rows pass and the control fails means
+ * the harness moved, not the fix.
+ */
+export interface CrossFieldAuthoredCase {
+  name: string;
+  /** The authored filter ARRAY, exactly as a client sends it. */
+  authored: unknown;
+  /** What `parseFilterAST` must lower it to. */
+  loweredTo: unknown;
+  /** Ids of matching rows, ascending — for the LOWERED filter, on both paths. */
+  expected: string[];
+  note?: string;
+}
+
+/**
+ * The four `$eq` spellings `AST_OPERATOR_MAP` carries (`=`, `==`, `equals`,
+ * `eq`), which is the whole set the sink folds into implicit equality — all
+ * four were bare before #7597, so all four are pinned.
+ */
+const EQUALITY_SPELLINGS: readonly string[] = ['=', '==', 'equals', 'eq'];
+
+export const CROSS_FIELD_AUTHORED_CASES: readonly CrossFieldAuthoredCase[] = [
+  // ── The equality spellings, on each storage class ────────────────────────
+  //
+  // Replicated across the three class pairs for the same reason the object
+  // cases are: the lowering is class-blind, so a class-dependent answer here
+  // would be a driver fact showing up in an authoring test.
+  ...CLASS_PAIRS.flatMap(({ label, target, ref }) =>
+    EQUALITY_SPELLINGS.map((op) => ({
+      name: `['${target}', '${op}', { $field: '${ref}' }] on the ${label} pair`,
+      authored: [target, op, { $field: ref }],
+      loweredTo: { [target]: { $eq: { $field: ref } } },
+      expected: ['3', '6'],
+      note: 'The `$eq` row set of the object corpus above — row 3 (equal) and row 6 (both NULL, which the memory evaluator matches and the emitted SQL is written TOTAL to match too).',
+    })),
+  ),
+
+  // ── The control: the spelling that never lost its operator ───────────────
+  {
+    name: "['amount', '>', { $field: 'budget' }] still lowers to $gt",
+    authored: ['amount', '>', { $field: 'budget' }],
+    loweredTo: { amount: { $gt: { $field: 'budget' } } },
+    expected: ['1'],
+    note: 'Untouched by #7597 and asserted anyway: if this moves, the harness moved rather than the lowering.',
+  },
+
+  // ── The sugar's own structures, carrying a reference leaf ────────────────
+  {
+    name: 'a legacy flat array ANDs an equality reference with a literal',
+    authored: [['amount', '=', { $field: 'budget' }], ['stage', '=', 'mid']],
+    loweredTo: { $and: [{ amount: { $eq: { $field: 'budget' } } }, { stage: 'mid' }] },
+    expected: ['3'],
+    note: 'Row 6 drops out on the literal conjunct — which also pins that the LITERAL comparand keeps its implicit-equality lowering (`{ stage: "mid" }`, not `{ stage: { $eq: "mid" } }`). The fix branches on the comparand, not on the operator.',
+  },
+  {
+    name: 'an explicit `or` node carries an equality reference branch',
+    authored: ['or', ['amount', '=', { $field: 'budget' }], ['stage', '=', 'lost']],
+    loweredTo: { $or: [{ amount: { $eq: { $field: 'budget' } } }, { stage: 'lost' }] },
+    expected: ['2', '3', '6'],
+    note: 'The lowering is applied at the comparison leaf, so nesting cannot route around it.',
+  },
+] as const;
+
 /**
  * The refusal arm — the boundary of v1, and the half of this issue that is a
  * SECURITY surface rather than a capability one.

packages/drivers/driver-sql/src/index.ts

@@ -37,12 +37,14 @@ export type {
 // which is the argument `@objectstack/spec/data` makes for exporting its own
 // conformance corpora. Test-only DATA — no runtime path in this package reads it.
 export {
+  CROSS_FIELD_AUTHORED_CASES,
   CROSS_FIELD_CASES,
   CROSS_FIELD_OBJECT_FIELDS,
   CROSS_FIELD_REFUSALS,
   CROSS_FIELD_ROWS,
 } from './cross-field-conformance-cases.js';
 export type {
+  CrossFieldAuthoredCase,
   CrossFieldCase,
   CrossFieldRefusalCase,
   CrossFieldRow,

packages/drivers/driver-sql/src/sql-driver-cross-field-conformance.test.ts

@@ -54,10 +54,11 @@
 
 import { describe, it, expect, beforeAll, afterAll } from 'vitest';
 import { matchesFilterCondition } from '@objectstack/formula';
-import type { FilterCondition } from '@objectstack/spec/data';
+import { parseFilterAST, type FilterCondition } from '@objectstack/spec/data';
 import { SqlDriver } from './index.js';
 import { DIALECT_CELLS, declareUnprovisionedCell, type DialectCell } from './live-dialect-matrix.testkit.js';
 import {
+  CROSS_FIELD_AUTHORED_CASES,
   CROSS_FIELD_CASES,
   CROSS_FIELD_OBJECT_FIELDS,
   CROSS_FIELD_REFUSALS,
@@ -136,6 +137,27 @@ describe(`[#5222] driver-sql — cross-field \`$field\` push-down conformance ($
     });
   }
 
+  describe('[#7597] the AUTHORING arm — the array sugar a client actually sends', () => {
+    // The sink these cases enter through (`parseFilterAST`) is where #7597
+    // was: the four equality spellings dropped the operator on a `{ $field }`
+    // comparand and produced a field spec no backend reads as an equality, so
+    // `['amount', '=', ref]` silently matched nothing while `['amount', '>',
+    // ref]` worked. Lowering and row set are asserted together because either
+    // one alone can be right while the pair is wrong.
+    for (const authoredCase of CROSS_FIELD_AUTHORED_CASES) {
+      it(`${authoredCase.name} — lowers as declared, same rows on both paths`, async () => {
+        const note = authoredCase.note ? `\n${authoredCase.note}` : '';
+        const lowered = parseFilterAST(authoredCase.authored);
+        expect(lowered, `parseFilterAST lowered the authored array to an unexpected shape${note}`)
+          .toEqual(authoredCase.loweredTo);
+
+        const expected = [...authoredCase.expected].sort();
+        expect(memoryIds(lowered), `in-memory evaluator disagreed${note}`).toEqual(expected);
+        expect(await sqlIds(lowered), `SQL push-down disagreed${note}`).toEqual(expected);
+      });
+    }
+  });
+
   describe('the refusal arm — narrowed, never removed (ADR-0112 envelope)', () => {
     for (const refusal of CROSS_FIELD_REFUSALS) {
       it(`${refusal.name} → 400 INVALID_FILTER`, async () => {

packages/drivers/driver-sql/src/sql-driver-cross-field-reference.test.ts

@@ -227,17 +227,36 @@ describe('[#5222] SqlDriver `$field` position matrix — compiled vs refused', (
     });
 
     it('the bare `{ field: { $field } }` spelling names the operator form to use', async () => {
-      // What `parseFilterAST(['amount', '=', { $field: 'budget' }])` lowers to.
-      // Refused because the in-memory evaluator answers `false` for it rather
-      // than reading it as an equality — compiling it would open a divergence
-      // in the change that closes one — so the message points at `$eq`.
-      const lowered = parseFilterAST([['amount', '=', { $field: 'budget' }]] as any);
-      const err = await refusalOf(() => find(lowered));
+      // HAND-AUTHORED, and that spelling matters (#7597). This used to be
+      // derived from `parseFilterAST(['amount', '=', ref])`, because the sink
+      // dropped the operator on an equality triple and produced exactly this
+      // shape — the defect #7597 fixed. The sink now lowers that triple to
+      // `{ $eq: ref }` (pinned in the conformance suite's authoring arm), so
+      // the bare form no longer has an authoring route into the driver.
+      //
+      // The refusal itself is UNCHANGED and stays pinned here on the shape a
+      // caller can still write by hand: the in-memory evaluator answers
+      // `false` for it rather than reading it as an equality (#6520's
+      // unknown-operator posture, deliberately kept), so compiling it would
+      // open a divergence — and the message points at the `$eq` spelling that
+      // does compile.
+      const err = await refusalOf(() => find({ amount: { $field: 'budget' } }));
       expect(err.code).toBe('INVALID_FILTER');
       expect(err.status).toBe(400);
       expect(err.message).toContain('$eq');
       expect(err.message).toContain('budget');
     });
+
+    it('the equality TRIPLE no longer lowers to that bare spelling (#7597)', async () => {
+      // The other half of the case above, and the reason it had to change:
+      // the authoring route that used to reach the bare form now reaches the
+      // compiled one. Asserted here — beside the refusal it replaced — so a
+      // regression that restores the bare lowering fails next to the pin
+      // whose comment explains it, not only in the conformance sweep.
+      const lowered = parseFilterAST([['amount', '=', { $field: 'budget' }]] as any);
+      expect(lowered).toEqual({ amount: { $eq: { $field: 'budget' } } });
+      await expect(find(lowered)).resolves.toBeDefined();
+    });
   });
 
   // ── The general arm #5041 installed, untouched by the narrowing ──────────

packages/drivers/driver-sql/src/sql-driver.ts

@@ -1026,22 +1026,28 @@ function crossFieldComparisonError(field: string, op: string, ref: string, index
  * field spec, i.e. in the implicit-equality position where a literal comparand
  * would mean `field = value`.
  *
- * It is not a hypothetical spelling: `parseFilterAST` lowers the authored
- * triple `['amount', '=', { $field: 'budget' }]` (and its `equals` word form)
- * to exactly this shape, while `['amount', '>', …]` lowers to `{ $gt: … }`.
- * So one authoring dialect produces both a supported and an unsupported
- * spelling of the same intent, and the caller cannot see why from the generic
- * "unsupported operator" message this used to fall through to.
- *
- * Refused rather than compiled to `$eq`, deliberately, and the reason is the
- * conformance rule the rest of this capability is held to: the in-memory
- * evaluator does NOT read this shape as an equality. `matches-filter.ts`
- * `evalField` sees an all-`$` key set and dispatches `$field` to `evalOp`,
- * which has no arm for it and answers `false` (its fail-closed default). So
- * compiling a column-to-column equality here would make SQL answer rows for a
- * filter the memory path answers `false` for — a NEW divergence, in the same
- * change that closes one. The two paths must move together, which is a spec
- * question rather than a driver one; filed separately.
+ * ## [#7597] It no longer has an AUTHORING route — and is still refused
+ *
+ * `parseFilterAST` used to lower the authored triple
+ * `['amount', '=', { $field: 'budget' }]` (and its `==` / `equals` / `eq`
+ * spellings) to exactly this shape, while `['amount', '>', …]` kept its
+ * operator and lowered to `{ $gt: … }` — one authoring dialect producing both
+ * a supported and an unsupported spelling of the same intent, with the
+ * unsupported one silent on the path that produced it. #7597 fixed the sink:
+ * an equality triple whose comparand is a `FieldReferenceSchema` now lowers to
+ * `{ $eq: ref }`, which this driver compiles. The array sugar therefore cannot
+ * reach this error any more.
+ *
+ * What CAN still reach it is a hand-authored `FilterCondition` carrying the
+ * bare form, and that keeps being refused rather than compiled to `$eq`,
+ * deliberately: the in-memory evaluator does NOT read this shape as an
+ * equality. `matches-filter.ts` `evalField` sees an all-`$` key set and
+ * dispatches `$field` to `evalOp`, which has no arm for it and answers `false`
+ * (its fail-closed default). Compiling a column-to-column equality here would
+ * make SQL answer rows for a filter the memory path answers `false` for — a
+ * NEW divergence. #7597 ruled that the evaluator's unknown-operator posture
+ * stays as #6520 left it and moved the LOWERING instead, so this message keeps
+ * pointing at the `$eq` spelling that does compile.
  */
 function bareFieldReferenceError(field: string, ref: string): Error {
   return unsupportedFilterError(

packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-cross-field-conformance.test.ts

@@ -29,8 +29,9 @@
 
 import { describe, it, expect, beforeAll, afterAll } from 'vitest';
 import { matchesFilterCondition } from '@objectstack/formula';
-import type { FilterCondition } from '@objectstack/spec/data';
+import { parseFilterAST, type FilterCondition } from '@objectstack/spec/data';
 import {
+  CROSS_FIELD_AUTHORED_CASES,
   CROSS_FIELD_CASES,
   CROSS_FIELD_OBJECT_FIELDS,
   CROSS_FIELD_REFUSALS,
@@ -92,6 +93,25 @@ describe('[#5222] driver-sqlite-wasm — cross-field `$field` push-down conforma
     });
   }
 
+  describe('[#7597] the AUTHORING arm — the array sugar a client actually sends', () => {
+    // Run here as well as on `driver-sql` for the reason the whole corpus is
+    // shared: this driver inherits that compiler but executes through its own
+    // sql.js dialect, and the lowered `$eq` reference has to mean the same
+    // rows on both. See the corpus header for the defect it pins.
+    for (const authoredCase of CROSS_FIELD_AUTHORED_CASES) {
+      it(`${authoredCase.name} — lowers as declared, same rows on both paths`, async () => {
+        const note = authoredCase.note ? `\n${authoredCase.note}` : '';
+        const lowered = parseFilterAST(authoredCase.authored);
+        expect(lowered, `parseFilterAST lowered the authored array to an unexpected shape${note}`)
+          .toEqual(authoredCase.loweredTo);
+
+        const expected = [...authoredCase.expected].sort();
+        expect(memoryIds(lowered), `in-memory evaluator disagreed${note}`).toEqual(expected);
+        expect(await sqlIds(lowered), `wasm push-down disagreed${note}`).toEqual(expected);
+      });
+    }
+  });
+
   describe('the refusal arm — narrowed, never removed (ADR-0112 envelope)', () => {
     for (const refusal of CROSS_FIELD_REFUSALS) {
       it(`${refusal.name} → 400 INVALID_FILTER`, async () => {