diff --git a/.changeset/20444-empty-operator-engine-arms.md b/.changeset/20444-empty-operator-engine-arms.md new file mode 100644 index 00000000000..6229e228fc3 --- /dev/null +++ b/.changeset/20444-empty-operator-engine-arms.md @@ -0,0 +1,24 @@ +--- +'@objectstack/driver-sql': minor +'@objectstack/driver-turso': minor +'@objectstack/driver-memory': minor +'@objectstack/driver-mongodb': minor +'@objectstack/formula': minor +'@objectstack/objectql': minor +'@objectstack/spec': minor +--- + +feat(drivers,formula,objectql): the engine's filter faces answer the staged `$empty` operator (#20444) + +Clause-②: yes (widening) + +`$empty: true | false` is declared by `@objectstack/spec` (`FieldOperatorsSchema`) with a per-type meaning: a text-like field is empty when it is null or `''`, a multi-value field (multiselect, checkboxes, tags, or a select / radio / lookup / user / file / image with `multiple: true`) when it is null or `[]`, and every other type only when it is null. `$empty: false` is the exact complement. Until now every face in this list refused it (`INVALID_FILTER` / 400), except `matchesFilterCondition`, which answered `false` for every record. **A driver or evaluator called directly now answers it:** + +- **By the field's declared type**, through the spec's one expansion (`expandEmptyOperator`): `driver-sql`'s filter compiler (and so `driver-sqlite-wasm` and `driver-turso`'s local transport, which inherit it), `driver-turso`'s remote transport, `driver-memory`'s query path (`find` / `count` / `update` / `delete`) and `driver-mongodb`'s `translateFilter` (its `find`, its aggregate `$match`). The declaration is the one each driver already receives — `initObjects` / `registerObjectMetadata` / `registerExternalObject` on the SQL family, `syncSchema` on the others. On SQL a multi-value field's empty list is tested as stored JSON per dialect (SQLite `json_array_length` behind a `json_valid` guard, PostgreSQL a `jsonb` comparison, MySQL `JSON_LENGTH`), never as an equality comparand. +- **By value** — null, a missing value, `''` and `[]` are empty (`isEmptyFilterValue`) — on the faces that read no field declaration: `@objectstack/formula`'s `matchesFilterCondition` (the RLS write-side `check`), `driver-memory`'s reference matcher, and `@objectstack/objectql`'s `having` and per-aggregation `filter`. In `having`, a `count` or `sum` holding `0` is not empty. + +**Refused, never guessed** (`INVALID_FILTER` / 400): `$empty` on a field whose declaration the driver does not hold (a table built outside its registration, a builtin column such as `id`, a field with no `type`, or `translateFilter` / `RemoteTransport` used standalone without a declaration), a multi-value field on a SQL dialect the driver does not model, and a flag that is not a boolean. `driver-memory`'s analytics (cube) face refuses `$empty` as an operator it cannot compile, as it does `$null`. + +New optional API: `translateFilter(where, temporalKind?, valueShape?)` in `@objectstack/driver-mongodb` takes a declared-value-shape resolver (type `ValueShapeResolver`), and `buildAggregationPipeline` a `valueShape` option; `RemoteTransport.setDeclaredValueShapeResolver` in `@objectstack/driver-turso`, which `TursoDriver` wires. `@objectstack/spec`'s shared `FILTER_LOGIC_CASES` table gains seven `$empty` cases: a backend that runs it answers `$empty` or goes red, and its harness must declare the fixture's columns. + +`$empty` stays staged: it is not in `FILTER_OPERATORS`, so the engine's front door still refuses it until the flip card adds it, and the view operators `is_empty` / `is_not_empty` still lower to `$null`. diff --git a/packages/drivers/driver-memory/src/filter-refusal.ts b/packages/drivers/driver-memory/src/filter-refusal.ts index ad59e3e54cd..9527a8ceda4 100644 --- a/packages/drivers/driver-memory/src/filter-refusal.ts +++ b/packages/drivers/driver-memory/src/filter-refusal.ts @@ -364,6 +364,13 @@ export const SUPPORTED_FIELD_OPERATORS: ReadonlySet = new Set([ ...FILTER_OPERATORS, '$like', '$ilike', + // [#20444] The staged emptiness flag, admitted BY HAND for the reason the + // `$like` paragraph above gives, and under its ordering rule: both arms land + // with this entry — the reference matcher judges the stored value + // (`isEmptyFilterValue`, the spec's reading for a face holding no field + // declaration) and the live query path the field's DECLARED row + // (`expandEmptyOperator`, from the declaration `syncSchema` recorded). + '$empty', ]); /** The vocabulary as it appears in a refusal message, in declaration order. */ @@ -611,6 +618,42 @@ export function nonBooleanNullComparandError(field: string, value: unknown, path ); } +/** + * [#20444] A non-boolean `$empty` comparand. The leading sentence is + * `driver-sql`'s `nonBooleanEmptyComparandError`, verbatim — one condition, + * one wording (#5240). + */ +export function nonBooleanEmptyComparandError(field: string, value: unknown, path: string): Error { + return unsupportedFilterError( + `Operator "$empty" on field "${field}" requires a boolean comparand (true or false). ` + + `Received ${describeFilterOperand(value)} (${safeShapePreview(value)}) at ${path}. ` + + `@objectstack/spec FieldOperatorsSchema declares $empty as a boolean: true asks for the ` + + `empty rows, false for their exact complement.`, + ); +} + +/** + * [#20444] `$empty` on the live query path, aimed at a field this driver holds + * no declaration for — an object never passed through `syncSchema`, a field + * its schema does not name, or one declared with no `type`. + * + * What counts as empty is the field's DECLARED row of the ruled table, and the + * live path reads it from the declaration rather than from a value, so without + * one there is no answer to give: refused, never guessed. The reference matcher + * (`memory-matcher.ts`) is the face that holds NO declarations at all, and it + * judges the stored value instead — the spec's reading for such a face. + */ +export function undeclaredEmptyOperatorFieldError(field: string, path: string): Error { + return unsupportedFilterError( + `Operator "$empty" on field "${field}" at ${path} targets a field whose declaration this ` + + `driver does not hold (no declared type — the object's schema was never synced, or does not ` + + `declare the field). What counts as empty is the field's DECLARED row of the ruled table — ` + + `null or '' for a text-like type, null or [] for a multi-value field, null only for every ` + + `other type — so the operator is refused rather than guessed. Declare the field, or use ` + + `"$null" for "has no value".`, + ); +} + /** * [#5702] A RETIRED filter operator in a field constraint. * @@ -895,6 +938,13 @@ function assertFieldConstraintShape( if (op === '$null' && typeof spec[op] !== 'boolean') { throw nonBooleanNullComparandError(field, spec[op], `${path}.$null`); } + // [#20444] `$empty`'s comparand is a boolean by the same declaration + // (`FieldOperatorsSchema`), refused on this walk for the same reason: both + // faces of this package evaluate `true` / `false` exhaustively, so a third + // value would land on whichever side each arm happens to default to. + if (op === '$empty' && typeof spec[op] !== 'boolean') { + throw nonBooleanEmptyComparandError(field, spec[op], `${path}.$empty`); + } // [#16810] An ARRAY comparand on a single-value comparison — the operator // spelling of the implicit-equality position refused at the top of this // function, and the same cell `@objectstack/spec`'s comparand door leaves diff --git a/packages/drivers/driver-memory/src/memory-20444-empty-operator.test.ts b/packages/drivers/driver-memory/src/memory-20444-empty-operator.test.ts new file mode 100644 index 00000000000..d409a025717 --- /dev/null +++ b/packages/drivers/driver-memory/src/memory-20444-empty-operator.test.ts @@ -0,0 +1,166 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20444] The staged `$empty` operator on this package's three filter faces. + * + * - **The live query path** (`InMemoryDriver.find` → mingo) holds the field + * declarations `syncSchema` recorded, so it answers by the field's DECLARED + * row of the ruled 「is empty」 table (ruling A on #20399, record 5865693155; + * the spec's `expandEmptyOperator`): text-like = null or `''`; multi-value = + * null or `[]`; every other type = null only; `$empty: false` the exact + * complement. A field it holds no declaration for is REFUSED. + * - **The reference matcher** (`match`) holds no declarations at all, so it + * takes the reading the spec gives such a face — by value + * (`isEmptyFilterValue`): null, a missing key, `''` and `[]` are empty. + * - **The analytics (cube) face** does not lower the flag (nor `$null`), and + * refuses it loudly as a declared operator it cannot compile. + * + * The two value-level faces agree on every value a field's own type can hold; + * the one place they part is a stored state the declaration does not predict, + * and that cell is pinned below so the divergence is a measurement, not a + * surprise. + */ + +import { beforeAll, describe, expect, it } from 'vitest'; +import type { Cube, FilterCondition } from '@objectstack/spec/data'; +import { InMemoryDriver } from './memory-driver.js'; +import { match } from './memory-matcher.js'; +import { MemoryAnalyticsService } from './memory-analytics.js'; + +const TABLE = 'os20444_empty'; + +const FIELDS = { + id: { type: 'text' }, + title: { type: 'text' }, + tags: { type: 'tags' }, + owners: { type: 'lookup', reference: TABLE, multiple: true }, + score: { type: 'number' }, +}; + +const ROWS: Array> = [ + { id: 'r1', title: 'x', tags: ['a'], owners: ['u1'], score: 5 }, + { id: 'r2', title: '', tags: [], owners: [], score: 0 }, + { id: 'r3', title: null, tags: null, owners: null, score: null }, + { id: 'r4', title: ' ', tags: ['a', 'b'], owners: ['u2'], score: -1 }, + // Every column MISSING — the other reading of "no value". + { id: 'r5' }, +]; + +const CASES: Array<{ where: FilterCondition; expected: string[] }> = [ + { where: { title: { $empty: true } }, expected: ['r2', 'r3', 'r5'] }, + { where: { title: { $empty: false } }, expected: ['r1', 'r4'] }, + { where: { tags: { $empty: true } }, expected: ['r2', 'r3', 'r5'] }, + { where: { tags: { $empty: false } }, expected: ['r1', 'r4'] }, + { where: { owners: { $empty: true } }, expected: ['r2', 'r3', 'r5'] }, + { where: { owners: { $empty: false } }, expected: ['r1', 'r4'] }, + { where: { score: { $empty: true } }, expected: ['r3', 'r5'] }, + { where: { score: { $empty: false } }, expected: ['r1', 'r2', 'r4'] }, + { where: { $not: { title: { $empty: true } } }, expected: ['r1', 'r4'] }, + { where: { $not: { tags: { $empty: false } } }, expected: ['r2', 'r3', 'r5'] }, + { where: { $or: [{ score: 5 }, { tags: { $empty: true } }] }, expected: ['r1', 'r2', 'r3', 'r5'] }, + { where: { $and: [{ title: { $empty: false } }, { owners: { $empty: false } }] }, expected: ['r1', 'r4'] }, + { where: { title: { $empty: false, $ne: 'x' } }, expected: ['r4'] }, + { where: { tags: { $empty: true, $ne: null } }, expected: ['r2'] }, +]; + +function refusal(run: () => unknown): Promise<{ code?: string; status?: number } | 'answered'> { + return Promise.resolve() + .then(run) + .then( + () => 'answered' as const, + (err) => ({ code: (err as { code?: string }).code, status: (err as { status?: number }).status }), + ); +} + +describe('[#20444] InMemoryDriver — $empty on the live path, the reference matcher and the analytics face', () => { + let driver: InMemoryDriver; + const ids = async (where: FilterCondition) => + ((await driver.find(TABLE, { where })) as Array>).map((r) => String(r.id)).sort(); + const reference = (where: FilterCondition) => + ROWS.filter((r) => match(r, where)).map((r) => String(r.id)).sort(); + + beforeAll(async () => { + driver = new InMemoryDriver({ persistence: false }); + await driver.connect(); + await driver.syncSchema(TABLE, { fields: FIELDS }); + for (const row of ROWS) await driver.create(TABLE, { ...row }); + }); + + it('the fixture is the five rows', async () => { + expect(await ids({})).toEqual(['r1', 'r2', 'r3', 'r4', 'r5']); + }); + + for (const c of CASES) { + it(`${JSON.stringify(c.where)} → ${JSON.stringify(c.expected)} on the live path AND the reference matcher`, async () => { + expect(await ids(c.where), 'live').toEqual(c.expected); + expect(reference(c.where), 'reference matcher').toEqual(c.expected); + }); + } + + it('$empty: false partitions every declared field with $empty: true, on both faces', async () => { + for (const field of ['title', 'tags', 'owners', 'score']) { + const empty = await ids({ [field]: { $empty: true } }); + const full = await ids({ [field]: { $empty: false } }); + expect([...empty, ...full].sort(), field).toEqual(['r1', 'r2', 'r3', 'r4', 'r5']); + expect(reference({ [field]: { $empty: true } }), field).toEqual(empty); + } + }); + + it('the live path REFUSES a field it holds no declaration for; the matcher judges the value', async () => { + expect(await refusal(() => driver.find(TABLE, { where: { nope: { $empty: true } } }))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + expect(await refusal(() => driver.find('never_synced', { where: { title: { $empty: true } } }))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + expect(reference({ nope: { $empty: true } })).toEqual(['r1', 'r2', 'r3', 'r4', 'r5']); + }); + + it('the one cell where the declared row and the by-value reading part: a stored state the type cannot hold', async () => { + // A number column holding '' is a write-door defect, never a value the + // declaration predicts. The declared null-only row does not count it; the + // declaration-free matcher does. Pinned so the divergence is known. + const odd = new InMemoryDriver({ persistence: false }); + await odd.connect(); + await odd.syncSchema('odd', { fields: { id: { type: 'text' }, score: { type: 'number' } } }); + const rows = [{ id: 'blank', score: '' }]; + for (const row of rows) await odd.create('odd', { ...row }); + const live = ((await odd.find('odd', { where: { score: { $empty: true } } })) as Array>) + .map((r) => r.id); + expect(live).toEqual([]); + expect(rows.filter((r) => match(r, { score: { $empty: true } })).map((r) => r.id)).toEqual(['blank']); + }); + + it('a non-boolean flag is refused by both faces, on the shared shape gate', async () => { + expect(await refusal(() => driver.find(TABLE, { where: { title: { $empty: 'yes' as never } } }))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + expect(await refusal(() => match(ROWS[0], { title: { $empty: 1 as never } }))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + // Even where an identity would settle the node before any arm ran. + expect(await refusal(() => match(ROWS[0], { $or: [{}, { title: { $empty: 'no' as never } }] }))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + }); + + it('the analytics (cube) face refuses $empty as a declared operator it cannot compile — never drops it', async () => { + const cube: Cube = { + name: TABLE, + title: 'empty', + sql: TABLE, + measures: { count: { name: 'count', label: 'Rows', type: 'count', sql: 'id' } }, + dimensions: { + id: { name: 'id', label: 'id', type: 'string', sql: 'id' }, + title: { name: 'title', label: 'title', type: 'string', sql: 'title' }, + }, + public: true, + } as Cube; + const service = new MemoryAnalyticsService({ driver, cubes: [cube] }); + expect( + await refusal(() => + service.query({ + cube: TABLE, + measures: [`${TABLE}.count`], + dimensions: [`${TABLE}.id`], + where: { title: { $empty: true } }, + } as never), + ), + ).toEqual({ code: 'INVALID_FILTER', status: 400 }); + }); +}); diff --git a/packages/drivers/driver-memory/src/memory-driver.ts b/packages/drivers/driver-memory/src/memory-driver.ts index 0b109dd9038..f4b9c5120d2 100644 --- a/packages/drivers/driver-memory/src/memory-driver.ts +++ b/packages/drivers/driver-memory/src/memory-driver.ts @@ -10,6 +10,9 @@ import { canonicalAstOperator, asciiCaseInsensitiveRegexSource } from '@objectst // the same translation `formula` evaluates and the same pattern `driver-sql` // hands to LIKE/GLOB, so this face cannot answer a pattern differently. import { hasDanglingLikeEscape, hasNulInLikePattern, likePatternToRegExp } from '@objectstack/spec/data'; +// [#20444] The `$empty` operator's ONE expansion — the field's declared row of +// the ruled 「is empty」 table, asked of the spec by the live query path. +import { expandEmptyOperator, type ValueShapeFieldDef } from '@objectstack/spec/data'; import type { DriverQuery, IDataDriver } from '@objectstack/spec/contracts'; import { Logger, createLogger, nextUtcCalendarDay } from '@objectstack/core'; import { Query, Aggregator } from 'mingo'; @@ -26,6 +29,9 @@ import { filterNodeListExpectedError, malformedBetweenError, nonBooleanNullComparandError, + // [#20444] The `$empty` refusals — a non-boolean flag, an undeclared field. + nonBooleanEmptyComparandError, + undeclaredEmptyOperatorFieldError, unknownFieldOperatorError, unknownLogicalOperatorError, unsupportedFilterError, @@ -301,6 +307,22 @@ interface MemoryTransaction { snapshot: Record; } +/** + * [#20444] Each field's declared value shape — the `type` and `multiple` slice + * `expandEmptyOperator` reads — for the fields that declare a `type`. The RAW + * `type` is read: a field with none has no row of the ruled 「is empty」 table, + * and inventing one here would answer `$empty` with a row nobody declared. + */ +function indexValueShapes(fields: Record | undefined): Map { + const out = new Map(); + for (const [name, def] of Object.entries(fields ?? {})) { + const type = (def as { type?: unknown } | null | undefined)?.type; + if (typeof type !== 'string' || type === '') continue; + out.set(name, { type, multiple: (def as { multiple?: unknown }).multiple === true }); + } + return out; +} + /** * In-Memory Driver for ObjectStack * @@ -397,6 +419,17 @@ export class InMemoryDriver implements IDataDriver { */ private temporalFields: Map> = new Map(); + /** + * [#20444] Each declared field's value shape — its `type` and `multiple`, the + * slice the spec's `expandEmptyOperator` reads — per object, populated by + * {@link syncSchema} beside {@link temporalFields} and with its lifetime. The + * live query path answers `$empty` by the field's DECLARED row from it + * ({@link emptyOperatorCondition}); a field it does not hold (an object never + * synced, a field its schema does not name, one declared with no `type`) is + * refused rather than answered by a row read off the data. + */ + private valueShapes: Map> = new Map(); + /** * [#13197, #13239] Declared unique constraints per object, populated by * {@link syncSchema} — both declaration surfaces in one list (field-level @@ -1507,7 +1540,19 @@ export class InMemoryDriver implements IDataDriver { result[key] = value; continue; } - const normalized = this.normalizeFieldOperators(value, this.temporalKind(object, key), key, here); + // [#20444] `$empty` lowers to a condition of its OWN, AND-ed beside the + // field's other operators rather than written into their operator map: + // its multi-value row is an OR over two tests, which no single mingo + // field operator spells, and a separate conjunct can never contest a + // lowered key with a sibling operator (the #13524 clobber class). + let fieldOps: Record = value; + if (Object.prototype.hasOwnProperty.call(value, '$empty')) { + const { $empty: flag, ...rest } = value as Record; + extraAndConditions.push(this.emptyOperatorCondition(object, key, flag, `${here}.$empty`)); + if (Object.keys(rest).length === 0) continue; + fieldOps = rest; + } + const normalized = this.normalizeFieldOperators(fieldOps, this.temporalKind(object, key), key, here); // [#13524] Lowered writes whose mingo key was already taken by a // sibling operator on the same field. They cannot be merged without one // of the two constraints silently overwriting the other, so each @@ -1768,6 +1813,52 @@ export class InMemoryDriver implements IDataDriver { return result; } + /** + * [#20444] Lower `{ field: { $empty: true | false } }` to a mingo condition + * by the field's DECLARED row of the ruled 「is empty」 table — ruling B on + * #20311 (record 5861435168), spelled as this operator by ruling A on #20399 + * (record 5865693155) — through the spec's one expansion, + * `expandEmptyOperator`, against the declaration {@link syncSchema} recorded: + * + * | declared row | `$empty: true` | `$empty: false` — the exact complement | + * |---|---|---| + * | `null_only` | `{ f: { $eq: null } }` | `{ f: { $ne: null } }` | + * | `text` | `{ f: { $in: [null, ''] } }` | `{ f: { $nin: [null, ''] } }` | + * | `multi_value` | `{ $or: [{ f: { $eq: null } }, { f: { $size: 0 } }] }` | the same pair under `$nor` | + * + * `null` in a mingo equality matches a missing key as well as a stored null, + * so both readings of "no value" are empty on every row — the answer the + * `$null` arm already gives. The empty list is tested with `$size: 0`, never + * as an equality comparand: measured on mingo 7.2, `$in: [null, []]` does NOT + * match a stored `[]` (mingo intersects an array value with the list), and + * `$eq: []` also matches an array holding an empty array. + * + * The flag's boolean shape was settled by `assertFilterConditionShape` before + * this ran; the re-check is the totality floor a translator owes itself. + */ + private emptyOperatorCondition(object: string | undefined, field: string, flag: unknown, path: string): Record { + if (typeof flag !== 'boolean') throw nonBooleanEmptyComparandError(field, flag, path); + const shape = object ? this.valueShapes.get(object)?.get(field) : undefined; + if (!shape) throw undeclaredEmptyOperatorFieldError(field, path); + const expansion = expandEmptyOperator(shape); + switch (expansion.arm) { + case 'null_only': + return { [field]: flag ? { $eq: null } : { $ne: null } }; + case 'text': + return { [field]: flag ? { $in: [null, ''] } : { $nin: [null, ''] } }; + case 'multi_value': { + const branches = [{ [field]: { $eq: null } }, { [field]: { $size: 0 } }]; + return flag ? { $or: branches } : { $nor: branches }; + } + default: { + // A closed union of three rows; a fourth is a spec change this driver + // was not taught, and it must fail loudly rather than answer for it. + const unknownArm: never = expansion.arm; + throw unsupportedFilterError(`No $empty arm for the declared row ${JSON.stringify(unknownArm)}.`); + } + } + } + /** * Escape special regex characters for safe literal matching. */ @@ -2000,6 +2091,9 @@ export class InMemoryDriver implements IDataDriver { // (ADR-0053 D-B3) and, like it, is idempotent. const kinds = indexTemporalFields(schema?.fields); this.temporalFields.set(object, kinds); + // [#20444] …and each field's declared value shape, in the same pass, for + // the `$empty` operator's declared row. + this.valueShapes.set(object, indexValueShapes(schema?.fields)); // [#13197, #13239] Learn the object's unique constraints in the same pass — // BOTH declaration surfaces `driver-sql` materializes uniqueness from: // field-level `unique` and object-level `indexes[]` entries carrying diff --git a/packages/drivers/driver-memory/src/memory-matcher.ts b/packages/drivers/driver-memory/src/memory-matcher.ts index a09ace0602e..ffd206ffdfb 100644 --- a/packages/drivers/driver-memory/src/memory-matcher.ts +++ b/packages/drivers/driver-memory/src/memory-matcher.ts @@ -26,6 +26,10 @@ // definition — shared with this package's query path, `formula`, and the SQL // family's emitters. import { reduceFilterVerdict, asciiCaseInsensitiveContains, matchesLikePattern } from '@objectstack/spec/data'; +// [#20444] `$empty`'s value-level half, the spec's one definition. This matcher +// holds no field declarations, so it takes the reading the spec gives such a +// face: null, `undefined`, `''` and `[]` are empty. +import { isEmptyFilterValue } from '@objectstack/spec/data'; import { assertFilterConditionShape } from './filter-refusal.js'; @@ -457,8 +461,10 @@ function checkCondition(value: any, condition: any): boolean { // over the VALUE would still reach arms whose no-value answer is ruled // elsewhere. Only the cells' STATE moved — refused at the door, rather // than held for a ruling. + // [#20444] `$empty` is about the absence too, so its arm answers the + // MISSING reading itself — `undefined` is empty, like `null`. if (value === undefined && op !== '$exists' && op !== '$null' && op !== '$eq' - && !noValueSatisfiesNegation(op)) { + && op !== '$empty' && !noValueSatisfiesNegation(op)) { return false; } @@ -657,6 +663,21 @@ function checkCondition(value: any, condition: any): boolean { if (target === true && value != null) return false; if (target === false && value == null) return false; break; + // [#20444] The staged emptiness flag — ruling A on #20399 (record + // 5865693155) gives a face holding NO field declarations the + // by-value reading, and this matcher holds none: null, `undefined`, + // `''` and `[]` are empty, through the spec's `isEmptyFilterValue` + // rather than a copy of it. `false` is the exact complement. The + // shape gate refused a non-boolean `target` before evaluation + // started, so the comparison is exhaustive. + // + // It differs from the live query path's DECLARED row only on a + // stored state the declaration does not predict — `''` in a + // non-text column (a write-door defect), `[]` in a scalar one — + // never on a value the field's own type can hold. + case '$empty': + if (isEmptyFilterValue(value) !== (target === true)) return false; + break; // [#5702] The `$regex` arm that stood here is GONE. It was the only // real regex evaluator in the repo, and the reason #4706 retired the // operator rather than standardising it: `new RegExp(target)` read diff --git a/packages/drivers/driver-memory/src/memory-operator-key-clobber.test.ts b/packages/drivers/driver-memory/src/memory-operator-key-clobber.test.ts index 4272d522b3c..80a4f9ef48c 100644 --- a/packages/drivers/driver-memory/src/memory-operator-key-clobber.test.ts +++ b/packages/drivers/driver-memory/src/memory-operator-key-clobber.test.ts @@ -111,6 +111,10 @@ beforeAll(async () => { sweepDriver = new InMemoryDriver({ persistence: false }); await sweepDriver.connect(); + // [#20444] `$empty` is answered by the field's DECLARED row and refused on an + // undeclared field, so the sweep declares `v` — a `text` field, which is not + // temporal, so no other operator's comparand changes form. + await sweepDriver.syncSchema('t', { fields: { v: { type: 'text' } } }); for (const row of SWEEP_ROWS) await sweepDriver.create('t', { ...row }); }); @@ -258,6 +262,9 @@ const SWEEP_COMPARANDS: Readonly> = Object.freeze({ $ilike: '%07-15', $null: false, $exists: true, + // [#20444] Lowered to a condition of its own beside the field's operator map, + // so it contests no key — the sweep proves it rather than assuming it. + $empty: false, }); describe('[#13524] the ENUMERATION — every declared operator, every pair, both orders', () => { diff --git a/packages/drivers/driver-mongodb/src/index.ts b/packages/drivers/driver-mongodb/src/index.ts index c074e4f6116..5af5f9ed271 100644 --- a/packages/drivers/driver-mongodb/src/index.ts +++ b/packages/drivers/driver-mongodb/src/index.ts @@ -5,6 +5,8 @@ import { MongoDBDriver } from './mongodb-driver.js'; export { MongoDBDriver }; export type { MongoDBDriverConfig } from './mongodb-driver.js'; export { translateFilter } from './mongodb-filter.js'; +// [#20444] `translateFilter`'s third parameter: the declared value shape `$empty` reads. +export type { ValueShapeResolver } from './mongodb-filter.js'; export { buildAggregationPipeline, postProcessAggregation, diff --git a/packages/drivers/driver-mongodb/src/mongodb-20444-empty-operator.test.ts b/packages/drivers/driver-mongodb/src/mongodb-20444-empty-operator.test.ts new file mode 100644 index 00000000000..7671417e8fe --- /dev/null +++ b/packages/drivers/driver-mongodb/src/mongodb-20444-empty-operator.test.ts @@ -0,0 +1,193 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20444] The staged `$empty` operator on `translateFilter`, translated by the + * field's DECLARED row of the ruled 「is empty」 table (ruling A on #20399, + * record 5865693155; the spec's `expandEmptyOperator`): + * + * | declared row | `$empty: true` | `$empty: false` | + * |---|---|---| + * | text-like | `{ f: { $in: [null, ''] } }` | `{ f: { $nin: [null, ''] } }` | + * | multi-value | `{ $or: [{ f: { $eq: null } }, { f: { $size: 0 } }] }` | the same pair under `$nor` | + * | every other type | `{ f: { $eq: null } }` | `{ f: { $ne: null } }` | + * + * Pinned twice: the emitted DOCUMENTS, and the rows they select under a + * server-free reading of the MongoDB semantics those documents use (null + * equality matches a missing field too; `$size` matches an array of that + * length and nothing else). A live `mongod` suite runs the same cases when the + * opt-in server is available. + */ + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import type { MongoMemoryServer } from 'mongodb-memory-server'; +import type { FilterCondition } from '@objectstack/spec/data'; +import { translateFilter, type ValueShapeResolver } from './mongodb-filter.js'; +import { buildAggregationPipeline } from './mongodb-aggregation.js'; +import { MongoDBDriver } from './mongodb-driver.js'; +import { createTestMongod } from './test-mongod.js'; + +const FIELDS: Record = { + title: { type: 'text' }, + tags: { type: 'tags' }, + owners: { type: 'lookup', reference: 'os20444_empty', multiple: true }, + score: { type: 'number' }, +}; +const SHAPES: ValueShapeResolver = (field) => FIELDS[field]; + +const ROWS: Array> = [ + { id: 'r1', title: 'x', tags: ['a'], owners: ['u1'], score: 5 }, + { id: 'r2', title: '', tags: [], owners: [], score: 0 }, + { id: 'r3', title: null, tags: null, owners: null, score: null }, + { id: 'r4', title: ' ', tags: ['a', 'b'], owners: ['u2'], score: -1 }, + { id: 'r5' }, +]; + +const CASES: Array<{ where: FilterCondition; expected: string[] }> = [ + { where: { title: { $empty: true } }, expected: ['r2', 'r3', 'r5'] }, + { where: { title: { $empty: false } }, expected: ['r1', 'r4'] }, + { where: { tags: { $empty: true } }, expected: ['r2', 'r3', 'r5'] }, + { where: { tags: { $empty: false } }, expected: ['r1', 'r4'] }, + { where: { owners: { $empty: true } }, expected: ['r2', 'r3', 'r5'] }, + { where: { owners: { $empty: false } }, expected: ['r1', 'r4'] }, + { where: { score: { $empty: true } }, expected: ['r3', 'r5'] }, + { where: { score: { $empty: false } }, expected: ['r1', 'r2', 'r4'] }, + { where: { $not: { title: { $empty: true } } }, expected: ['r1', 'r4'] }, + { where: { $not: { tags: { $empty: false } } }, expected: ['r2', 'r3', 'r5'] }, + { where: { $or: [{ score: 5 }, { tags: { $empty: true } }] }, expected: ['r1', 'r2', 'r3', 'r5'] }, + { where: { $and: [{ title: { $empty: false } }, { owners: { $empty: false } }] }, expected: ['r1', 'r4'] }, + { where: { title: { $empty: false, $ne: 'x' } }, expected: ['r4'] }, +]; + +// ── A server-free reading of the MongoDB vocabulary `$empty` lowers to ─────── + +/** MongoDB equality: a missing field equals null, and an array matches by element. */ +function mongoEquals(value: unknown, comparand: unknown): boolean { + if (comparand === null) { + return value === null || value === undefined || (Array.isArray(value) && value.includes(null)); + } + if (Array.isArray(value)) return value.some((element) => element === comparand); + return value === comparand; +} + +function matchOps(value: unknown, ops: Record): boolean { + for (const [op, arg] of Object.entries(ops)) { + switch (op) { + case '$eq': if (!mongoEquals(value, arg)) return false; break; + case '$ne': if (mongoEquals(value, arg)) return false; break; + case '$in': if (!(arg as unknown[]).some((member) => mongoEquals(value, member))) return false; break; + case '$nin': if ((arg as unknown[]).some((member) => mongoEquals(value, member))) return false; break; + case '$size': if (!Array.isArray(value) || value.length !== arg) return false; break; + default: throw new Error(`unmodelled field operator ${op}`); + } + } + return true; +} + +function matchDoc(row: Record, doc: Record): boolean { + for (const [key, value] of Object.entries(doc)) { + if (key === '$and') { if (!(value as Array>).every((d) => matchDoc(row, d))) return false; continue; } + if (key === '$or') { if (!(value as Array>).some((d) => matchDoc(row, d))) return false; continue; } + if (key === '$nor') { if ((value as Array>).some((d) => matchDoc(row, d))) return false; continue; } + if (key.startsWith('$')) throw new Error(`unmodelled document operator ${key}`); + const cond = value as Record; + const isOps = cond !== null && typeof cond === 'object' && !Array.isArray(cond) + && Object.keys(cond).every((k) => k.startsWith('$')); + if (isOps ? !matchOps(row[key], cond) : !mongoEquals(row[key], value)) return false; + } + return true; +} + +const select = (doc: Record) => ROWS.filter((r) => matchDoc(r, doc)).map((r) => String(r.id)).sort(); + +function refusal(run: () => unknown): { code?: string; status?: number } | 'answered' { + try { + run(); + } catch (err) { + return { code: (err as { code?: string }).code, status: (err as { status?: number }).status }; + } + return 'answered'; +} + +describe('[#20444] translateFilter — $empty by the declared row', () => { + it('emits the declared row per field kind, and its exact complement', () => { + expect(translateFilter({ title: { $empty: true } }, undefined, SHAPES)).toEqual({ title: { $in: [null, ''] } }); + expect(translateFilter({ title: { $empty: false } }, undefined, SHAPES)).toEqual({ title: { $nin: [null, ''] } }); + expect(translateFilter({ tags: { $empty: true } }, undefined, SHAPES)) + .toEqual({ $or: [{ tags: { $eq: null } }, { tags: { $size: 0 } }] }); + expect(translateFilter({ tags: { $empty: false } }, undefined, SHAPES)) + .toEqual({ $nor: [{ tags: { $eq: null } }, { tags: { $size: 0 } }] }); + expect(translateFilter({ score: { $empty: true } }, undefined, SHAPES)).toEqual({ score: { $eq: null } }); + expect(translateFilter({ score: { $empty: false } }, undefined, SHAPES)).toEqual({ score: { $ne: null } }); + }); + + it('never tests the empty list as an equality comparand', () => { + for (const field of ['tags', 'owners']) { + for (const flag of [true, false]) { + const doc = JSON.stringify(translateFilter({ [field]: { $empty: flag } }, undefined, SHAPES)); + expect(doc, doc).not.toContain('[]'); + } + } + }); + + it('beside a sibling operator on the same field, both constraints survive as separate conjuncts', () => { + expect(translateFilter({ title: { $empty: false, $ne: 'x' } }, undefined, SHAPES)) + .toEqual({ $and: [{ title: { $ne: 'x' } }, { title: { $nin: [null, ''] } }] }); + }); + + for (const c of CASES) { + it(`${JSON.stringify(c.where)} selects ${JSON.stringify(c.expected)}`, () => { + const doc = translateFilter(c.where, undefined, SHAPES) as Record; + expect(select(doc), JSON.stringify(doc)).toEqual(c.expected); + }); + } + + it('the aggregate $match translates it the way find() does', () => { + const pipeline = buildAggregationPipeline({ + where: { tags: { $empty: true } }, + aggregations: [{ function: 'count', alias: 'n' }] as never, + valueShape: SHAPES, + }); + expect(pipeline[0]).toEqual({ $match: { $or: [{ tags: { $eq: null } }, { tags: { $size: 0 } }] } }); + }); + + it('with no declaration it REFUSES — a standalone call, an unknown field, a type-less field', () => { + expect(refusal(() => translateFilter({ title: { $empty: true } }))).toEqual({ code: 'INVALID_FILTER', status: 400 }); + expect(refusal(() => translateFilter({ nope: { $empty: true } }, undefined, SHAPES))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + expect(refusal(() => translateFilter({ $or: [{ title: 'x' }, { nope: { $empty: false } }] }, undefined, SHAPES))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + }); + + it('a non-boolean flag is refused on the walk, even where an identity settles the node', () => { + expect(refusal(() => translateFilter({ title: { $empty: 'yes' as never } }, undefined, SHAPES))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + expect(refusal(() => translateFilter({ $or: [{}, { title: { $empty: 1 as never } }] }, undefined, SHAPES))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + }); +}); + +const sharedMongod: MongoMemoryServer | undefined = await createTestMongod('$empty operator'); + +describe.skipIf(!sharedMongod)('[#20444] MongoDBDriver — $empty against a live mongod', () => { + const mongod = sharedMongod as MongoMemoryServer; + let driver: MongoDBDriver; + + beforeAll(async () => { + driver = new MongoDBDriver({ url: mongod.getUri(), database: 'os20444_empty' }); + await driver.connect(); + await driver.syncSchema('os20444_empty', { name: 'os20444_empty', fields: FIELDS }); + for (const row of ROWS) await driver.create('os20444_empty', { ...row }); + }, 90_000); + + afterAll(async () => { + if (driver) await driver.disconnect(); + if (sharedMongod) await sharedMongod.stop(); + }); + + for (const c of CASES) { + it(`${JSON.stringify(c.where)} selects ${JSON.stringify(c.expected)}`, async () => { + const rows = (await driver.find('os20444_empty', { where: c.where })) as Array>; + expect(rows.map((r) => String(r.id)).sort()).toEqual(c.expected); + }); + } +}); diff --git a/packages/drivers/driver-mongodb/src/mongodb-aggregation.ts b/packages/drivers/driver-mongodb/src/mongodb-aggregation.ts index fc601da367b..362110478c2 100644 --- a/packages/drivers/driver-mongodb/src/mongodb-aggregation.ts +++ b/packages/drivers/driver-mongodb/src/mongodb-aggregation.ts @@ -11,7 +11,7 @@ import type { Document } from 'mongodb'; import { StandardErrorCode } from '@objectstack/spec/api'; import { AggregationFunction } from '@objectstack/spec/data'; import type { DateGranularityValue, GroupByNode } from '@objectstack/spec/data'; -import { translateFilter } from './mongodb-filter.js'; +import { translateFilter, type ValueShapeResolver } from './mongodb-filter.js'; import type { TemporalFieldKindResolver } from './mongodb-temporal.js'; /** @@ -542,12 +542,18 @@ export function buildAggregationPipeline(opts: { * on which one the caller took. */ temporalKind?: TemporalFieldKindResolver; + /** + * [#20444] Declared value shapes of the aggregated object, so a `$match` + * carrying `$empty` translates the field's declared row — the answer + * `find()` gives the same filter. + */ + valueShape?: ValueShapeResolver; }): Document[] { const pipeline: Document[] = []; // $match stage if (opts.where) { - const matchFilter = translateFilter(opts.where, opts.temporalKind); + const matchFilter = translateFilter(opts.where, opts.temporalKind, opts.valueShape); if (Object.keys(matchFilter).length > 0) { pipeline.push({ $match: matchFilter }); } diff --git a/packages/drivers/driver-mongodb/src/mongodb-driver.ts b/packages/drivers/driver-mongodb/src/mongodb-driver.ts index 2ba14c86d2e..fd15ec34c80 100644 --- a/packages/drivers/driver-mongodb/src/mongodb-driver.ts +++ b/packages/drivers/driver-mongodb/src/mongodb-driver.ts @@ -8,7 +8,7 @@ * ObjectStack's query protocol, aggregations, transactions, and streaming. */ -import type { DriverOptions } from '@objectstack/spec/data'; +import type { DriverOptions, ValueShapeFieldDef } from '@objectstack/spec/data'; import type { DriverQuery, IDataDriver } from '@objectstack/spec/contracts'; import { MongoClient, @@ -21,7 +21,23 @@ import { type MongoClientOptions, } from 'mongodb'; import { nanoid } from 'nanoid'; -import { translateFilter } from './mongodb-filter.js'; +import { translateFilter, type ValueShapeResolver } from './mongodb-filter.js'; + +/** + * [#20444] Each field's declared value shape — the `type` and `multiple` slice + * `expandEmptyOperator` reads — for the fields that declare a `type`. The RAW + * `type` is read: a field with none has no row of the ruled 「is empty」 table, + * and inventing one here would answer `$empty` with a row nobody declared. + */ +function indexValueShapes(fields: Record | undefined): Map { + const out = new Map(); + for (const [name, def] of Object.entries(fields ?? {})) { + const type = (def as { type?: unknown } | null | undefined)?.type; + if (typeof type !== 'string' || type === '') continue; + out.set(name, { type, multiple: (def as { multiple?: unknown }).multiple === true }); + } + return out; +} import { coerceTemporalValue, indexTemporalFields, @@ -180,6 +196,16 @@ export class MongoDBDriver implements IDataDriver { */ private temporalFields = new Map>(); + /** + * [#20444] Each declared field's value shape — its `type` and `multiple`, the + * slice the spec's `expandEmptyOperator` reads — per object, populated by + * {@link syncSchema} beside {@link temporalFields} and for the same reason: + * the `$empty` operator is answered by the field's DECLARED row, and a field + * this map does not hold is refused rather than given a row guessed from the + * data. + */ + private valueShapes = new Map>(); + constructor(config: MongoDBDriverConfig) { // Refuse to even EXIST in a multi-tenant deployment (#3724). The check is // repeated in `connect()`; construction just fails earliest, before a host @@ -336,7 +362,7 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const filter = translateFilter(query.where, this.temporalKindFor(object)); + const filter = translateFilter(query.where, this.temporalKindFor(object), this.valueShapeFor(object)); const findOptions = this.buildFindOptions(query, session); const cursor = collection.find(filter, findOptions); @@ -356,7 +382,7 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const filter = translateFilter(query.where, this.temporalKindFor(object)); + const filter = translateFilter(query.where, this.temporalKindFor(object), this.valueShapeFor(object)); // `singleRowLookup`: honour the caller's ordering, impose none of our own — // the engine sends `limit: 1`, which is indistinguishable from "page one of // a walk with page size 1", and the two want opposite things @@ -501,7 +527,7 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const filter = query?.where ? translateFilter(query.where, this.temporalKindFor(object)) : {}; + const filter = query?.where ? translateFilter(query.where, this.temporalKindFor(object), this.valueShapeFor(object)) : {}; return await collection.countDocuments(filter, { session }); } @@ -572,7 +598,7 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const filter = translateFilter(query.where, this.temporalKindFor(object)); + const filter = translateFilter(query.where, this.temporalKindFor(object), this.valueShapeFor(object)); const { _id, id, ...rawUpdate } = data; const updateData: Record = { ...this.toStorageForms(object, rawUpdate) }; updateData.updated_at = new Date(); @@ -590,7 +616,7 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const filter = translateFilter(query.where, this.temporalKindFor(object)); + const filter = translateFilter(query.where, this.temporalKindFor(object), this.valueShapeFor(object)); const result = await collection.deleteMany(filter, { session }); return result.deletedCount; } @@ -624,6 +650,9 @@ export class MongoDBDriver implements IDataDriver { limit: query.limit, offset: query.offset, temporalKind: this.temporalKindFor(object), + // [#20444] …and the declared value shapes, so `$empty` in the `$match` + // answers the row `find()` answers. + valueShape: this.valueShapeFor(object), }); const results = await collection.aggregate(pipeline, { session }).toArray(); @@ -665,6 +694,8 @@ export class MongoDBDriver implements IDataDriver { // Learn which fields are temporal BEFORE any write can land, so the write // path and the filter path share one storage convention (#4047). this.temporalFields.set(object, indexTemporalFields(objectDef.fields)); + // [#20444] …and each field's declared value shape, for `$empty`. + this.valueShapes.set(object, indexValueShapes(objectDef.fields)); await syncCollectionSchema(this.db, object, objectDef); } @@ -688,7 +719,7 @@ export class MongoDBDriver implements IDataDriver { async explain(object: string, query: DriverQuery, _options?: DriverOptions): Promise { const collection = this.getCollection(object); - const filter = translateFilter(query.where, this.temporalKindFor(object)); + const filter = translateFilter(query.where, this.temporalKindFor(object), this.valueShapeFor(object)); const explanation = await collection.find(filter).explain('executionStats'); return explanation; } @@ -800,6 +831,17 @@ export class MongoDBDriver implements IDataDriver { return (field: string) => kinds.get(field); } + /** + * [#20444] The declared-value-shape lookup for one object, handed to + * {@link translateFilter} so `$empty` translates the field's declared row. + * `undefined` for an undeclared object — `$empty` is then refused. + */ + private valueShapeFor(object: string): ValueShapeResolver | undefined { + const shapes = this.valueShapes.get(object); + if (!shapes || shapes.size === 0) return undefined; + return (field: string) => shapes.get(field); + } + /** * Put every declared temporal field of a document into its storage form — * the write half of the convention {@link translateFilter} reads against. diff --git a/packages/drivers/driver-mongodb/src/mongodb-filter-logic-translation.test.ts b/packages/drivers/driver-mongodb/src/mongodb-filter-logic-translation.test.ts index e6bc0ddea87..114177a8bce 100644 --- a/packages/drivers/driver-mongodb/src/mongodb-filter-logic-translation.test.ts +++ b/packages/drivers/driver-mongodb/src/mongodb-filter-logic-translation.test.ts @@ -47,7 +47,15 @@ import { describe, it, expect } from 'vitest'; import { FILTER_LOGIC_CASES, FILTER_LOGIC_ROWS, type FilterLogicRow } from '@objectstack/spec/data'; -import { translateFilter } from './mongodb-filter.js'; +import { translateFilter, type ValueShapeResolver } from './mongodb-filter.js'; + +/** + * [#20444] The fixture's columns as the driver would hold them after + * `syncSchema`: every column a declared `text` field. The table's `$empty` rows + * are answered by the field's DECLARED row, and `translateFilter` refuses the + * operator when it is handed no declaration — the standalone call's default. + */ +const FIXTURE_SHAPES: ValueShapeResolver = () => ({ type: 'text' }); // ── A deliberately strict reader of the emitted document ──────────────────── @@ -207,7 +215,7 @@ function documentLevelNots(doc: unknown, path = '$'): string[] { describe('translateFilter — filter logic conformance, without a server (#4405)', () => { for (const c of FILTER_LOGIC_CASES) { it(c.name, () => { - const doc = translateFilter(c.filter) as Record; + const doc = translateFilter(c.filter, undefined, FIXTURE_SHAPES) as Record; expect(select(doc), `${c.note ?? ''}\nemitted: ${JSON.stringify(doc)}`).toEqual([ ...c.expected, ]); @@ -220,7 +228,7 @@ describe('translateFilter — filter logic conformance, without a server (#4405) describe('translateFilter — the shapes MongoDB spells differently', () => { it('never emits a document-level $not: the server rejects it outright', () => { for (const c of FILTER_LOGIC_CASES) { - const doc = translateFilter(c.filter) as Record; + const doc = translateFilter(c.filter, undefined, FIXTURE_SHAPES) as Record; expect(documentLevelNots(doc), `${c.name} emitted ${JSON.stringify(doc)}`).toEqual([]); } }); diff --git a/packages/drivers/driver-mongodb/src/mongodb-filter.ts b/packages/drivers/driver-mongodb/src/mongodb-filter.ts index 7b33578be43..30452b5bd00 100644 --- a/packages/drivers/driver-mongodb/src/mongodb-filter.ts +++ b/packages/drivers/driver-mongodb/src/mongodb-filter.ts @@ -48,6 +48,9 @@ import { asciiCaseInsensitiveRegexSource } from '@objectstack/spec/data'; // [#13524] The declared authorable field vocabulary, IN DECLARATION ORDER — // the canonical order `FIELD_OPERATOR_RANK` below reads. import { FILTER_OPERATORS } from '@objectstack/spec/data'; +// [#20444] The `$empty` operator's ONE expansion — the field's declared row of +// the ruled 「is empty」 table, asked of the spec per translation. +import { expandEmptyOperator, type ValueShapeFieldDef } from '@objectstack/spec/data'; import { coerceTemporalValue, type TemporalFieldKind, @@ -303,6 +306,16 @@ function classifyFilterKey(key: string, value: unknown, here: string): FilterVer throw nonBooleanNullComparandError(key, value.$null, `${here}.$null`); } + // [#20444] `$empty`'s comparand is a boolean by the same declaration, gated on + // this walk for the same evaluation-order reason as `$null` above. + if ( + isFilterNode(value) && + Object.prototype.hasOwnProperty.call(value, '$empty') && + typeof value.$empty !== 'boolean' + ) { + throw nonBooleanEmptyComparandError(key, value.$empty, `${here}.$empty`); + } + // [#6520] `$icontains`' comparand is a NON-EMPTY string, gated on the WALK for // the same reason `$null` is one paragraph up: a gate in the emitter fires or // not depending on whether a boolean identity settled the enclosing node @@ -525,6 +538,100 @@ function nonBooleanNullComparandError(field: string, value: unknown, path: strin ); } +/** + * [#20444] `$empty` whose comparand is not a boolean. The leading sentence is + * `driver-sql`'s `nonBooleanEmptyComparandError`, verbatim — one condition, + * one wording (#5240). + */ +function nonBooleanEmptyComparandError(field: string, value: unknown, path: string): Error { + return unsupportedFilterError( + `Operator "$empty" on field "${field}" requires a boolean comparand (true or false). ` + + `Received ${describeFilterOperand(value)} (${safeShapePreview(value)}) at ${path}. ` + + `@objectstack/spec FieldOperatorsSchema declares $empty as a boolean: true asks for the ` + + `empty rows, false for their exact complement.`, + ); +} + +/** + * [#20444] `$empty` aimed at a field whose declaration this translator was not + * handed — an object never passed through `syncSchema`, a field its schema + * does not name, one declared with no `type`, or a standalone call to + * {@link translateFilter} with no {@link ValueShapeResolver}. What counts as + * empty is the field's DECLARED row of the ruled table, so there is nothing to + * translate without it: refused, never guessed. + */ +function undeclaredEmptyOperatorFieldError(field: string, path: string): Error { + return unsupportedFilterError( + `Operator "$empty" on field "${field}" at ${path} targets a field whose declaration this ` + + `driver does not hold (no declared type — the object's schema was never synced, or does not ` + + `declare the field). What counts as empty is the field's DECLARED row of the ruled table — ` + + `null or '' for a text-like type, null or [] for a multi-value field, null only for every ` + + `other type — so the operator is refused rather than guessed. Declare the field, or use ` + + `"$null" for "has no value".`, + ); +} + +/** + * [#20444] A field's DECLARED value shape — its `type` and `multiple`, the + * slice the spec's `expandEmptyOperator` reads — or `undefined` when the + * declaration is not held. `MongoDBDriver` answers it from the declaration + * `syncSchema` recorded, the way it answers {@link TemporalFieldKindResolver}. + */ +export type ValueShapeResolver = (field: string) => ValueShapeFieldDef | undefined; + +/** + * [#20444] Translate `{ field: { $empty: true | false } }` by the field's + * DECLARED row of the ruled 「is empty」 table — ruling B on #20311 (record + * 5861435168), spelled as this operator by ruling A on #20399 (record + * 5865693155) — through the spec's one expansion, `expandEmptyOperator`: + * + * | declared row | `$empty: true` | `$empty: false` — the exact complement | + * |---|---|---| + * | `null_only` | `{ f: { $eq: null } }` | `{ f: { $ne: null } }` | + * | `text` | `{ f: { $in: [null, ''] } }` | `{ f: { $nin: [null, ''] } }` | + * | `multi_value` | `{ $or: [{ f: { $eq: null } }, { f: { $size: 0 } }] }` | the same pair under `$nor` | + * + * MongoDB's `null` equality matches a missing field as well as a stored null, + * so both readings of "no value" are empty on every row — the answer the + * `$null` arm already gives. The empty list is tested with `$size: 0`, never + * as an equality comparand (`{ f: [] }` also matches an array HOLDING an empty + * array, and ruling 乙 on #19757 keeps `[]` out of the equality slot anyway). + * + * Emitted as its OWN document, AND-ed beside the field's other operators by + * {@link translateCondition}, rather than written into their operator map: the + * multi-value row is an OR over two tests no single field operator spells, and + * a separate conjunct can never contest a lowered key with a sibling operator + * (the {@link assembleLoweredWrites} clobber class). The flag's boolean shape + * was settled on the walk; the re-check is the totality floor. + */ +function translateEmptyOperator( + field: string, + flag: unknown, + valueShape: ValueShapeResolver | undefined, + path: string, +): Filter { + if (typeof flag !== 'boolean') throw nonBooleanEmptyComparandError(field, flag, path); + const shape = valueShape?.(field); + if (!shape) throw undeclaredEmptyOperatorFieldError(field, path); + const expansion = expandEmptyOperator(shape); + switch (expansion.arm) { + case 'null_only': + return { [field]: flag ? { $eq: null } : { $ne: null } }; + case 'text': + return { [field]: flag ? { $in: [null, ''] } : { $nin: [null, ''] } }; + case 'multi_value': { + const branches: Filter[] = [{ [field]: { $eq: null } }, { [field]: { $size: 0 } }]; + return flag ? { $or: branches } : { $nor: branches }; + } + default: { + // A closed union of three rows; a fourth is a spec change this driver + // was not taught, and it must fail loudly rather than answer for it. + const unknownArm: never = expansion.arm; + throw unsupportedFilterError(`No $empty arm for the declared row ${JSON.stringify(unknownArm)}.`); + } + } +} + /** [#5376] Is this field spec `{}` — a field constrained by ZERO operators? */ function isEmptyFieldConstraint(spec: unknown): boolean { return isFilterNode(spec) && Object.keys(spec).length === 0; @@ -698,6 +805,10 @@ function safeShapePreview(value: unknown): string { export function translateFilter( where: unknown, temporalKind?: TemporalFieldKindResolver, + // [#20444] The declared value shape of each field, for `$empty`'s declared + // row. Omitted, `$empty` is refused — the pure shape translation has no + // declaration to read a row from. + valueShape?: ValueShapeResolver, ): Filter { if (!where) return {}; @@ -717,7 +828,7 @@ export function translateFilter( if (verdict === 'true') return {}; if (verdict === 'false') return matchNothing(); - return translateCondition(node, temporalKind, 'filter'); + return translateCondition(node, temporalKind, 'filter', valueShape); } /** @@ -738,6 +849,8 @@ function translateCondition( // position it refused — the same `filter.$or[0].stage` spelling driver-sql // and driver-memory print. path = 'filter', + // [#20444] See {@link translateFilter}. + valueShape?: ValueShapeResolver, ): Filter { const mongoFilter: Record = {}; const andClauses: Filter[] = []; @@ -763,7 +876,7 @@ function translateCondition( const branches = (value as unknown[]) .map((sub, index) => ({ sub: sub as Record, index })) .filter(({ sub, index }) => reduceFilterNode(sub, `${here}[${index}]`) === 'clause') - .map(({ sub, index }) => translateCondition(sub, temporalKind, `${here}[${index}]`)); + .map(({ sub, index }) => translateCondition(sub, temporalKind, `${here}[${index}]`, valueShape)); andClauses.push(key === '$and' ? { $and: branches } : { $or: branches }); break; } @@ -780,6 +893,7 @@ function translateCondition( value as Record, temporalKind, `${path}.$not`, + valueShape, ); // MongoDB $not applies per-field; for top-level negation use $nor andClauses.push({ $nor: [inner] }); @@ -792,8 +906,16 @@ function translateCondition( if (value !== null && typeof value === 'object' && !Array.isArray(value) && !(value instanceof Date)) { // Check if this is an operator object (has $ keys) - const objValue = value as Record; + let objValue = value as Record; const hasOps = Object.keys(objValue).some((k) => k.startsWith('$')); + // [#20444] `$empty` becomes a document of its own, AND-ed beside the + // field's other operators — see {@link translateEmptyOperator}. + if (hasOps && Object.prototype.hasOwnProperty.call(objValue, '$empty')) { + const { $empty: flag, ...rest } = objValue; + andClauses.push(translateEmptyOperator(key, flag, valueShape, `${path}.${key}.$empty`)); + if (Object.keys(rest).length === 0) continue; + objValue = rest; + } if (hasOps) { const translated = translateFieldOperators(objValue, temporalKind?.(key), key, `${path}.${key}`); // [#13524] Lowered writes whose MongoDB key was already taken by a diff --git a/packages/drivers/driver-sql/src/sql-driver-20444-empty-operator.test.ts b/packages/drivers/driver-sql/src/sql-driver-20444-empty-operator.test.ts new file mode 100644 index 00000000000..c63d892e08a --- /dev/null +++ b/packages/drivers/driver-sql/src/sql-driver-20444-empty-operator.test.ts @@ -0,0 +1,151 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20444] The staged `$empty` operator on `driver-sql`'s filter compiler + * (`applyFilterCondition`), answered by the field's DECLARED row of the ruled + * 「is empty」 table — ruling B on #20311 (record 5861435168), spelled as this + * operator by ruling A on #20399 (record 5865693155) — through the spec's one + * expansion, `expandEmptyOperator`: + * + * | declared row | `$empty: true` | `$empty: false` | + * |---|---|---| + * | text-like (`text`) | null or `''` | the exact complement | + * | multi-value (`tags`; `lookup` with `multiple: true`) | null or `[]` | the exact complement | + * | every other type (`number`; `lookup` single) | null only | the exact complement | + * + * The shared `FILTER_LOGIC_CASES` rows cannot tell those rows apart (their + * fixture stores neither `''` nor `[]`); this file can. It runs on every cell + * of the live dialect matrix, because the multi-value row is a different JSON + * construct per dialect — SQLite always, PostgreSQL and MySQL where the + * `Temporal Conformance (live PG + MySQL)` job provisions them. + * + * `$empty` is staged out of `FILTER_OPERATORS` (「照 $like 先例分阶段」, record + * 5868169573), so the engine's front door still refuses it; the driver is + * driven directly here, which is exactly the caller the arm answers today. + */ + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import type { FilterCondition } from '@objectstack/spec/data'; +import { SqlDriver } from './sql-driver.js'; +import { DIALECT_CELLS, declareDialectCell, type DialectCell } from './live-dialect-matrix.testkit.js'; + +const TABLE = 'os20444_empty_operator'; + +const shape = (name: string) => ({ + name, + fields: { + title: { type: 'text' }, + tags: { type: 'tags' }, + owners: { type: 'lookup', reference: name, multiple: true }, + parent: { type: 'lookup', reference: name }, + score: { type: 'number' }, + }, +}) as any; + +/** + * Every stored state a column of each row can hold: a value, the empty string, + * the empty list, NULL — plus `' '` (not `''`) and `0` (not empty), the two + * values a lenient reading would count as empty. + */ +const ROWS = [ + { id: 'r1', title: 'x', tags: ['a'], owners: ['u1'], parent: 'p1', score: 5 }, + { id: 'r2', title: '', tags: [], owners: [], parent: null, score: 0 }, + { id: 'r3', title: null, tags: null, owners: null, parent: null, score: null }, + { id: 'r4', title: ' ', tags: ['a', 'b'], owners: ['u2'], parent: 'p2', score: -1 }, +]; + +const NO_AUDIT = { bypassTenantAudit: true }; + +function measure(cell: DialectCell): void { + describe(`[#20444] $empty by the declared row — ${cell.label}`, () => { + let driver: SqlDriver; + const ids = async (where: FilterCondition) => + (await driver.find(TABLE, { where }, NO_AUDIT)).map((r) => String(r.id)).sort(); + const refusal = async (where: FilterCondition) => { + try { + await driver.find(TABLE, { where }, NO_AUDIT); + } catch (err) { + return { code: (err as { code?: string }).code, status: (err as { status?: number }).status }; + } + return 'answered'; + }; + + beforeAll(async () => { + driver = new SqlDriver(cell.config()); + await driver.execute(`drop table if exists ${TABLE}`).catch(() => {}); + await driver.initObjects([shape(TABLE)]); + for (const row of ROWS) await driver.create(TABLE, { ...row }, NO_AUDIT); + }); + + afterAll(async () => { + await driver.execute(`drop table if exists ${TABLE}`).catch(() => {}); + await driver.disconnect(); + }); + + it('the fixture is the four rows, stored as written', async () => { + expect(await ids({})).toEqual(['r1', 'r2', 'r3', 'r4']); + const r2 = await driver.findOne(TABLE, { where: { id: 'r2' } }, NO_AUDIT); + // The two stored states the operator exists to recognise, read back. + expect(r2?.title).toBe(''); + expect(r2?.tags).toEqual([]); + }); + + it("text-like: null or '' — not ' '", async () => { + expect(await ids({ title: { $empty: true } })).toEqual(['r2', 'r3']); + expect(await ids({ title: { $empty: false } })).toEqual(['r1', 'r4']); + }); + + it('multi-value: null or [] — tags and a lookup with multiple: true', async () => { + for (const field of ['tags', 'owners']) { + expect(await ids({ [field]: { $empty: true } }), field).toEqual(['r2', 'r3']); + expect(await ids({ [field]: { $empty: false } }), field).toEqual(['r1', 'r4']); + } + }); + + it('every other type: null only — 0 is a value, and a single lookup is not a list', async () => { + expect(await ids({ score: { $empty: true } })).toEqual(['r3']); + expect(await ids({ score: { $empty: false } })).toEqual(['r1', 'r2', 'r4']); + expect(await ids({ parent: { $empty: true } })).toEqual(['r2', 'r3']); + expect(await ids({ parent: { $empty: false } })).toEqual(['r1', 'r4']); + }); + + it('$empty: false is the exact complement on every field — the two partition the table', async () => { + for (const field of ['title', 'tags', 'owners', 'parent', 'score']) { + const empty = await ids({ [field]: { $empty: true } }); + const full = await ids({ [field]: { $empty: false } }); + expect([...empty, ...full].sort(), field).toEqual(['r1', 'r2', 'r3', 'r4']); + expect(empty.filter((id) => full.includes(id)), field).toEqual([]); + } + }); + + it('under $not the answer is the complement — no row is lost to UNKNOWN', async () => { + expect(await ids({ $not: { title: { $empty: true } } })).toEqual(['r1', 'r4']); + expect(await ids({ $not: { tags: { $empty: false } } })).toEqual(['r2', 'r3']); + expect(await ids({ $not: { score: { $empty: true } } })).toEqual(['r1', 'r2', 'r4']); + }); + + it('nests under $and / $or like any predicate, and ANDs with a sibling operator', async () => { + expect(await ids({ $or: [{ score: 5 }, { tags: { $empty: true } }] })).toEqual(['r1', 'r2', 'r3']); + expect(await ids({ $and: [{ title: { $empty: false } }, { owners: { $empty: false } }] })).toEqual(['r1', 'r4']); + expect(await ids({ $or: [{ $and: [{ score: { $empty: false } }, { title: { $empty: true } }] }, { id: 'r1' }] })) + .toEqual(['r1', 'r2']); + expect(await ids({ title: { $empty: false, $ne: 'x' } })).toEqual(['r4']); + }); + + it('a field with no declaration is REFUSED, never guessed', async () => { + // `id` is a builtin column no field declares. + expect(await refusal({ id: { $empty: true } })).toEqual({ code: 'INVALID_FILTER', status: 400 }); + expect(await refusal({ $or: [{ id: 'r1' }, { nope: { $empty: false } }] })) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + }); + + it('a non-boolean flag is REFUSED on the walk, even where an identity would skip the emitter', async () => { + expect(await refusal({ title: { $empty: 'yes' as unknown as boolean } })) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + expect(await refusal({ $or: [{}, { title: { $empty: 1 as unknown as boolean } }] })) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + }); + }); +} + +for (const cell of DIALECT_CELLS) declareDialectCell(cell, '$empty operator', measure); diff --git a/packages/drivers/driver-sql/src/sql-driver-compile-refusal-seam.test.ts b/packages/drivers/driver-sql/src/sql-driver-compile-refusal-seam.test.ts index 408e9cb3a3e..1b022cb4f3a 100644 --- a/packages/drivers/driver-sql/src/sql-driver-compile-refusal-seam.test.ts +++ b/packages/drivers/driver-sql/src/sql-driver-compile-refusal-seam.test.ts @@ -74,8 +74,16 @@ type Door = { readonly authorExtendsLog?: true; /** The refusal fires only when the array IS the `where` root — no merged arm. */ readonly rootOnly?: true; + /** + * [#20444] Put the driver in the state the refusal needs before each case — + * for a refusal no filter alone can reach on this fixture's SQLite client. + */ + readonly setup?: (driver: SqlDriver) => void; }; +/** [#20444] A column no field of the fixture declares. */ +const UNDECLARED_COL = 'secret_undeclared_col'; + const DOORS: readonly Door[] = [ // ── #20039: the class this file closes ──────────────────────────────────── { @@ -220,6 +228,30 @@ const DOORS: readonly Door[] = [ secrets: [POLICY_COL, SECRET], klass: 'Operator "$exists" in this filter requires a boolean comparand', }, + // ── #20444: the staged `$empty` operator, born in the seam ────────────────── + { + builder: 'nonBooleanEmptyComparandError', + where: () => ({ [POLICY_COL]: { $empty: SECRET } }), + secrets: [POLICY_COL, SECRET], + klass: 'Operator "$empty" in this filter requires a boolean comparand', + }, + { + builder: 'undeclaredEmptyOperatorFieldError', + where: () => ({ [UNDECLARED_COL]: { $empty: true } }), + secrets: [UNDECLARED_COL], + klass: 'targets a field whose declaration this driver does not hold', + }, + { + builder: 'emptyListUnsupportedDialectError', + where: () => ({ [JSON_COL]: { $empty: false } }), + secrets: [JSON_COL], + klass: 'whose empty list is tested with a JSON function that differs per SQL dialect', + // A knex client this driver does not model (mssql, oracle) reads as the + // `'unknown'` dialect, where the multi-value row has no construct. + setup: (driver) => { + Object.defineProperty(driver, 'dialectName', { get: () => 'unknown', configurable: true }); + }, + }, ]; // ── Half 1: the enumeration ─────────────────────────────────────────────────── @@ -396,6 +428,10 @@ describe('[#20039] every filter-compile refusal × filter-subtree provenance', ( for (const door of DOORS) { describe(door.builder, () => { + beforeEach(() => { + door.setup?.(driver); + }); + it('policy-marked ⇒ same code and status, operands WITHHELD, and in the server log', async () => { expectWithheld(await refusalOf(markFilterSubtreeProvenance(door.where(), 'policy')), door); }); diff --git a/packages/drivers/driver-sql/src/sql-driver-or-filter.test.ts b/packages/drivers/driver-sql/src/sql-driver-or-filter.test.ts index d22f30b70b0..69da08c28c5 100644 --- a/packages/drivers/driver-sql/src/sql-driver-or-filter.test.ts +++ b/packages/drivers/driver-sql/src/sql-driver-or-filter.test.ts @@ -95,6 +95,18 @@ function declareFilterLogicSweep(cell: DialectCell): void { t.string('parent_id'); }); await knexInstance(FILTER_TABLE).insert([...FILTER_LOGIC_ROWS]); + // [#20444] The table is built through knex, so the driver holds no field + // declaration for it — and the table's `$empty` rows are answered by the + // field's DECLARED row, refused without one. Register the declaration + // (metadata only, no DDL) the way a real object reaches the driver. + driver.registerObjectMetadata([ + { + name: FILTER_TABLE, + fields: Object.fromEntries( + ['a', 'b', 'c', 'd', 'owner', 'status', 'parent_object', 'parent_id'].map((f) => [f, { type: 'text' }]), + ), + }, + ]); }); afterAll(async () => { diff --git a/packages/drivers/driver-sql/src/sql-driver.ts b/packages/drivers/driver-sql/src/sql-driver.ts index 63e5dac4557..d58ddd20c9d 100644 --- a/packages/drivers/driver-sql/src/sql-driver.ts +++ b/packages/drivers/driver-sql/src/sql-driver.ts @@ -32,6 +32,12 @@ import { numericColumnFor } from '@objectstack/spec/data'; // the write check `@objectstack/formula` evaluates judge one comparison by one // rule. import { crossFieldColumnVerdict, type CrossFieldComparisonClass } from '@objectstack/spec/data'; +// [#20444] The `$empty` operator's ONE expansion (ruling A on #20399, record +// 5865693155): the field's declared row of the ruled 「is empty」 table, asked +// of the spec at compile time by {@link SqlDriver.applyEmptyOperator} against +// the declaration {@link SqlDriver.valueShapeFields} recorded. This driver keeps +// no copy of the table. +import { expandEmptyOperator, type ValueShapeFieldDef } from '@objectstack/spec/data'; // [#5659] The Filter Protocol's boolean identity reduction — `$and: []` is TRUE, // `$or: []` is FALSE, `{}` is a TRUE disjunct, `$not: {}` is FALSE. One // implementation for all four consumers, proven against the same @@ -1963,6 +1969,24 @@ function isMultiValuedColumn(type: string, field: { multiple?: unknown } | null return isMultiValueField({ type, multiple: field?.multiple === true }); } +/** + * [#20444] The declared value shape of every field that declares a `type` — + * the {@link SqlDriver.valueShapeFields} entry for one object. Reads the RAW + * `type`, deliberately not the `field.type || 'string'` default the column + * registries apply: a field with no type has no row of the ruled 「is empty」 + * table, and inventing `'string'` for it here would answer `$empty` with a row + * nobody declared. + */ +function declaredValueShapes(fields: Record | undefined): Record { + const shapes: Record = {}; + for (const [name, field] of Object.entries(fields ?? {})) { + const type = (field as { type?: unknown } | null | undefined)?.type; + if (typeof type !== 'string' || type === '') continue; + shapes[name] = { type, multiple: (field as { multiple?: unknown }).multiple === true }; + } + return shapes; +} + /** * [#17231] ADR-0113's physical column constraint, asked once for every column * {@link SqlDriver.createColumn} builds. @@ -2441,7 +2465,7 @@ function retiredFilterOperatorError( */ const SUPPORTED_FILTER_OPERATORS_SENTENCE = 'Supported operators: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin, $between, $contains, ' + - '$notContains, $startsWith, $endsWith, $icontains, $like, $ilike, $null, $exists.'; + '$notContains, $startsWith, $endsWith, $icontains, $like, $ilike, $null, $exists, $empty.'; /** * An operator outside the emitter's vocabulary and outside the retired table. @@ -4406,6 +4430,135 @@ function nonBooleanFlagWithheldMessage(op: '$null' | '$exists'): string { ); } +// ── [#20444] The `$empty` operator ─────────────────────────────────────────── + +/** + * [#20444] A non-boolean `$empty` comparand. `FieldOperatorsSchema` declares + * `$empty: z.boolean()`, and the spec's save door already refuses anything else + * (`filter-save-door-refusals.ts` counts it among the boolean flags beside + * `$null` / `$exists`). Refused here on the validating walk for the reason its + * two siblings are: a two-branch emitter has a DEFAULT side, and a third value + * would silently land on it. + */ +function nonBooleanEmptyComparandError( + field: string, + value: unknown, + path: string, + subtree?: unknown, +): Error { + return withheldFilterError( + 'Operator "$empty" in this filter requires a boolean comparand (true or false). ' + + '@objectstack/spec FieldOperatorsSchema declares $empty as a boolean, and a non-boolean is ' + + 'refused rather than coerced: true asks for the empty rows, false for their exact ' + + 'complement, and any other value would land on whichever side a compiler defaults to. The ' + + 'field it was aimed at and the value it received are withheld from the message; the full ' + + 'diagnostic is in the server log.', + `Operator "$empty" on field "${field}" requires a boolean comparand (true or false). ` + + `Received ${describeFilterOperand(value)} (${safeShapePreview(value)}) at ${path}. ` + + `@objectstack/spec FieldOperatorsSchema declares $empty as a boolean: true asks for the ` + + `empty rows, false for their exact complement.`, + subtree, + ); +} + +/** + * [#20444] `$empty` aimed at a column whose DECLARATION this driver does not + * hold — a table created outside `initObjects` / `registerObjectMetadata` / + * `registerExternalObject`, a builtin column no field declares (`id`), or a + * field declared with no `type`. + * + * What counts as empty is the field's declared row of the ruled table (null or + * `''` for a text-like type, null or `[]` for a multi-value field, null only + * for every other type), so without the declaration there is no answer to + * compile. The spec's by-value reading — the one it gives the faces that hold + * NO field declarations — has no SQL form here: `amount = ''` is a type error + * on PostgreSQL, and an empty list is only recognisable as JSON. So the filter + * is refused rather than guessed. + */ +function undeclaredEmptyOperatorFieldError(field: string, subtree?: unknown): Error { + const why = + "What counts as empty is the field's DECLARED row of the ruled table — null or '' for a " + + 'text-like type, null or [] for a multi-value field, null only for every other type — so ' + + 'the operator is refused rather than guessed. Filter on a declared field, or use "$null" for ' + + '"has no value".'; + return withheldFilterError( + 'Operator "$empty" in this filter targets a field whose declaration this driver does not ' + + `hold (no declared type). ${why} The field is withheld from the message; the full ` + + 'diagnostic is in the server log.', + `Operator "$empty" on field "${field}" targets a field whose declaration this driver does not ` + + `hold (no declared type). ${why}`, + subtree, + ); +} + +/** + * [#20444] `$empty` on a multi-value field, over a knex client whose dialect + * this driver does not model (`dialectName === 'unknown'`). The empty list is + * tested with a JSON function that differs per dialect and none of the three + * this driver speaks parses everywhere, so the multi-value row has no construct + * there. The text and null-only rows need no dialect and compile everywhere. + */ +function emptyListUnsupportedDialectError(field: string, subtree?: unknown): Error { + const why = + 'a multi-value field, whose empty list is tested with a JSON function that differs per SQL ' + + 'dialect, and this connection\'s dialect is not one this driver models (SQLite, PostgreSQL, ' + + 'MySQL). It is refused rather than guessed; "$null" answers "has no value" on every dialect.'; + return withheldFilterError( + `Operator "$empty" in this filter targets ${why} The field is withheld from the message; the ` + + 'full diagnostic is in the server log.', + `Operator "$empty" on field "${field}" targets ${why}`, + subtree, + ); +} + +/** + * [#20444] "Is this stored JSON value the empty list?", as ONE boolean SQL + * expression that is FALSE — never NULL, never an error — for every other + * stored JSON value, or `null` for a dialect with no construct. + * + * A multi-value field is a JSON column here ({@link SqlDriver.jsonColumn}: TEXT + * on SQLite, `json` on PostgreSQL and MySQL — {@link isMultiValuedColumn} keys + * the DDL), so the question is asked of the stored JSON, never as a `$eq: []` + * comparand (ruling 乙 on #19757 refuses an empty list in the equality slot, and + * this operator does not reopen it). + * + * - **SQLite** — the column is TEXT, so bytes that are not JSON are physically + * storable (the legacy form {@link jsonMembershipPredicate} guards the same + * way). `json_valid` is asked first, inside a `CASE` whose branches are + * evaluated lazily, so a malformed cell answers FALSE rather than failing the + * statement; `json_type` keeps a non-array JSON value (for which + * `json_array_length` answers 0) from counting as an empty list. + * - **PostgreSQL** — `json` has no equality operator, so the value is compared + * as `jsonb`, whose equality is structural (`[ ]` equals `[]`). + * - **MySQL** — `JSON_LENGTH` answers 0 for an empty OBJECT too, so the type is + * asked beside it. + * + * The same three constructs `service-analytics`' SQL compilers emit for the + * same row (`empty-operator-sql.ts`); that package depends on no driver, so + * the construct is restated rather than imported, and both are held to the + * ruled table by their own suites. + */ +function emptyJsonListPredicate( + dialect: SqlDialectName, + field: string, +): { sql: string; bindings: string[] } | null { + switch (dialect) { + case 'sqlite': + return { + sql: + "(CASE WHEN json_valid(??) THEN json_type(??) = 'array' AND json_array_length(??) = 0 " + + 'ELSE 0 END)', + bindings: [field, field, field], + }; + case 'postgres': + return { sql: "(CAST(?? AS jsonb) = CAST('[]' AS jsonb))", bindings: [field] }; + case 'mysql': + return { sql: "(JSON_TYPE(??) = 'ARRAY' AND JSON_LENGTH(??) = 0)", bindings: [field, field] }; + default: + return null; + } +} + /** * [#6050] `undefined` in a COMPARAND position. * @@ -4802,6 +4955,17 @@ function classifyFilterKey( throw nonBooleanExistsComparandError(key, value.$exists, `${here}.$exists`, value); } + // [#20444] `$empty`'s comparand is a boolean by the same declaration, refused + // on the same walk for the same evaluation-order reason — its own `if`, not a + // loop over a flag list, for the reason the `$exists` gate above gives. + if ( + isFilterNode(value) && + Object.prototype.hasOwnProperty.call(value, '$empty') && + typeof value.$empty !== 'boolean' + ) { + throw nonBooleanEmptyComparandError(key, value.$empty, `${here}.$empty`, value); + } + // [#5702] `$icontains`'s comparand is a NON-EMPTY string by declaration, // refused on this walk for the same evaluation-order reason as the two gates // above: an empty comparand makes the predicate match every row, and a gate @@ -4920,6 +5084,10 @@ function nullValueSatisfiesOperator(op: string, value: unknown): boolean { // made #5347 rewrite the `$null` arm does not exist here, because both // spellings agree on both surviving values. case '$exists': return value === false; + // [#20444] Null counts as empty on EVERY row of the ruled table, so a NULL + // column satisfies `$empty: true` and fails its complement. The walk refuses + // a non-boolean before this table is consulted. + case '$empty': return value === true; // Negative-polarity set/substring tests: "not among" / "does not contain" // hold vacuously for a value that is absent. case '$nin': return true; @@ -4949,6 +5117,11 @@ function operatorIsNullTotal(op: string, value: unknown): boolean { // Compile to `IS NULL` / `IS NOT NULL` — two-valued by construction. case '$null': case '$exists': + // [#20444] `$empty` spells its NULL case out in both polarities — + // `(col IS NULL OR …)` / `(col IS NOT NULL AND NOT …)`, the `…` FALSE and + // never NULL for a stored value ({@link emptyJsonListPredicate}) — so it + // is TRUE or FALSE for every row and `NOT` over it is the exact complement. + case '$empty': return true; // A null comparand makes these null PREDICATES too (see the `$eq`/`$ne` // arms of the emitter below), not comparisons. @@ -5621,6 +5794,29 @@ export class SqlDriver implements IDataDriver { protected knex: Knex; protected config: Knex.Config; protected jsonFields: Record = {}; + /** + * [#20444] Each field's DECLARED value shape — its `type` and `multiple`, the + * slice {@link expandEmptyOperator} reads — per table, filled at the same + * three places {@link jsonFields} is ({@link registerManagedObjectMetadata}, + * {@link registerExternalObject} and the shard alias), from the declaration + * and nothing else. + * + * It is the one input the `$empty` operator needs that no other registry + * holds: `jsonFields` cannot say it, because a structured JSON type (`json`, + * `address`, …) is a JSON column whose row of the ruled table is null-only, + * while a multi-value field is a JSON column whose row counts `[]`; and no + * registry here names the text-like types at all. + * + * A field declared with no `type` is not recorded, and neither is a column no + * field declares (`id`, a table built outside this driver's registration), so + * `$empty` on either is refused ({@link undeclaredEmptyOperatorFieldError}) + * rather than answered by a guessed row. A declared type the spec does not + * list among the text-like or multi-value types — including the + * driver-internal aliases an introspected or test object carries (`string`, + * `object`, `array`) — takes the row the spec's expansion gives it, which is + * null-only. + */ + protected valueShapeFields: Record> = {}; /** * SINGLE-VALUE file-family columns per table (`image` / `file` / `avatar` / * `video` / `audio`), filled at the same two registration sites as @@ -11493,6 +11689,7 @@ export class SqlDriver implements IDataDriver { */ protected aliasShardBookkeeping(base: string, shard: string): void { this.jsonFields[shard] = this.jsonFields[base] ?? []; + this.valueShapeFields[shard] = this.valueShapeFields[base] ?? {}; this.mediaFields[shard] = this.mediaFields[base] ?? []; this.booleanFields[shard] = this.booleanFields[base] ?? []; this.numericFields[shard] = this.numericFields[base] ?? []; @@ -11663,6 +11860,8 @@ export class SqlDriver implements IDataDriver { } } this.jsonFields[key] = jsonCols; + // [#20444] The declared value shapes `$empty` expands — see {@link valueShapeFields}. + this.valueShapeFields[key] = declaredValueShapes(schema.fields); this.mediaFields[key] = mediaCols; this.booleanFields[key] = booleanCols; this.numericFields[key] = numericCols; @@ -11778,6 +11977,8 @@ export class SqlDriver implements IDataDriver { } } this.jsonFields[tableName] = jsonCols; + // [#20444] The declared value shapes `$empty` expands — see {@link valueShapeFields}. + this.valueShapeFields[tableName] = declaredValueShapes(obj.fields); this.mediaFields[tableName] = mediaCols; this.booleanFields[tableName] = booleanCols; this.numericFields[tableName] = numericCols; @@ -15913,6 +16114,100 @@ export class SqlDriver implements IDataDriver { this.applyFalseConstant(builder, logicalOp); } + /** + * [#20444] The field's DECLARED value shape, or `undefined` when this driver + * holds no declaration for it — see {@link valueShapeFields}. Keyed like + * {@link isJsonColumn}: the registry key the builder's table resolves to and + * the LOCAL field name. `driver-turso`'s remote transport asks the same + * question through this method, so its two faces read one registry. + */ + protected declaredValueShape(table: string | null | undefined, localField: string): ValueShapeFieldDef | undefined { + if (!table) return undefined; + const shapes = this.valueShapeFields[table]; + return shapes && Object.prototype.hasOwnProperty.call(shapes, localField) ? shapes[localField] : undefined; + } + + /** + * [#20444] Compile `{ field: { $empty: true | false } }` — the staged + * emptiness operator, answered by the field's DECLARED row of the ruled + * 「is empty」 table (ruling B on #20311, record 5861435168; spelled as this + * operator by ruling A on #20399, record 5865693155), through the spec's one + * expansion, {@link expandEmptyOperator}: + * + * | declared row | `$empty: true` | `$empty: false` — the exact complement | + * |---|---|---| + * | `null_only` (every other type) | `col IS NULL` | `col IS NOT NULL` | + * | `text` (the text-like types) | `(col IS NULL OR col = '')` | `(col IS NOT NULL AND col <> '')` | + * | `multi_value` (a list-valued field) | `(col IS NULL OR L)` | `(col IS NOT NULL AND NOT L)` | + * + * `L` is {@link emptyJsonListPredicate}, the dialect's test for "this stored + * JSON value is the empty list". An empty list is tested as a STORED VALUE, + * never bound as a `$eq: []` comparand (ruling 乙 on #19757 still refuses an + * empty list there). + * + * Every predicate is TOTAL — TRUE or FALSE on every row, never UNKNOWN — + * because both polarities spell the NULL case out and `L` is never NULL for a + * stored value. So a `$not` over `$empty` needs no guard + * ({@link operatorIsNullTotal} answers `true` for it) and `NOT (…)` is the + * exact complement, with no three-valued-logic hole. Each predicate is one + * knex group, so its `OR` can never re-associate with a sibling conjunct. + * + * Refused, before anything is emitted: a field with no declaration here + * ({@link undeclaredEmptyOperatorFieldError}), and the multi-value row on a + * dialect this driver does not model ({@link emptyListUnsupportedDialectError}). + * + * ⚠️ Staged: `$empty` is not in `FILTER_OPERATORS` yet (the maintainer's + * amendment of ruling A, record 5868169573: 「照 $like 先例分阶段」), so the + * engine's front door still refuses it; this arm answers a caller that + * reaches the driver directly, and it is what the flip card turns on. + */ + private applyEmptyOperator( + builder: any, + logicalOp: 'and' | 'or', + table: string | null | undefined, + localField: string, + field: string, + empty: boolean, + // The field's operator map — the node a refusal is resolved against (#8220). + subtree: unknown, + ): void { + const shape = this.declaredValueShape(table, localField); + if (!shape) throw undeclaredEmptyOperatorFieldError(field, subtree); + const expansion = expandEmptyOperator(shape); + const method = logicalOp === 'or' ? 'orWhere' : 'where'; + switch (expansion.arm) { + case 'null_only': + builder[ + empty + ? (logicalOp === 'or' ? 'orWhereNull' : 'whereNull') + : (logicalOp === 'or' ? 'orWhereNotNull' : 'whereNotNull') + ](field); + return; + case 'text': + builder[method]((qb: any) => { + if (empty) qb.whereNull(field).orWhere(field, ''); + else qb.whereNotNull(field).andWhere(field, '<>', ''); + }); + return; + case 'multi_value': { + const list = emptyJsonListPredicate(this.dialectName, field); + if (!list) throw emptyListUnsupportedDialectError(field, subtree); + builder[method]((qb: any) => { + if (empty) qb.whereNull(field).orWhereRaw(list.sql, list.bindings); + else qb.whereNotNull(field).andWhereRaw(`NOT ${list.sql}`, list.bindings); + }); + return; + } + default: { + // The spec's `EmptyOperatorArm` is a closed union of the three rows + // above; a fourth reaching here is a spec change this driver was not + // taught, and it must fail loudly rather than answer for it. + const unknownArm: never = expansion.arm; + throw new Error(`[sql-driver] no $empty arm for the declared row ${JSON.stringify(unknownArm)}`); + } + } + } + /** * [#7398] The column-type half of the filter gate: refuse a DECLARED operator * that the column it was aimed at cannot give a meaningful answer for. @@ -16594,6 +16889,15 @@ export class SqlDriver implements IDataDriver { this.applyTextOperatorOverNonTextColumn(builder, logicalOp, rawOp); continue; } + // [#20444] `$empty` — answered by the field's DECLARED row of the + // ruled table, AFTER every refusal above (its non-boolean comparand + // was refused on the walk) and BEFORE the calendar-day rewrites, + // the comparand coercion and the normalised-column emitter: its + // flag is not a value of the column, so none of them applies. + if (rawOp === '$empty') { + this.applyEmptyOperator(builder, logicalOp, table, localField, field, opValue === true, value); + continue; + } // Calendar-day upper bounds first (#3777): `$lte` on a bare // `YYYY-MM-DD` against a datetime column compiles half-open, and a // `$between` whose max is a bare day decomposes into the same pair — diff --git a/packages/drivers/driver-turso/src/remote-transport-compile-refusal-seam.test.ts b/packages/drivers/driver-turso/src/remote-transport-compile-refusal-seam.test.ts index 4a41420ec1e..26fa6fcbc91 100644 --- a/packages/drivers/driver-turso/src/remote-transport-compile-refusal-seam.test.ts +++ b/packages/drivers/driver-turso/src/remote-transport-compile-refusal-seam.test.ts @@ -150,6 +150,21 @@ const DOORS: readonly Door[] = [ secrets: [POLICY_COL, SECRET], klass: 'Operator "$exists" in this filter requires a boolean comparand', }, + // ── #20444: the staged `$empty` operator, born in the seam ────────────────── + { + builder: 'nonBooleanEmptyComparand', + where: () => ({ [POLICY_COL]: { $empty: SECRET } }), + secrets: [POLICY_COL, SECRET], + klass: 'Operator "$empty" in this filter requires a boolean comparand', + }, + { + // A bare transport holds no declaration for any field, so the flag on a + // real column is refused here — the standalone half of the rule. + builder: 'undeclaredEmptyOperatorField', + where: () => ({ [POLICY_COL]: { $empty: true } }), + secrets: [POLICY_COL], + klass: 'targets a field whose declaration this driver does not hold', + }, { builder: 'uncompilableComparand', where: () => ({ [POLICY_COL]: { $contains: { k: SECRET } } }), @@ -465,6 +480,9 @@ describe('[#20039] TursoDriver LOCAL and REMOTE withhold these classes alike', ( ['undeclared combinator', () => ({ [UNDECLARED_KEY]: 'x' }), UNDECLARED_KEY], // [#20041] Written on both compilers as one sentence from the start. ['U+0000 in a pattern', () => ({ [POLICY_COL]: { $like: `${SECRET}${String.fromCharCode(0x00)}` } }), SECRET], + // [#20444] …and so were the staged `$empty` operator's two refusals. + ['$empty non-boolean flag', () => ({ [POLICY_COL]: { $empty: SECRET } }), SECRET], + ['$empty on an undeclared field', () => ({ secret_undeclared_col: { $empty: true } }), 'secret_undeclared_col'], ]; for (const [label, where, secret] of SHARED) { diff --git a/packages/drivers/driver-turso/src/remote-transport.ts b/packages/drivers/driver-turso/src/remote-transport.ts index 3d005ad41e3..068a2eef16f 100644 --- a/packages/drivers/driver-turso/src/remote-transport.ts +++ b/packages/drivers/driver-turso/src/remote-transport.ts @@ -34,6 +34,10 @@ import { // on what a pattern means (the fork `turso-local-remote-*` suites exist to catch). // [#20041] And the U+0000 gate beside the dangling-escape one. import { hasDanglingLikeEscape, hasNulInLikePattern, likePatternToGlobPattern } from '@objectstack/spec/data'; +// [#20444] The `$empty` operator's ONE expansion — the field's declared row of +// the ruled 「is empty」 table (ruling A on #20399, record 5865693155), asked of +// the spec per compile rather than restated here. +import { expandEmptyOperator, type ValueShapeFieldDef } from '@objectstack/spec/data'; // [#8220] The read-scope provenance mark's consumer half — same resolution the // SqlDriver family applies, so which transport answered stays unobservable. import { resolveFilterSubtreeProvenance } from '@objectstack/spec/data'; @@ -206,6 +210,11 @@ const SUPPORTED_FILTER_OPERATORS = [ '$ilike', '$null', '$exists', + // [#20444] The staged emptiness flag, declared by `FieldOperatorsSchema` and + // compiled here by the field's declared row — listed for the reason `$like` + // is: the LOCAL twin compiles it, and a transport refusing what its own local + // mode answers is the fork the `turso-local-remote-*` suites exist to prevent. + '$empty', ] as const; /** @@ -440,6 +449,10 @@ function nullValueSatisfiesOperator(op: string, value: unknown): boolean { // `$null: true` and `$exists: false` are the same question, so these two // arms are each other's MIRROR, not each other's copy. case '$exists': return value === false; + // [#20444] Null counts as empty on every row of the ruled table, so a NULL + // column satisfies `$empty: true` and fails its complement — the answer + // `driver-sql`'s table gives, read by identity like the two above. + case '$empty': return value === true; // Negative-polarity set / substring tests hold vacuously for an absent // value — the #5298 half of the ruling. case '$nin': return true; @@ -459,6 +472,9 @@ function operatorIsNullTotal(op: string, value: unknown): boolean { // Compile to `IS NULL` / `IS NOT NULL` — two-valued by construction. case '$null': case '$exists': + // [#20444] Spells its NULL case out in both polarities, so it is TOTAL — + // see {@link RemoteTransport.pushEmptyOperator}. + case '$empty': return true; // A null comparand makes these null PREDICATES too, not comparisons — see // the `$eq` / `$ne` arms of the emitter. [#6050] `undefined` dropped here @@ -1130,6 +1146,18 @@ export type FilterColumnSqlResolver = ( */ export type NonTextColumnResolver = (object: string, field: string) => boolean; +/** + * [#20444] A field's DECLARED value shape — its `type` and `multiple`, the + * slice `expandEmptyOperator` reads — or `undefined` when the declaration is + * not held. Injected by TursoDriver exactly the way + * {@link NonTextColumnResolver} is, and answered from the same registration + * (`registerRemoteFieldMetadata` → `SqlDriver.registerExternalObject` fills + * `SqlDriver.valueShapeFields`), so this transport and its local twin read one + * declaration. Absent (a transport driven standalone), every field reads as + * undeclared and `$empty` is refused rather than guessed. + */ +export type DeclaredValueShapeResolver = (object: string, field: string) => ValueShapeFieldDef | undefined; + /** * Remote transport that executes all queries via @libsql/client. * @@ -1171,6 +1199,13 @@ export class RemoteTransport { */ private nonTextColumn: NonTextColumnResolver | null = null; + /** + * [#20444] The driver's declared value shape for a field — see + * {@link setDeclaredValueShapeResolver}. Absent means "no declaration is + * held", and `$empty` is refused rather than answered by a guessed row. + */ + private declaredValueShape: DeclaredValueShapeResolver | null = null; + /** * [#7929] Where the withheld half of a redacted refusal is written. * @@ -1320,6 +1355,18 @@ export class RemoteTransport { this.nonTextColumn = resolver; } + /** + * [#20444] Hand this transport the driver's declared value shape per field, + * so `$empty` compiles the field's declared row of the ruled table + * ({@link pushEmptyOperator}). Same shape as + * {@link setNonTextColumnResolver} and for the same reason: the declaration + * lives on the driver, and this transport asks rather than re-deriving a row + * from a value it will only see at run time. + */ + setDeclaredValueShapeResolver(resolver: DeclaredValueShapeResolver): void { + this.declaredValueShape = resolver; + } + /** * Get the current @libsql/client instance. */ @@ -3277,6 +3324,16 @@ export class RemoteTransport { // `$null: true` and `$exists: false` are one question asked twice. clauses.push(`${column} IS ${opValue === false ? 'NULL' : 'NOT NULL'}`); break; + // [#20444] `$empty` — the staged emptiness flag, answered by the + // field's DECLARED row of the ruled table on the plain column (a + // presence question like the two above, so no storage form applies). + // Refused, as its two siblings are, unless the comparand is boolean. + case '$empty': + if (typeof opValue !== 'boolean') { + throw this.nonBooleanEmptyComparand(object, key, opValue, value); + } + this.pushEmptyOperator(clauses, args, object, key, column, opValue, value); + break; default: // Declared = enforced. This arm used to compile ANY unknown // operator to `column = ?` against its comparand — so a @@ -3872,6 +3929,118 @@ export class RemoteTransport { ); } + /** + * [#20444] The error for an `$empty` whose comparand is not a boolean — + * `driver-sql`'s `nonBooleanEmptyComparandError`, one package over, in this + * transport's location convention. `FieldOperatorsSchema` declares + * `$empty: z.boolean()`, and the spec's save door refuses anything else. + */ + private nonBooleanEmptyComparand(object: string, field: string, value: unknown, subtree?: unknown): Error { + const shown = value === null ? 'null' : value === undefined ? 'undefined' : describeValue(value); + return this.withheldRefusal( + '[RemoteTransport] Operator "$empty" in this filter requires a boolean comparand (true or ' + + 'false). @objectstack/spec FieldOperatorsSchema declares $empty as a boolean, and a ' + + 'non-boolean is refused rather than coerced: true asks for the empty rows, false for their ' + + 'exact complement, and any other value would land on whichever side a compiler defaults to. ' + + 'The field it was aimed at and the value it received are withheld from the message; the ' + + 'full diagnostic is in the server log.', + subtree, + `[RemoteTransport] Operator "$empty" on field "${field}" requires a boolean comparand (true or ` + + `false). Received ${shown} (${preview(value)}) at '${object}.${field}'.$empty. ` + + `@objectstack/spec FieldOperatorsSchema declares $empty as a boolean: true asks for the ` + + `empty rows, false for their exact complement.`, + ); + } + + /** + * [#20444] Compile `{ field: { $empty: true | false } }` by the field's + * DECLARED row of the ruled 「is empty」 table — ruling B on #20311 (record + * 5861435168), spelled as this operator by ruling A on #20399 (record + * 5865693155) — through the spec's one expansion, `expandEmptyOperator`: + * + * | declared row | `$empty: true` | `$empty: false` — the exact complement | + * |---|---|---| + * | `null_only` | `col IS NULL` | `col IS NOT NULL` | + * | `text` | `(col IS NULL OR col = ?)`, `''` bound | `(col IS NOT NULL AND col <> ?)` | + * | `multi_value` | `(col IS NULL OR L)` | `(col IS NOT NULL AND NOT L)` | + * + * The SQL is the local twin's (`SqlDriver.applyEmptyOperator`) on its SQLite + * dialect — libSQL IS SQLite, where a multi-value field is a TEXT column + * holding JSON — so `L` is `driver-sql`'s SQLite construct, `json_valid` + * asked first inside a lazily evaluated `CASE` so a malformed cell answers + * FALSE rather than failing the statement, `json_type` beside + * `json_array_length` so a non-array JSON value is not an empty list. The + * `turso-local-remote-*` parity suites hold the two faces to one row set. + * + * Every clause is parenthesised as ONE conjunct — this transport joins a + * node's clauses with a bare ` AND `, so a loose `OR` would bind looser than + * it and widen the filter — and TOTAL (never UNKNOWN), so the `$not` rewrite + * needs no guard for it ({@link operatorIsNullTotal}). + * + * Refused, before anything is pushed: a field whose declaration the driver + * does not hold (or a transport nobody handed the resolver), because a row + * of the table cannot be read off a value this transport sees only at run + * time. + */ + private pushEmptyOperator( + clauses: string[], + args: any[], + object: string, + field: string, + column: string, + empty: boolean, + subtree: unknown, + ): void { + const shape = this.declaredValueShape?.(object, field); + if (!shape) throw this.undeclaredEmptyOperatorField(object, field, subtree); + const expansion = expandEmptyOperator(shape); + switch (expansion.arm) { + case 'null_only': + clauses.push(`${column} IS ${empty ? 'NULL' : 'NOT NULL'}`); + return; + case 'text': + clauses.push(empty ? `(${column} IS NULL OR ${column} = ?)` : `(${column} IS NOT NULL AND ${column} <> ?)`); + args.push(''); + return; + case 'multi_value': { + const list = + `(CASE WHEN json_valid(${column}) THEN json_type(${column}) = 'array' ` + + `AND json_array_length(${column}) = 0 ELSE 0 END)`; + clauses.push(empty ? `(${column} IS NULL OR ${list})` : `(${column} IS NOT NULL AND NOT ${list})`); + return; + } + default: { + // A closed union of three rows; a fourth is a spec change this + // transport was not taught, and it must fail loudly. + const unknownArm: never = expansion.arm; + throw new Error(`[RemoteTransport] no $empty arm for the declared row ${JSON.stringify(unknownArm)}`); + } + } + } + + /** + * [#20444] `$empty` aimed at a field whose declaration this transport was not + * handed — `driver-sql`'s `undeclaredEmptyOperatorFieldError`, whose withheld + * sentence this one is behind the `[RemoteTransport]` prefix (the declaration + * the caller is missing is the DRIVER's registry, which this transport reads), + * in this transport's location convention. + */ + private undeclaredEmptyOperatorField(object: string, field: string, subtree?: unknown): Error { + const why = + "What counts as empty is the field's DECLARED row of the ruled table — null or '' for a " + + 'text-like type, null or [] for a multi-value field, null only for every other type — so ' + + 'the operator is refused rather than guessed. Filter on a declared field, or use "$null" for ' + + '"has no value".'; + return this.withheldRefusal( + '[RemoteTransport] Operator "$empty" in this filter targets a field whose declaration this ' + + `driver does not hold (no declared type). ${why} The field is withheld from the message; ` + + 'the full diagnostic is in the server log.', + subtree, + `[RemoteTransport] Operator "$empty" on field '${object}.${field}' targets a field whose ` + + `declaration this driver does not hold (no declared type). ${why}`, + ); + } + /** * The error for an operator this transport does not compile. * diff --git a/packages/drivers/driver-turso/src/turso-20444-empty-operator.test.ts b/packages/drivers/driver-turso/src/turso-20444-empty-operator.test.ts new file mode 100644 index 00000000000..f04aad45514 --- /dev/null +++ b/packages/drivers/driver-turso/src/turso-20444-empty-operator.test.ts @@ -0,0 +1,146 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20444] The staged `$empty` operator on BOTH of TursoDriver's faces — the + * local transport (which inherits `SqlDriver.applyFilterCondition`) and the + * remote one (`RemoteTransport.buildWhereSQL`, an independent compiler) — held + * to one row set, by the field's DECLARED row of the ruled 「is empty」 table + * (ruling A on #20399, record 5865693155; the spec's `expandEmptyOperator`): + * text-like = null or `''`; multi-value = null or `[]`; every other type = null + * only; `$empty: false` the exact complement. + * + * The remote transport keeps no schema: the driver hands it the declaration + * (`setDeclaredValueShapeResolver`, answered from the registration + * `registerRemoteFieldMetadata` performs), and a transport nobody handed it to + * refuses the operator rather than guessing a row. + * + * libSQL IS SQLite, so `makeLibsqlSqliteStub` gives the remote transport real + * SQLite semantics with no network — the same stub the remote conformance + * suites use. + */ + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import type { FilterCondition } from '@objectstack/spec/data'; +import { TursoDriver } from './turso-driver.js'; +import { RemoteTransport } from './remote-transport.js'; +import { makeLibsqlSqliteStub, type LibsqlSqliteStub } from './libsql-sqlite-stub.testkit.js'; + +const OBJECT = { + name: 'os20444_empty', + fields: { + title: { type: 'text' }, + tags: { type: 'tags' }, + score: { type: 'number' }, + }, +}; + +const ROWS = [ + { id: 'r1', title: 'x', tags: ['a'], score: 5 }, + { id: 'r2', title: '', tags: [], score: 0 }, + { id: 'r3', title: null, tags: null, score: null }, + { id: 'r4', title: ' ', tags: ['a', 'b'], score: -1 }, +]; + +/** Each filter and the ids it must select, on both faces. */ +const CASES: Array<{ where: FilterCondition; expected: string[] }> = [ + { where: { title: { $empty: true } }, expected: ['r2', 'r3'] }, + { where: { title: { $empty: false } }, expected: ['r1', 'r4'] }, + { where: { tags: { $empty: true } }, expected: ['r2', 'r3'] }, + { where: { tags: { $empty: false } }, expected: ['r1', 'r4'] }, + { where: { score: { $empty: true } }, expected: ['r3'] }, + { where: { score: { $empty: false } }, expected: ['r1', 'r2', 'r4'] }, + { where: { $not: { title: { $empty: true } } }, expected: ['r1', 'r4'] }, + { where: { $not: { tags: { $empty: false } } }, expected: ['r2', 'r3'] }, + { where: { $not: { score: { $empty: true } } }, expected: ['r1', 'r2', 'r4'] }, + { where: { $or: [{ score: 5 }, { tags: { $empty: true } }] }, expected: ['r1', 'r2', 'r3'] }, + { where: { $and: [{ title: { $empty: false } }, { tags: { $empty: false } }] }, expected: ['r1', 'r4'] }, + { where: { title: { $empty: false, $ne: 'x' } }, expected: ['r4'] }, +]; + +const NO_AUDIT = { bypassTenantAudit: true }; + +async function refusal(run: () => Promise): Promise<{ code?: string; status?: number } | 'answered'> { + try { + await run(); + } catch (err) { + return { code: (err as { code?: string }).code, status: (err as { status?: number }).status }; + } + return 'answered'; +} + +describe('[#20444] TursoDriver — $empty on the local and the remote face', () => { + let local: TursoDriver; + let remote: TursoDriver; + let stub: LibsqlSqliteStub; + const ids = async (driver: TursoDriver, where: FilterCondition) => + (await driver.find(OBJECT.name, { where }, NO_AUDIT)).map((r) => String(r.id)).sort(); + + beforeAll(async () => { + local = new TursoDriver({ url: ':memory:' }); + expect(local.transportMode).toBe('local'); + await local.initObjects([OBJECT]); + + stub = makeLibsqlSqliteStub(); + remote = new TursoDriver({ url: 'libsql://os20444.turso.io', client: stub as never }); + await remote.connect(); + expect(remote.transportMode).toBe('remote'); + await remote.syncSchema(OBJECT.name, OBJECT); + + for (const row of ROWS) { + await local.create(OBJECT.name, { ...row }, NO_AUDIT); + await remote.create(OBJECT.name, { ...row }, NO_AUDIT); + } + }); + + afterAll(async () => { + await local?.disconnect(); + await remote?.disconnect(); + stub?.close(); + }); + + it('the fixture landed on both faces', async () => { + expect(await ids(local, {})).toEqual(['r1', 'r2', 'r3', 'r4']); + expect(await ids(remote, {})).toEqual(['r1', 'r2', 'r3', 'r4']); + }); + + for (const c of CASES) { + it(`${JSON.stringify(c.where)} → ${JSON.stringify(c.expected)} on both faces`, async () => { + expect(await ids(local, c.where), 'local').toEqual(c.expected); + expect(await ids(remote, c.where), 'remote').toEqual(c.expected); + }); + } + + it('count() agrees with find() on the remote face', async () => { + for (const c of CASES) { + expect(await remote.count(OBJECT.name, { where: c.where }, NO_AUDIT), JSON.stringify(c.where)).toBe(c.expected.length); + } + }); + + it('a field with no declaration is refused on both faces, never guessed', async () => { + for (const driver of [local, remote]) { + expect(await refusal(() => driver.find(OBJECT.name, { where: { id: { $empty: true } } }, NO_AUDIT))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + } + }); + + it('a non-boolean flag is refused on both faces', async () => { + for (const driver of [local, remote]) { + expect(await refusal(() => driver.find(OBJECT.name, { where: { title: { $empty: 'yes' as never } } }, NO_AUDIT))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + } + }); + + it('a RemoteTransport nobody handed the declaration refuses $empty rather than guessing a row', async () => { + const bare = new RemoteTransport(); + let executed = false; + bare.setClient({ + execute: async () => { + executed = true; + return { rows: [], columns: [] }; + }, + } as never); + expect(await refusal(() => bare.find(OBJECT.name, { where: { title: { $empty: true } } } as never))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + expect(executed, 'the refusal came before any statement ran').toBe(false); + }); +}); diff --git a/packages/drivers/driver-turso/src/turso-driver.ts b/packages/drivers/driver-turso/src/turso-driver.ts index 3db777a5eb7..cd680a22392 100644 --- a/packages/drivers/driver-turso/src/turso-driver.ts +++ b/packages/drivers/driver-turso/src/turso-driver.ts @@ -1590,6 +1590,14 @@ export class TursoDriver extends SqlDriver { this.isNonTextColumn(object, field), ); + // [#20444] The declaration `$empty` expands, handed down the same way: + // `registerRemoteFieldMetadata` → `registerExternalObject` fills the SAME + // `valueShapeFields` registry the local compiler reads, keyed by object + // name, so both transports answer `$empty` by one declared row. + this.remoteTransport.setDeclaredValueShapeResolver((object, field) => + this.declaredValueShape(object, field), + ); + // [#7929] The server-side half of a REDACTED filter refusal. The remote // compiler withholds the operands of a cross-field comparison for the // same reason the inherited local one does — an RLS rule's columns are @@ -2543,6 +2551,9 @@ export class TursoDriver extends SqlDriver { case '$regex': case '$null': case '$exists': + // [#20444] A presence flag like the two above — its boolean is not a + // value of the column, so the temporal coercion keeps its hands off. + case '$empty': out[op] = raw; break; default: diff --git a/packages/formula/src/matches-filter-empty-operator.test.ts b/packages/formula/src/matches-filter-empty-operator.test.ts new file mode 100644 index 00000000000..aad3aa708a5 --- /dev/null +++ b/packages/formula/src/matches-filter-empty-operator.test.ts @@ -0,0 +1,76 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20444] The staged `$empty` operator on `matchesFilterCondition` — the RLS + * write-side `check` evaluator. + * + * This face judges a RECORD, not a declaration, so ruling A on #20399 (record + * 5865693155) gives it the BY-VALUE reading: null, a missing key, `''` and `[]` + * are empty (the spec's `isEmptyFilterValue`), and `$empty: false` is the exact + * complement. It differs from the declared-type faces only on a stored state + * the declaration does not predict — `''` in a non-text column, the write-door + * class #20308 closed. + * + * Before its arm, a DECLARED `$empty` got this face's silent `false` for every + * record and both flags — the defect the module header says a declared + * operator must never get. The first test pins the arm by requiring both + * answers from both flags. + */ + +import { describe, expect, it } from 'vitest'; +import type { FilterCondition } from '@objectstack/spec/data'; +import { matchesFilterCondition } from './matches-filter'; + +const RECORDS: Array> = [ + { id: 'r1', title: 'x', tags: ['a'], score: 5 }, + { id: 'r2', title: '', tags: [], score: 0 }, + { id: 'r3', title: null, tags: null, score: null }, + { id: 'r4', title: ' ', tags: ['a', 'b'], score: -1 }, + { id: 'r5' }, +]; + +const ids = (filter: FilterCondition) => + RECORDS.filter((r) => matchesFilterCondition(r, filter)).map((r) => String(r.id)); + +describe('[#20444] matchesFilterCondition — $empty, judged by value', () => { + it('is ANSWERED, not given the silent false: each flag admits some record and refuses another', () => { + for (const flag of [true, false]) { + const admitted = ids({ title: { $empty: flag } }); + expect(admitted.length, `$empty: ${flag}`).toBeGreaterThan(0); + expect(admitted.length, `$empty: ${flag}`).toBeLessThan(RECORDS.length); + } + }); + + it("text: null, a missing key and '' are empty — ' ' is a value", () => { + expect(ids({ title: { $empty: true } })).toEqual(['r2', 'r3', 'r5']); + expect(ids({ title: { $empty: false } })).toEqual(['r1', 'r4']); + }); + + it('a list: null, a missing key and [] are empty', () => { + expect(ids({ tags: { $empty: true } })).toEqual(['r2', 'r3', 'r5']); + expect(ids({ tags: { $empty: false } })).toEqual(['r1', 'r4']); + }); + + it('a number: 0 is a value; only null and a missing key are empty', () => { + expect(ids({ score: { $empty: true } })).toEqual(['r3', 'r5']); + expect(ids({ score: { $empty: false } })).toEqual(['r1', 'r2', 'r4']); + }); + + it("the one reading a declared-type face does not share: '' in a number column counts as empty here", () => { + expect(matchesFilterCondition({ score: '' }, { score: { $empty: true } })).toBe(true); + }); + + it('nests under $and / $or / $not, and ANDs with a sibling operator on the same field', () => { + expect(ids({ $not: { title: { $empty: true } } })).toEqual(['r1', 'r4']); + expect(ids({ $not: { tags: { $empty: false } } })).toEqual(['r2', 'r3', 'r5']); + expect(ids({ $or: [{ score: 5 }, { tags: { $empty: true } }] })).toEqual(['r1', 'r2', 'r3', 'r5']); + expect(ids({ $and: [{ title: { $empty: false } }, { tags: { $empty: false } }] })).toEqual(['r1', 'r4']); + expect(ids({ title: { $empty: false, $ne: 'x' } })).toEqual(['r4']); + }); + + it('a non-boolean flag denies the write — this face\'s answer to an unevaluable condition', () => { + for (const bad of ['true', 1, null, { $field: 'title' }]) { + expect(ids({ title: { $empty: bad as never } }), JSON.stringify(bad)).toEqual([]); + } + }); +}); diff --git a/packages/formula/src/matches-filter.ts b/packages/formula/src/matches-filter.ts index 8e920434e12..b0191336762 100644 --- a/packages/formula/src/matches-filter.ts +++ b/packages/formula/src/matches-filter.ts @@ -47,6 +47,19 @@ * that test a silent `false` for `$like` would have been the same defect * under a new name. * + * [#20444] The claim was FALSE for a while, and is true again because of an + * arm, not a rewording. `$empty` was declared by `FieldOperatorsSchema` / + * `SpecialOperatorSchema` (#20311) and staged out of `FILTER_OPERATORS` like + * `$like`, but its arms were placed in per-lane cards rather than in the + * declaring PR — so from that declaration until this face's arm landed, an + * RLS `check` written with `$empty` was answered by the silent `false` + * below, for every record and both flags: the defect this paragraph names. + * It now has its arm in {@link evalOp}, judged by the stored value (this + * face's reading, ruling A on #20399). Today the declared-but-staged names + * are `$like`, `$ilike` and `$empty`, and each is answered here; a name the + * protocol declares NEXT is owed an arm here by the PR that lets an author + * write it, or by the lane card its staging names. + * * What stays open, deliberately and on the record: a RETIRED spelling * (`$regex` / `$options`) still gets the silent `false` here while the other five * faces print `RETIRED_FILTER_OPERATORS`' prescription naming `$icontains`. That @@ -106,6 +119,9 @@ import { nextUtcCalendarDay, utcInstantMs, asciiCaseInsensitiveContains } from ' // one to `LIKE`/`GLOB`, and a translation written twice would agree on the day // it was typed and never again. import { matchesLikePattern } from '@objectstack/spec/data'; +// [#20444] `$empty`'s value-level half — the spec's one definition of what a +// stored value counts as empty for a face that reads no field declaration. +import { isEmptyFilterValue } from '@objectstack/spec/data'; import { StandardErrorCode } from '@objectstack/spec/api'; /** @@ -724,6 +740,32 @@ function evalOp(actual: unknown, op: string, raw: unknown, record: Record> = [ + { id: 'g1', region: 'north', n: 3, top: 'z', codes: ['a'] }, + { id: 'g2', region: '', n: 0, top: null, codes: [] }, + { id: 'g3', region: null, n: 7, top: '', codes: null }, + { id: 'g4', region: 'south', n: 1, top: 'b' }, +]; + +const ids = (having: FilterCondition) => applyHaving(ROWS, having).map((r) => String(r.id)); + +function refusal(run: () => unknown): { code?: string; status?: number } | 'answered' { + try { + run(); + } catch (err) { + return { code: (err as { code?: string }).code, status: (err as { status?: number }).status }; + } + return 'answered'; +} + +describe('[#20444] having — $empty, judged by value over the aggregated row', () => { + it('a numeric aggregate holding 0 is NOT empty; only a missing value would be', () => { + expect(ids({ n: { $empty: true } })).toEqual([]); + expect(ids({ n: { $empty: false } })).toEqual(['g1', 'g2', 'g3', 'g4']); + }); + + it("a groupBy text column holding '' IS empty, as null is", () => { + expect(ids({ region: { $empty: true } })).toEqual(['g2', 'g3']); + expect(ids({ region: { $empty: false } })).toEqual(['g1', 'g4']); + }); + + it("a max over no values (null) and one landing on '' are empty", () => { + expect(ids({ top: { $empty: true } })).toEqual(['g2', 'g3']); + }); + + it('a list value: [] and null are empty, and a column the row does not carry is empty too', () => { + expect(ids({ codes: { $empty: true } })).toEqual(['g2', 'g3', 'g4']); + expect(ids({ codes: { $empty: false } })).toEqual(['g1']); + }); + + it('nests under $and / $or / $not, and ANDs with a sibling operator on the same column', () => { + expect(ids({ $not: { region: { $empty: true } } })).toEqual(['g1', 'g4']); + expect(ids({ $or: [{ n: { $gt: 5 } }, { codes: { $empty: false } }] })).toEqual(['g1', 'g3']); + expect(ids({ $and: [{ region: { $empty: false } }, { n: { $gte: 1 } }] })).toEqual(['g1', 'g4']); + expect(ids({ region: { $empty: false, $ne: 'north' } })).toEqual(['g4']); + }); + + it('the per-aggregation filter shares the walker and the reading', () => { + const source = [{ stage: '' }, { stage: 'won' }, { stage: null }, {}]; + const kept = source.filter((row) => matchesAggregationFilter(row, { stage: { $empty: true } }, 0)); + expect(kept).toEqual([{ stage: '' }, { stage: null }, {}]); + }); + + it('a non-boolean flag is REFUSED — judged once, before any row, and per row as the floor', () => { + expect(refusal(() => assertHavingIsEvaluable({ n: { $empty: 'yes' } }, ['id', 'region', 'n', 'top', 'codes']))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + expect(refusal(() => applyHaving(ROWS, { n: { $empty: 1 as never } }))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + // An empty grouped set still refuses: the verdict is the filter's, not the data's. + expect(refusal(() => assertHavingIsEvaluable({ $or: [{ n: 1 }, { n: { $empty: null } }] }, ['n']))) + .toEqual({ code: 'INVALID_FILTER', status: 400 }); + }); + + it('a well-formed $empty passes the whole-clause judgement', () => { + expect(refusal(() => assertHavingIsEvaluable({ $not: { region: { $empty: false } } }, ['region']))).toBe('answered'); + }); +}); diff --git a/packages/objectql/src/having-filter.ts b/packages/objectql/src/having-filter.ts index a0e3b850859..746b76bd294 100644 --- a/packages/objectql/src/having-filter.ts +++ b/packages/objectql/src/having-filter.ts @@ -143,6 +143,9 @@ import { asciiCaseInsensitiveContains } from '@objectstack/spec/data'; // evaluator — the lift `@objectstack/formula` applies to the same pairing — so a // `Date` bound here compares the way the same bound in a `where` does. import { utcInstantMs } from '@objectstack/spec/data'; +// [#20444] `$empty`'s value-level half — the spec's one definition of what a +// stored value counts as empty for a face that judges by value. +import { isEmptyFilterValue } from '@objectstack/spec/data'; // [#20176] The storage rule a temporal column puts a value in — ONE function, // shared with `driver-sql`'s and `driver-memory`'s `where` — and the whole-day // reading of a bare-day upper bound on a `datetime` column (ADR-0053 D-D), from @@ -209,10 +212,17 @@ export function aggregationFilterClause(index: number): FilterClause { // by all six JS evaluation faces, so this face needs no fold of its own. The // #5499 freeze was lifted for this operator as a sanctioned one-off (maintainer // ruling, 2026-08-08), strictly for semantic parity. +// +// [#20444] `$empty` IS here — the staged emptiness flag (declared by +// `FieldOperatorsSchema`, out of `FILTER_OPERATORS` until its flip card), with +// its arm in {@link checkCondition} and its comparand gate beside +// `$icontains`' — judged BY VALUE, the reading ruling A on #20399 (record +// 5865693155) gives this face. See {@link emptyFlagComparandError}. const CONDITION_OPERATORS = [ '$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$between', '$in', '$nin', '$exists', '$null', '$contains', '$notContains', '$startsWith', '$endsWith', '$icontains', + '$empty', ] as const; /** @@ -319,6 +329,23 @@ function unknownOperator( ); } +/** + * [#20444] `$empty` received a comparand that is not a boolean. + * `FieldOperatorsSchema` declares `$empty: z.boolean()`: `true` asks for the + * empty rows, `false` for their exact complement. A third value is refused + * rather than read — this face refuses the malformations it can see, where a + * two-branch reading would silently constrain nothing (the lenient `$null` + * arm's standing hazard). + */ +function emptyFlagComparandError(field: string, value: unknown, path: string): Error { + const shown = JSON.stringify(value) ?? String(value); + return invalidFilterError( + `Operator "$empty" on field "${field}" at ${path} requires a boolean comparand (true or false), ` + + `received ${shown}. @objectstack/spec FieldOperatorsSchema declares $empty as a boolean: true ` + + `asks for the empty rows, false for their exact complement.`, + ); +} + /** * [#7158] `$icontains` received a comparand that is not a non-empty string. * @@ -596,6 +623,9 @@ function offsetPairViolation( */ const NO_VALUE_ANSWERED_BY_OPERATOR: ReadonlySet = new Set([ '$exists', '$ne', '$null', '$nin', '$notContains', + // [#20444] About the absence too: a column the row does not carry is EMPTY, + // so `$empty: true` must reach its arm rather than the exit's `false`. + '$empty', ]); /** @@ -1041,6 +1071,12 @@ function assertConditionIsEvaluable( if (op === '$icontains' && (typeof target !== 'string' || target === '')) { throw icontainsComparandError(field, target, `${path}.${op}`); } + // [#20444] Before the vocabulary check for the same reason as the gate + // above: the flag is a known operator, and its only malformation is a + // comparand that is not a boolean. + if (op === '$empty' && typeof target !== 'boolean') { + throw emptyFlagComparandError(field, target, `${path}.${op}`); + } if (!(CONDITION_OPERATORS as readonly string[]).includes(op)) { throw unknownOperator(op, 'condition', keys, scope.clause); } @@ -1348,6 +1384,11 @@ function checkCondition( if (op === '$icontains' && (typeof target !== 'string' || target === '')) { throw icontainsComparandError(field, target, `${path}.${op}`); } + // [#20444] The flag's shape is the filter's, not the row's — above the + // no-value exit for the #7158 reason the gate above gives. + if (op === '$empty' && typeof target !== 'boolean') { + throw emptyFlagComparandError(field, target, `${path}.${op}`); + } if (value === undefined && !NO_VALUE_ANSWERED_BY_OPERATOR.has(op)) return false; // [#20099] A `{ $field }` reference as the whole comparand of a scalar // comparison is RESOLVED against this row. The arms below would compare the @@ -1397,6 +1438,18 @@ function checkCondition( if (target === true && value != null) return false; if (target === false && value == null) return false; break; + // [#20444] The staged emptiness flag, judged BY VALUE — the aggregated + // row carries no field declaration of its own, so this face takes the + // reading ruling A on #20399 gives it: null, a missing column, `''` and + // `[]` are empty (the spec's `isEmptyFilterValue`, not a copy of it), and + // `false` is the exact complement. So `0` from a `count` / `sum` is NOT + // empty — a group with no rows to count is a zero, not a missing value — + // and a groupBy text column holding `''` IS empty, the row a declared + // text field takes too. The one divergence from a declared-type face is + // `''` in a non-text column, the write-door class #20308 closed. + case '$empty': + if (isEmptyFilterValue(value) !== (target === true)) return false; + break; case '$contains': if (typeof value !== 'string' || !value.includes(target)) return false; break; // [#5905] The mirror of `$contains`, NOT its copy-with-a-negated-test. // `$contains` fails a non-string value because "contains" cannot hold for diff --git a/packages/services/service-analytics/src/__tests__/native-sql-filter-logic-conformance.test.ts b/packages/services/service-analytics/src/__tests__/native-sql-filter-logic-conformance.test.ts index 4cb26852ce5..65372082cb0 100644 --- a/packages/services/service-analytics/src/__tests__/native-sql-filter-logic-conformance.test.ts +++ b/packages/services/service-analytics/src/__tests__/native-sql-filter-logic-conformance.test.ts @@ -107,6 +107,10 @@ describe('NativeSQLStrategy — filter logic conformance', () => { ctx = { getCube: (name: string) => (name === 'logic' ? CUBE : undefined), + // [#20444] The fixture's columns, declared: every one a `text` field. + // The table's `$empty` rows are answered by the field's DECLARED row, + // which this strategy reads from this hook and refuses without. + declaredValueShape: () => ({ type: 'text' }), queryCapabilities: () => ({ nativeSql: true, objectqlAggregate: false, inMemory: false }), // The strategy binds `$1`-style placeholders in ascending order, each // pushed immediately before it is referenced, so a positional rewrite to diff --git a/packages/services/service-analytics/src/__tests__/read-scope-sql-conformance.test.ts b/packages/services/service-analytics/src/__tests__/read-scope-sql-conformance.test.ts index 58c117e5d65..13e693b02a8 100644 --- a/packages/services/service-analytics/src/__tests__/read-scope-sql-conformance.test.ts +++ b/packages/services/service-analytics/src/__tests__/read-scope-sql-conformance.test.ts @@ -55,6 +55,14 @@ import { compileScopedFilterToSql } from '../read-scope-sql.js'; const ALIAS = 't'; +/** + * [#20444] The fixture's columns, declared: every one a `text` field. The + * table's `$empty` rows are answered by the field's DECLARED row, which this + * compiler reads from `declaredValueShape` and refuses without — so the sweep + * hands it the declaration a host answers from its field metadata. + */ +const FIXTURE_SHAPES = { declaredValueShape: () => ({ type: 'text' }) } as const; + /** Point sql.js at the `.wasm` shipped inside its own package (Node-safe). */ async function locateWasm(): Promise<((file: string) => string) | undefined> { try { @@ -106,7 +114,7 @@ describe('compileScopedFilterToSql — filter logic conformance', () => { for (const c of FILTER_LOGIC_CASES) { it(c.name, () => { - const { sql, params } = compileScopedFilterToSql(c.filter, ALIAS); + const { sql, params } = compileScopedFilterToSql(c.filter, ALIAS, FIXTURE_SHAPES); // The compiler returns a boolean expression, exactly as the analytics // query builder splices it — including the unparenthesized top level. // `''` is the compiler's TRUE (#5322: `{$and: []}` and an absorbed `$or` diff --git a/packages/spec/src/data/filter-logic-conformance.ts b/packages/spec/src/data/filter-logic-conformance.ts index c9b32acbaa5..fafac0c1c24 100644 --- a/packages/spec/src/data/filter-logic-conformance.ts +++ b/packages/spec/src/data/filter-logic-conformance.ts @@ -59,7 +59,8 @@ * * The predicates are deliberately boring: string equality, `$in` / `$nin`, * `$ne`, `$gte` / `$lt` on lexicographic strings, `$notContains` over plain - * substrings, and the value-presence pair `$null` / `$exists`. Dates, numeric + * substrings, the value-presence pair `$null` / `$exists`, and (since #20444) + * the staged emptiness flag `$empty` on the same nullable column. Dates, numeric * coercion, `LIKE` escaping and case sensitivity are still out — those * legitimately differ between a SQL engine and a JS matcher, and folding them * in would make the table unpassable rather than more useful. Keep it that way: a case belongs here only if @@ -74,6 +75,28 @@ * {@link FilterLogicRow.d} column carries it, and the eight `d`-column cases * below enforce it on every backend. * + * **`$empty` is IN, as of #20444, and its rows measure less than they look + * like.** The flag is answered by the field's DECLARED row of the ruled + * 「is empty」 table (text-like: null or `''`; multi-value: null or `[]`; every + * other type: null only — `expandEmptyOperator`, `./filter-empty-operator.ts`) + * on the faces that hold declarations, and by value (`isEmptyFilterValue`) on + * the ones that do not. This fixture stores neither `''` nor `[]`, so on it + * every row of the table and the by-value reading give ONE answer — which is + * what makes the rows below enrollable on every backend, and also why they do + * NOT pin the per-type rows: those are each face's own suite, over a text, a + * multi-value and a scalar column. What these rows DO pin is that every face + * has an arm (a face without one refuses, and the case goes red), that `$not` + * over it is total (no row lost to UNKNOWN), and that it composes with the + * combinators and with a sibling operator on the same field like any other + * predicate. + * + * ⚠️ A declared-type face REFUSES `$empty` on a field whose declaration it does + * not hold — there is no row to compile without one. So every harness must + * DECLARE the fixture's columns (any string type: `text` takes the text row, + * the driver-internal `string` the null-only row, and both answer this fixture + * identically); a harness that builds its table outside the driver's + * registration has to register the declaration beside it. + * * ⚠️ That answer — the INCLUDE direction — was reversed by a ruling on * 2026-08-10 and RE-AFFIRMED the same day, once the reversal's full cost had * been measured. So the four `d`-column cases below ARE the settled semantics, @@ -522,6 +545,56 @@ export const FILTER_LOGIC_CASES: readonly FilterLogicCase[] = [ note: 'The direction the ruling called the hardest live harm: a key-presence reading returns NOTHING here, silently emptying every "field is not set" scope. Enrolled beside its twin so a single-direction blind spot cannot rebuild.', }, + // ── The staged emptiness flag `$empty` (#20444) ─────────────────────────── + // + // Ruling A on #20399 (record 5865693155): `$empty: boolean`, answered by the + // field's declared row of the ruled table on the faces that hold field + // declarations and by value on the ones that do not. `d` stores a value or + // NULL and never `''` / `[]`, so every row of the table agrees here — see + // this file's header for what that does and does not pin. Staged: the + // engine's front door refuses the operator until its flip card adds it to + // `FILTER_OPERATORS`, so these cases reach each face directly. + { + name: '$empty true selects exactly the no-value rows', + filter: { d: { $empty: true } }, + expected: ['3', '4'], + note: '#20444: null is empty on every row of the ruled table. A face with no arm refuses, which is red here, never a silent answer.', + }, + { + name: '$empty false selects exactly the valued rows', + filter: { d: { $empty: false } }, + expected: ['1', '2'], + note: '#20444: the exact complement, so `$empty` is pinned as a partition of the table rather than one half of one.', + }, + { + name: '$not over $empty true returns the valued rows', + filter: { $not: { d: { $empty: true } } }, + expected: ['1', '2'], + note: '#20444: `$empty` spells its NULL case out, so it is never UNKNOWN; a three-valued `NOT (d IS NULL OR …)` that dropped a row would fail here.', + }, + { + name: '$not over $empty false returns the no-value rows', + filter: { $not: { d: { $empty: false } } }, + expected: ['3', '4'], + note: '#20444: the negation of the complement is the empty partition, rows 3-4 — the rows an unguarded `NOT (d IS NOT NULL AND …)` loses to UNKNOWN.', + }, + { + name: '$empty inside a $or branch OR-s with its sibling branch', + filter: { $or: [{ b: 'y' }, { d: { $empty: false } }] }, + expected: ['1', '2', '3'], + }, + { + name: '$empty inside a $and ANDs with its sibling', + filter: { $and: [{ b: 'zz' }, { d: { $empty: true } }] }, + expected: ['4'], + }, + { + name: '$empty ANDs with a sibling operator on the same field', + filter: { d: { $empty: false, $ne: 'v1' } }, + expected: ['2'], + note: '#20444: a face that lowers `$empty` beside the field\'s other operators must not let either overwrite the other.', + }, + // ── Shapes read scopes are actually written in ──────────────────────────── { name: 'read scope: own AND active, OR another owner\'s row',