feat(spec,objectql,plugin-security): one shared filter lowering, run once at the engine and RLS seams (#5930 step 2)

.changeset/5930-shared-filter-lowering.md

@@ -0,0 +1,26 @@
+---
+'@objectstack/spec': minor
+'@objectstack/objectql': patch
+'@objectstack/plugin-security': minor
+---
+
+feat(spec, objectql, plugin-security): one shared filter lowering, run once at the engine and RLS seams (ADR-0053 D-D1, amended)
+
+Clause-②: yes
+
+`@objectstack/spec/data` exports `lowerFilterCondition(filter, options?)` and its `FilterLoweringOptions` type. It is not exported from the package root entry. It is a pure `FilterCondition → FilterCondition` rewrite that applies three rules once:
+
+- `$between` becomes `$gte` its minimum and `$lte` its maximum.
+- A `$lte` whose comparand is a bare `YYYY-MM-DD` day becomes `$lt` the next day, in the calendar-string domain. On the last supported day (`9999-12-31`) a lone `$lte` becomes `{ $null: false }`, and a `$between` keeps only its minimum.
+- The NULL-polarity guards the drivers already compile. A `$ne` of a value, a `$nin` or a `$notContains` holds for a row with no value. Every leaf of a `$not` operand is made total.
+
+The rewrite is copy-on-write, idempotent and never refuses. A node it rewrites keeps its filter-subtree provenance mark. With `options.isDatetimeColumn` (a typed seam), the first two rules change only a declared `datetime` column. Without it they apply to every column.
+
+As ADR-0053 D-D1 (amended 2026-09-30) requires, the seams now run it once, after the comparand doors and after filter-token resolution:
+
+- **`@objectstack/objectql`** runs it on every filter position, typed by the object's declared fields. That covers `where` on `find`, `findOne`, `count`, `update` and `delete`, and `aggregate`'s `where`, `aggregations[i].filter` and `having`. `having` is typed by the aggregated row's columns, so `max` of a `datetime` field counts as a `datetime`. Drivers receive the lowered filter. A date macro such as `{today}` is resolved before the lowering reads it.
+- **`@objectstack/plugin-security`** runs it on every compiled RLS policy filter (`using` and `check`), right after the two comparand faces. `SecurityPlugin` now hands the compile seam the object's declared `datetime` columns (`RlsFieldGuard.datetime`). A guard without that set treats no column as `datetime`.
+
+Row answers stay the same on every driver. Each driver keeps its own copy of these rules, and every copy gives the same answer on lowered input. One result changes. The engine evaluates `aggregate`'s `aggregations[i].filter` and `having` itself, and that evaluator now treats a row or group with no value the way every driver's `where` already does. It no longer counts such a row in a `$between` on a `datetime` column. It now keeps such a row under a `$not` over an ordering such as `$lt`.
+
+Nothing is removed or renamed, and there is nothing to migrate.

packages/objectql/src/engine-filter-array-lowering.test.ts

@@ -673,8 +673,11 @@ describe('Door 2 lowers FilterArray to FilterCondition before the driver (#5158)
     await engine.find('deal', asFilterArrayQuery([['stage', 'in', ['won', 'lost']]]));
     expect(lastWhere()).toEqual({ stage: { $in: ['won', 'lost'] } });
 
+    // [ADR-0053 D-D1, amended — #5930] `$nin` reaches the driver with the
+    // shared lowering's NULL escape around it, the #5298 reading every face
+    // already gives it; the list itself is untouched.
     await engine.find('deal', asFilterArrayQuery([['stage', 'not_in', ['lost']]]));
-    expect(lastWhere()).toEqual({ stage: { $nin: ['lost'] } });
+    expect(lastWhere()).toEqual({ $and: [{ $or: [{ stage: { $null: true } }, { stage: { $nin: ['lost'] } }] }] });
 
     await engine.find('deal', asFilterArrayQuery([['amount', 'between', [5, 25]]]));
     expect(lastWhere()).toEqual({ amount: { $between: [5, 25] } });
@@ -686,7 +689,8 @@ describe('Door 2 lowers FilterArray to FilterCondition before the driver (#5158)
     await engine.find('deal', { where: { stage: { $in: [] } } });
     expect(lastWhere()).toEqual({ stage: { $in: [] } });
     await engine.find('deal', { where: { stage: { $nin: [] } } });
-    expect(lastWhere()).toEqual({ stage: { $nin: [] } });
+    // [ADR-0053 D-D1, amended — #5930] …inside the shared lowering's NULL escape.
+    expect(lastWhere()).toEqual({ $and: [{ $or: [{ stage: { $null: true } }, { stage: { $nin: [] } }] }] });
   });
 
   it('the gate does not re-judge list MEMBERS — that is #5234, on another face', async () => {
@@ -728,8 +732,10 @@ describe('Door 2 lowers FilterArray to FilterCondition before the driver (#5158)
   });
 
   it('a scalar on a NON-collection operator is untouched', async () => {
+    // [ADR-0053 D-D1, amended — #5930] `$ne` reaches the driver inside the
+    // shared lowering's NULL escape; the scalar comparand is untouched.
     await engine.find('deal', asFilterArrayQuery([['stage', '!=', 'won']]));
-    expect(lastWhere()).toEqual({ stage: { $ne: 'won' } });
+    expect(lastWhere()).toEqual({ $and: [{ $or: [{ stage: { $null: true } }, { stage: { $ne: 'won' } }] }] });
     // String bounds on a range comparison stay legal, and since #5685 the
     // declaration agrees: `FieldOperatorsSchema` now declares `$gt` as
     // number|Date|string|FieldReference, matching the ISO strings the showcase

packages/objectql/src/engine-number-comparand-declared-type-door.test.ts

@@ -52,6 +52,7 @@ import {
   NUMBER_COMPARAND_DOOR_FIXTURE_OBJECT,
   NUMBER_COMPARAND_DOOR_LIST_OPERATORS,
   NUMBER_COMPARAND_DOOR_SCALAR_OPERATORS,
+  lowerFilterCondition,
   type EngineAggregateOptions,
   type EngineQueryOptions,
   type FilterCondition,
@@ -68,6 +69,19 @@ import {
 
 const OBJECT = NUMBER_COMPARAND_DOOR_FIXTURE_OBJECT;
 
+/**
+ * [ADR-0053 D-D1, amended — #5930] What a driver receives is the door's output
+ * after the engine's shared lowering (the NULL-polarity guards on `$ne` / `$nin`
+ * / `$notContains`, the whole-day rule on a declared `datetime`), which runs
+ * after this door on every verb. The door adds nothing beyond that, so a pin on
+ * the driver's input compares against the same lowering of the door's answer.
+ */
+const lowered = (where: unknown): unknown =>
+  lowerFilterCondition(where, {
+    isDatetimeColumn: (column) =>
+      (NUMBER_COMPARAND_DOOR_FIXTURE.fields as Record<string, { type?: string } | undefined>)[column]?.type === 'datetime',
+  });
+
 interface SeenRead { ast: any }
 
 /** Minimal recording driver — the same witness shape as the sibling door suites. */
@@ -195,7 +209,7 @@ describe('[#20351] the number-comparand declared-type door at the engine collect
       const asWritten = JSON.stringify(filter);
       await expect(engine.find(OBJECT, { where: filter }), c.name).resolves.toBeDefined();
       expect(reads, `${c.name}: the driver must have been read`).toHaveLength(1);
-      expect(reads[0]?.ast?.where, `${c.name}: the driver must receive the number`).toEqual(c.expectedFilter());
+      expect(reads[0]?.ast?.where, `${c.name}: the driver must receive the number`).toEqual(lowered(c.expectedFilter()));
       // Copy-on-write: the filter belongs to the caller (view metadata, flow config).
       expect(JSON.stringify(filter), `${c.name}: the caller's filter must not be edited`).toBe(asWritten);
     }
@@ -207,7 +221,7 @@ describe('[#20351] the number-comparand declared-type door at the engine collect
       const filter = c.filter();
       await expect(engine.find(OBJECT, { where: filter }), c.name).resolves.toBeDefined();
       expect(reads, `${c.name}: the driver must have been read`).toHaveLength(1);
-      expect(reads[0]?.ast?.where, `${c.name}: the filter must reach the driver unchanged`).toEqual(filter);
+      expect(reads[0]?.ast?.where, `${c.name}: the filter must reach the driver unchanged`).toEqual(lowered(filter));
     }
   });
 
@@ -291,7 +305,7 @@ describe('[#20351] the number-comparand declared-type door at the engine collect
       expect(reads).toHaveLength(0);
     }
     await engine.find(OBJECT, { where: { $or: [{ f_text: 'a' }, { $not: { f_number: { $in: ['12', 5] } } }] } });
-    expect(reads[0]?.ast?.where).toEqual({ $or: [{ f_text: 'a' }, { $not: { f_number: { $in: [12, 5] } } }] });
+    expect(reads[0]?.ast?.where).toEqual(lowered({ $or: [{ f_text: 'a' }, { $not: { f_number: { $in: [12, 5] } } }] }));
   });
 
   it('refuses a {placeholder} against a number field UNRESOLVED — before the token resolver, in the door\'s words', async () => {

packages/objectql/src/engine-shared-filter-lowering-seam.test.ts

@@ -0,0 +1,172 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * [ADR-0053 D-D1, amended 2026-09-30 — #5930] The engine's placement of the
+ * shared `FilterCondition → FilterCondition` lowering (`lowerFilterCondition`,
+ * `@objectstack/spec/data`): once per filter position, AFTER the comparand doors
+ * and AFTER filter-token resolution (the amendment's item 3), on every verb that
+ * takes a filter — `find`, `findOne`, `count`, `update`, `delete`, and
+ * `aggregate`'s three positions (`where`, `aggregations[i].filter`, `having`).
+ *
+ * The witness is what leaves the engine: the recording driver's `where` for the
+ * verbs a driver compiles, and the operation context a middleware reads for the
+ * two positions the engine evaluates itself. Row answers are the faces' suites'
+ * business; these pins say WHERE the rule runs, so reverting any one seam call
+ * turns its pin red.
+ *
+ * Fixture: `closed_at` is the declared `datetime` the whole-day rule is for,
+ * `due_on` a `date` the typed seam must leave byte-identical (item 7).
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import type { EngineAggregateOptions, EngineQueryOptions, FilterCondition } from '@objectstack/spec/data';
+import { ObjectQL } from './engine.js';
+
+const OBJECT = 'lowering_probe';
+
+const SCHEMA = {
+  name: OBJECT,
+  label: 'Lowering probe',
+  fields: {
+    id: { name: 'id', type: 'text' },
+    stage: { name: 'stage', type: 'text' },
+    amount: { name: 'amount', type: 'number' },
+    closed_at: { name: 'closed_at', type: 'datetime' },
+    due_on: { name: 'due_on', type: 'date' },
+  },
+};
+
+interface Seen { verb: string; ast: any }
+
+function makeRecordingDriver() {
+  const seen: Seen[] = [];
+  const driver: any = {
+    name: 'recording', version: '0.0.0', supports: {},
+    async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; },
+    async find(_o: string, ast: any) { seen.push({ verb: 'find', ast }); return []; },
+    async findOne(_o: string, ast: any) { seen.push({ verb: 'findOne', ast }); return null; },
+    async count(_o: string, ast: any) { seen.push({ verb: 'count', ast }); return 0; },
+    async create(_o: string, data: Record<string, unknown>) { return { ...data, id: 'r1' }; },
+    async update(_o: string, id: string, data: Record<string, unknown>) { return { ...data, id }; },
+    async updateMany(_o: string, ast: any) { seen.push({ verb: 'updateMany', ast }); return 0; },
+    async delete() { return true; },
+    async deleteMany(_o: string, ast: any) { seen.push({ verb: 'deleteMany', ast }); return 0; },
+    async beginTransaction() { return { commit: async () => {}, rollback: async () => {} }; },
+    async commit() {}, async rollback() {},
+  };
+  return { driver, seen };
+}
+
+/** A bare-day upper bound on the datetime column, and what the lowering makes of it. */
+const WHOLE_DAY_IN = { closed_at: { $lte: '2026-07-28' } };
+const WHOLE_DAY_OUT = { closed_at: { $lt: '2026-07-29' } };
+/** A negative-polarity leaf, and its NULL escape. */
+const NEGATIVE_IN = { stage: { $ne: 'won' } };
+const NEGATIVE_OUT = { $and: [{ $or: [{ stage: { $null: true } }, { stage: { $ne: 'won' } }] }] };
+
+describe('[ADR-0053 D-D1 amended — #5930] the engine runs the shared lowering once per filter position', () => {
+  let engine: ObjectQL;
+  let seen: Seen[];
+
+  beforeEach(async () => {
+    const rec = makeRecordingDriver();
+    seen = rec.seen;
+    engine = new ObjectQL();
+    engine.registerDriver(rec.driver, true);
+    await engine.init();
+    engine.registry.registerObject(SCHEMA as any, 'test');
+    seen.length = 0;
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  const lastWhere = (verb: string) => [...seen].reverse().find((s) => s.verb === verb)?.ast?.where;
+
+  it('find: the driver receives the lowered where — whole-day bound and NULL escape', async () => {
+    await engine.find(OBJECT, { where: WHOLE_DAY_IN });
+    expect(lastWhere('find')).toEqual(WHOLE_DAY_OUT);
+    await engine.find(OBJECT, { where: NEGATIVE_IN });
+    expect(lastWhere('find')).toEqual(NEGATIVE_OUT);
+  });
+
+  it('findOne: the driver receives the lowered where', async () => {
+    await engine.findOne(OBJECT, { where: WHOLE_DAY_IN });
+    expect(lastWhere('findOne') ?? lastWhere('find')).toEqual(WHOLE_DAY_OUT);
+  });
+
+  it('count: the driver receives the lowered where', async () => {
+    await engine.count(OBJECT, { where: NEGATIVE_IN });
+    expect(lastWhere('count')).toEqual(NEGATIVE_OUT);
+  });
+
+  it('update (multi): the driver receives the lowered where', async () => {
+    await engine.update(OBJECT, { stage: 'x' }, { where: WHOLE_DAY_IN, multi: true } as any);
+    expect(lastWhere('updateMany')).toEqual(WHOLE_DAY_OUT);
+  });
+
+  it('delete (multi): the driver receives the lowered where', async () => {
+    await engine.delete(OBJECT, { where: NEGATIVE_IN, multi: true } as any);
+    expect(lastWhere('deleteMany')).toEqual(NEGATIVE_OUT);
+  });
+
+  it('aggregate: `where`, `aggregations[i].filter` and `having` are each lowered, before the middleware chain', async () => {
+    let atMiddleware: any;
+    engine.registerMiddleware(async (opCtx: any, next: any) => {
+      if (opCtx.operation === 'aggregate') atMiddleware = structuredClone(opCtx.ast);
+      return next();
+    });
+    await engine.aggregate(OBJECT, {
+      where: WHOLE_DAY_IN,
+      groupBy: ['stage'],
+      aggregations: [
+        { function: 'count', alias: 'n' },
+        { function: 'count', alias: 'open_n', filter: NEGATIVE_IN },
+        { function: 'max', field: 'closed_at', alias: 'last_closed' },
+      ],
+      having: { last_closed: { $between: ['2026-07-01', '2026-07-28'] } },
+    } as EngineAggregateOptions);
+    expect(atMiddleware.where).toEqual(WHOLE_DAY_OUT);
+    expect(atMiddleware.aggregations[0].filter).toBeUndefined();
+    expect(atMiddleware.aggregations[1].filter).toEqual(NEGATIVE_OUT);
+    // `max(closed_at)` is a `datetime` column of the aggregated row, so the
+    // whole-day rule reaches it; the aggregated row's type is what `having` reads.
+    expect(atMiddleware.having).toEqual({ last_closed: { $gte: '2026-07-01', $lt: '2026-07-29' } });
+    // The rows path asks the driver for rows with the lowered `where` too.
+    expect(lastWhere('find')).toEqual(WHOLE_DAY_OUT);
+  });
+
+  it('item 3: the lowering runs AFTER token resolution — `{today}` resolves to the day, then widens', async () => {
+    vi.useFakeTimers({ toFake: ['Date'] });
+    vi.setSystemTime(new Date('2026-03-10T12:00:00.000Z'));
+    await engine.find(OBJECT, { where: { closed_at: { $lte: '{today}' } } } as EngineQueryOptions);
+    expect(lastWhere('find')).toEqual({ closed_at: { $lt: '2026-03-11' } });
+    await engine.update(OBJECT, { stage: 'x' }, { where: { closed_at: { $lte: '{today}' } }, multi: true } as any);
+    expect(lastWhere('updateMany')).toEqual({ closed_at: { $lt: '2026-03-11' } });
+  });
+
+  it('item 7: a typed seam leaves a `date` column, and a `$between` on a number, byte-identical', async () => {
+    const where = { due_on: { $lte: '2026-07-28' }, amount: { $between: [5, 25] } };
+    await engine.find(OBJECT, { where });
+    expect(lastWhere('find')).toBe(where);
+  });
+
+  it('the last supported day: `$lte` keeps only { $null: false }', async () => {
+    await engine.find(OBJECT, { where: { closed_at: { $lte: '9999-12-31' } } });
+    expect(lastWhere('find')).toEqual({ closed_at: { $null: false } });
+  });
+
+  it('copy-on-write: the caller\'s filter object is never edited', async () => {
+    const where: FilterCondition = { closed_at: { $lte: '2026-07-28' }, stage: { $ne: 'won' } };
+    const asWritten = JSON.stringify(where);
+    await engine.find(OBJECT, { where });
+    await engine.update(OBJECT, { stage: 'x' }, { where, multi: true } as any);
+    expect(JSON.stringify(where)).toBe(asWritten);
+  });
+
+  it('the judge runs the same stage and gains no verdict from it', () => {
+    expect(engine.judgeFilter(OBJECT, { closed_at: { $between: ['2026-07-01', '2026-07-28'] } })).toEqual({ ok: true });
+    expect(engine.judgeFilter(OBJECT, { $not: { stage: { $ne: 'won' } } })).toEqual({ ok: true });
+  });
+});

packages/objectql/src/engine-text-operator-declared-type-door.test.ts

@@ -68,6 +68,7 @@ import {
   TEXT_OPERATOR_DOOR_FIXTURE,
   TEXT_OPERATOR_DOOR_FIXTURE_OBJECT,
   TEXT_OPERATOR_DOOR_TYPE_CLASSES,
+  lowerFilterCondition,
   type FilterTextCase,
   type FilterTextRowsCase,
   type TextOperatorDoorCase,
@@ -81,6 +82,19 @@ import { findTextOperatorOverNonTextField } from './text-operator-declared-type-
 
 const OBJECT = TEXT_OPERATOR_DOOR_FIXTURE_OBJECT;
 
+/**
+ * [ADR-0053 D-D1, amended — #5930] What a driver receives is the door's output
+ * after the engine's shared lowering (the NULL-polarity guards on `$ne` / `$nin`
+ * / `$notContains`, the whole-day rule on a declared `datetime`), which runs
+ * after this door on every verb. The door rewrites nothing, so a pin on the
+ * driver's input compares against the same lowering of the caller's filter.
+ */
+const lowered = (where: unknown): unknown =>
+  lowerFilterCondition(where, {
+    isDatetimeColumn: (column) =>
+      (TEXT_OPERATOR_DOOR_FIXTURE.fields as Record<string, { type?: string } | undefined>)[column]?.type === 'datetime',
+  });
+
 interface SeenRead { ast: any }
 
 /** Minimal recording driver — the same witness shape as the #7872 door suite. */
@@ -187,7 +201,7 @@ describe('[#15773] the text-operator declared-type door at the engine collection
       const filter = c.filter();
       await expect(engine.find(OBJECT, { where: filter }), c.name).resolves.toBeDefined();
       expect(reads, `${c.name}: the driver must have been read`).toHaveLength(1);
-      expect(reads[0]?.ast?.where, `${c.name}: the filter must reach the driver unchanged`).toEqual(filter);
+      expect(reads[0]?.ast?.where, `${c.name}: the filter must reach the driver unchanged`).toEqual(lowered(filter));
     }
   });
 
@@ -197,7 +211,7 @@ describe('[#15773] the text-operator declared-type door at the engine collection
       const filter = c.filter();
       await expect(engine.find(OBJECT, { where: filter }), c.name).resolves.toBeDefined();
       expect(reads, `${c.name}: the driver must have been read`).toHaveLength(1);
-      expect(reads[0]?.ast?.where, `${c.name}: the filter must reach the driver unchanged`).toEqual(filter);
+      expect(reads[0]?.ast?.where, `${c.name}: the filter must reach the driver unchanged`).toEqual(lowered(filter));
     }
   });