From 8157ab9828c29f200b6748e145fa1720fee1d41d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 09:02:43 +0000 Subject: [PATCH 1/8] fix(core)!: the JSON-column refusal set covers the text operators other than membership $startsWith, $endsWith, $icontains and the staged $like / $ilike join JSON_COLUMN_INCOMPATIBLE_OPERATORS: on a JSON-stored column each matched the serialization as text (SQLite), failed at query time (PostgreSQL), or counted nothing (the per-aggregation filter). $contains / $notContains stay out: they are the membership pair. Claude-Session: https://claude.ai/code/session_01Ujdtvqs7ree7WyQmEDwEnG Co-authored-by: Claude --- .../json-column-operator-refusal.test.ts | 26 +++++++-- .../src/utils/json-column-operator-refusal.ts | 53 +++++++++++++++---- 2 files changed, 65 insertions(+), 14 deletions(-) diff --git a/packages/core/src/utils/json-column-operator-refusal.test.ts b/packages/core/src/utils/json-column-operator-refusal.test.ts index e45e9ba83fc..5cd099dc812 100644 --- a/packages/core/src/utils/json-column-operator-refusal.test.ts +++ b/packages/core/src/utils/json-column-operator-refusal.test.ts @@ -7,7 +7,8 @@ * Two pins, both against what `driver-sql` answered BEFORE the move: * * - **The set** — the 22 spellings `driver-sql`'s module-private - * `JSON_COLUMN_INCOMPATIBLE_OPERATORS` held, member for member. + * `JSON_COLUMN_INCOMPATIBLE_OPERATORS` held, member for member, and + * [#21009] the five text operators that joined them since. * - **The words** — the SHA-256 of each text, captured from `driver-sql`'s * built `jsonColumnOperatorError` at the commit before the move (`8f784959c`) * through a real `SqlDriver` over SQLite: the withheld message (one text for @@ -28,16 +29,23 @@ import { JSON_COLUMN_INCOMPATIBLE_OPERATORS, jsonColumnOperatorRefusalText } fro const sha256 = (text: string): string => createHash('sha256').update(text, 'utf8').digest('hex'); describe('[#21007] JSON_COLUMN_INCOMPATIBLE_OPERATORS', () => { - it('holds exactly the spellings driver-sql refused before the move', () => { + it('holds exactly the spellings driver-sql refused before the move, and the text family [#21009] added', () => { expect([...JSON_COLUMN_INCOMPATIBLE_OPERATORS].sort()).toEqual([ - '!=', '$between', '$eq', '$gt', '$gte', '$in', '$lt', '$lte', '$ne', '$nin', + '!=', '$between', '$endsWith', '$eq', '$gt', '$gte', '$icontains', '$ilike', '$in', '$like', + '$lt', '$lte', '$ne', '$nin', '$startsWith', '<', '<=', '<>', '=', '==', '>', '>=', 'between', 'in', 'nin', 'not_in', 'notin', ]); }); - it('leaves out the membership spelling, the rest of the text family and the null predicates', () => { - for (const op of ['$contains', '$notContains', '$startsWith', '$endsWith', '$icontains', '$null', '$exists', '$empty']) { + it('[#21009] holds every text operator except the membership pair', () => { + for (const op of ['$startsWith', '$endsWith', '$icontains', '$like', '$ilike']) { + expect(JSON_COLUMN_INCOMPATIBLE_OPERATORS.has(op), op).toBe(true); + } + }); + + it('leaves out the membership pair and the null predicates', () => { + for (const op of ['$contains', '$notContains', '$null', '$exists', '$empty']) { expect(JSON_COLUMN_INCOMPATIBLE_OPERATORS.has(op), op).toBe(false); } }); @@ -56,6 +64,14 @@ describe('[#21007] jsonColumnOperatorRefusalText — byte for byte what driver-s expect({ sha: sha256(text.diagnostic), length: text.diagnostic.length }).toEqual(diagnostic); }); + it('[#21009] a text operator reads the very message the equality family reads, and its diagnostic names it', () => { + for (const op of ['$startsWith', '$endsWith', '$icontains', '$like', '$ilike']) { + const text = jsonColumnOperatorRefusalText('members', op, false); + expect({ sha: sha256(text.message), length: text.message.length }, op).toEqual(MESSAGE); + expect(text.diagnostic, op).toContain(`Operator "${op}" on field "members" WAS NOT APPLIED`); + } + }); + it('the message names neither the field nor the operator, and prescribes $contains and an $or of it', () => { const { message, diagnostic } = jsonColumnOperatorRefusalText('secret_col', '$nin', false); expect(message).not.toContain('secret_col'); diff --git a/packages/core/src/utils/json-column-operator-refusal.ts b/packages/core/src/utils/json-column-operator-refusal.ts index 7bf8d41ca42..8799778c586 100644 --- a/packages/core/src/utils/json-column-operator-refusal.ts +++ b/packages/core/src/utils/json-column-operator-refusal.ts @@ -5,6 +5,7 @@ * field stored as a JSON column — a `multiple: true` field, an inherently * multi-value option type (`tags`, `multiselect`, `checkboxes`) or a * structured-JSON type (`json`, `address`, …): the operator set and the words. + * [#21009] The text operators other than the membership pair get it too. * * ## Two faces, one rule * @@ -30,15 +31,18 @@ * `$contains` is the membership spelling on such a column (`FILTER_OPERATORS`' * `$contains` docblock, `@objectstack/spec`), and it is what the refusal * prescribes — `$contains` for one member, an `$or` of `$contains` for any-of. - * That is why it is ABSENT from the set below, with the rest of the text family - * and the null predicates. + * That is why it is ABSENT from the set below, with its complement + * `$notContains` and the null predicates. [#21009] The remainder of the text + * family is IN the set: it has no membership reading, so it matched the + * serialization. */ /** * [#7398] Operators whose SQL lowering compares a column's STORED SCALAR to a * value — every spelling either of `driver-sql`'s two comparison emitters * answers (`applyFilterCondition`'s plain-column switch and - * `applyNormalizedComparison`'s normalised arms). + * `applyNormalizedComparison`'s normalised arms). [#21009] Or MATCHES that + * stored scalar as text: the text family other than the membership pair. * * The bare infix forms are here for the same reason they are in `driver-sql`'s * `SCALAR_COMPARAND_OPERATORS`: `applyNormalizedComparison` really does @@ -53,15 +57,40 @@ * the halves and compiling the compound would be the same wrong answer at one * more spelling. * - * Deliberately ABSENT, and this is the load-bearing half of the set: the `LIKE` - * family (`$contains`, `$notContains`, `$startsWith`, `$endsWith`, - * `$icontains`) and the null predicates (`$null`, `$exists`). `$contains` is - * the ONLY working membership spelling on a JSON-array column and downstream - * code depends on it (#7398's own tables), while `IS NULL` asks about the - * column's presence, which is a well-formed question whatever the column holds. + * Deliberately ABSENT, and this is the load-bearing half of the set: the + * membership pair (`$contains`, `$notContains`) and the null predicates + * (`$null`, `$exists`, `$empty`). `$contains` is the ONLY working membership + * spelling on a JSON-array column and downstream code depends on it (#7398's + * own tables) — `driver-sql` compiles it as a real per-dialect membership test + * (#17590), and `$notContains` as its exact complement — while `IS NULL` asks + * about the column's presence, which is a well-formed question whatever the + * column holds. * * [#21007] Moved here from `driver-sql`, unchanged, so the per-aggregation * `filter` refuses exactly the operators `where` refuses. + * + * [#21009] The remainder of the text family joined the set: `$startsWith`, + * `$endsWith`, `$icontains`, and the staged pattern pair `$like` / `$ilike` + * that `driver-sql` answers ahead of `FILTER_OPERATORS`. None has a membership + * reading, so on a JSON column each matched the SERIALIZATION as text, and the + * answers were wrong the same three ways the equality family's were. Measured + * through `POST /api/v1/data/:object/query` on a multi-value lookup holding + * `["u1","u2"]`: + * + * - SQLite: `$startsWith: '['` and `$endsWith: ']'` matched EVERY row with a + * value, while `$startsWith: 'u1'` matched none; `$icontains: 'U1'` matched + * the row holding only `u10`, and `$icontains: '","'` matched every row with + * two members. + * - PostgreSQL: a `json` column has no `LIKE` operator, so all five failed at + * query time — a `500` `DATABASE_ERROR` for a filter the caller can fix. + * - The per-aggregation `filter` counted `0` for `$startsWith`, `$endsWith` and + * `$icontains` (it already refuses the staged pair as unsupported). + * + * Each now gets this set's `400`. ⛔ No membership reading is invented for a + * prefix, suffix or case-folded test: the prescription stays `$contains`. The + * infix spellings `like` / `ilike` are not members because no emitter answers + * them as operators — the normalised arms carry no text family, and the + * operator switch refuses them as unsupported. */ export const JSON_COLUMN_INCOMPATIBLE_OPERATORS: ReadonlySet = new Set([ '$eq', '=', '==', @@ -70,6 +99,7 @@ export const JSON_COLUMN_INCOMPATIBLE_OPERATORS: ReadonlySet = new Set([ '$in', 'in', '$nin', 'nin', 'not_in', 'notin', '$between', 'between', + '$startsWith', '$endsWith', '$icontains', '$like', '$ilike', ]); /** The two texts of one JSON-column refusal — see {@link jsonColumnOperatorRefusalText}. */ @@ -116,6 +146,11 @@ export interface JsonColumnOperatorRefusalText { * [#21007] Moved here from `driver-sql`'s `jsonColumnOperatorError`, byte for * byte, so `where` and the per-aggregation `filter` print one sentence. The * caller builds the error: this returns only the text. + * + * [#21009] The text family that joined the set reads these same words, + * unchanged. Its prescription holds as written — membership is `$contains` — + * while the "scalar comparison" wording and the two directions the closing + * sentence names are the equality family's. */ export function jsonColumnOperatorRefusalText( field: string, From d6c9c63f827bbdbcf905efe973bfd27c85ab415f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 09:06:55 +0000 Subject: [PATCH 2/8] test(driver-sql): pin the text-operator refusal on a JSON column, per dialect Claude-Session: https://claude.ai/code/session_01Ujdtvqs7ree7WyQmEDwEnG Co-authored-by: Claude --- ...-json-column-text-operator-refusal.test.ts | 189 ++++++++++++++++++ ...river-json-column-operator-refusal.test.ts | 34 +++- 2 files changed, 219 insertions(+), 4 deletions(-) create mode 100644 packages/drivers/driver-sql/src/sql-driver-21009-json-column-text-operator-refusal.test.ts diff --git a/packages/drivers/driver-sql/src/sql-driver-21009-json-column-text-operator-refusal.test.ts b/packages/drivers/driver-sql/src/sql-driver-21009-json-column-text-operator-refusal.test.ts new file mode 100644 index 00000000000..983ea5dbae5 --- /dev/null +++ b/packages/drivers/driver-sql/src/sql-driver-21009-json-column-text-operator-refusal.test.ts @@ -0,0 +1,189 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21009] The text operators other than the membership pair — `$startsWith`, + * `$endsWith`, `$icontains` and the staged `$like` / `$ilike` — are REFUSED on a + * column this driver stores as JSON text, with the `400` the equality family + * already gets there (#7398), on every dialect alike. A scalar text column is + * unaffected (the control), and the membership pair keeps answering. + * + * ## What each dialect answered before (`origin/main` `7a606a9a3`) + * + * Measured through `POST /api/v1/data/:object/query` over a multi-value lookup + * and a `tags` field, on SQLite and a private PostgreSQL 16.14: + * + * | filter on a multi-value column | SQLite | PostgreSQL | + * |:--|:--|:--| + * | `$startsWith: '['` | every row with a value (the brackets) | 500 `DATABASE_ERROR` | + * | `$startsWith: 'u1'` | no row, though two rows hold `u1` | 500 `DATABASE_ERROR` | + * | `$endsWith: ']'` | every row with a value | 500 `DATABASE_ERROR` | + * | `$icontains: 'U1'` | the row holding only `u10` too | 500 `DATABASE_ERROR` | + * | `$icontains: '","'` | every row with two members | 500 `DATABASE_ERROR` | + * | `$like` / `$ilike` | a substring of the serialization | 500 `DATABASE_ERROR` | + * + * PostgreSQL's `json` column has no `LIKE` operator, so the filter failed at + * query time — a `500` for a filter the caller can fix. SQLite answered the + * serialization, which is a wrong answer rather than a narrow one. Neither + * dialect ever answered a membership question with these operators, and none is + * invented for them here: the prescription is `$contains`. + * + * ## What this file pins, per dialect cell + * + * - Each operator, on a multi-value lookup and on a `tags` column: `INVALID_FILTER` + * / `400`, through `find` and `count`. An author's own filter reads the + * operator and the field named; any other reads the shared withheld message — + * byte for byte the one the equality family reads. + * - The control: the same operators on a scalar text column answer the rows + * they always did. + * - `$contains` / `$notContains` still answer MEMBERSHIP on the same columns. + * + * The SQLite cell always runs; the PostgreSQL and MySQL cells run where + * `OS_TEST_POSTGRES_URL` / `OS_TEST_MYSQL_URL` are set — the + * `Temporal Conformance (live PG + MySQL)` job provisions both — and are a + * named skip otherwise. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import type { Knex } from 'knex'; +import { jsonColumnOperatorRefusalText } from '@objectstack/core'; +import { markFilterSubtreeProvenance } from '@objectstack/spec/data'; +import type { DriverOptions, FilterCondition } from '@objectstack/spec/data'; +import { SqlDriver } from './sql-driver.js'; +import { + DIALECT_CELLS, + declareDialectCell, + LIVE_CELL_TIMEOUT_MS, + type DialectCell, +} from './live-dialect-matrix.testkit.js'; + +/** Issue-prefixed: each live cell owns its table, dropped before and after. */ +const OBJECT = 'os21009_text_ops'; + +/** Diagnostics-only; it never changes which rows a read touches. */ +const BYPASS: DriverOptions = { bypassTenantAudit: true }; + +const FIELDS: Record> = { + // The scalar control. + label: { type: 'text' }, + // The ruling's own column: a multi-valued lookup. + owners: { type: 'lookup', reference: 'os21009_owner', multiple: true }, + // An inherently multi-valued option type, stored the same way. + tags_: { type: 'tags' }, +}; + +const ROWS = [ + { id: '1', label: 'u1 memo', owners: ['u1', 'u2'], tags_: ['red', 'blue'] }, + { id: '2', label: 'red', owners: ['u10'], tags_: ['redwood'] }, + { id: '3', label: 'about u10', owners: ['u3', 'u1'], tags_: ['blue'] }, + { id: '4', label: 'none', owners: [], tags_: [] }, +] as const; + +/** + * The refused operators, each with a comparand its own contract accepts — so a + * refusal here is always the column-type gate, never a comparand-shape one — + * and the one that used to match on SQLite where there is one. + */ +const TEXT_REFUSED: ReadonlyArray = [ + ['$startsWith', '['], + ['$endsWith', ']'], + ['$icontains', 'U1'], + ['$like', '%u1%'], + ['$ilike', '%U1%'], +]; + +/** The same operators on the scalar `label` column, and the rows they answer. */ +const CONTROL: ReadonlyArray = [ + ['$startsWith', 'u1', ['1']], + ['$endsWith', 'u10', ['3']], + ['$icontains', 'U1', ['1', '3']], + ['$like', '%u1%', ['1', '3']], + ['$ilike', '%U1%', ['1', '3']], +]; + +interface WireBearingError extends Error { + code?: string; + status?: number; +} + +async function refusalOf(run: () => Promise): Promise { + try { + await run(); + } catch (e) { + return e as WireBearingError; + } + throw new Error('expected the driver to refuse this filter, but it resolved'); +} + +for (const cell of DIALECT_CELLS) { + declareDialectCell(cell, '[#21009] the JSON-column text-operator refusal', declareTextOperatorCell); +} + +function declareTextOperatorCell(cell: DialectCell): void { + describe(`[#21009] SqlDriver — the text operators other than membership are refused on a JSON column (${cell.label})`, () => { + let driver: SqlDriver; + let knexInstance: Knex; + + beforeAll(async () => { + driver = new SqlDriver(cell.config()); + knexInstance = driver.getKnex(); + await knexInstance.schema.dropTableIfExists(OBJECT); + await driver.initObjects([{ name: OBJECT, fields: FIELDS } as never]); + for (const row of ROWS) { + await driver.create(OBJECT, { ...row, owners: [...row.owners], tags_: [...row.tags_] }, BYPASS); + } + }, LIVE_CELL_TIMEOUT_MS); + + afterAll(async () => { + await knexInstance?.schema.dropTableIfExists(OBJECT).catch(() => {}); + await driver?.disconnect?.(); + }); + + const ids = async (where: FilterCondition): Promise => { + const rows = await driver.find(OBJECT, { where }, BYPASS); + return rows.map((r) => String(r.id)).sort((a, b) => a.localeCompare(b)); + }; + + it('stored all four rows — the premise of every answer below', async () => { + expect(await ids({})).toEqual(['1', '2', '3', '4']); + }); + + for (const field of ['owners', 'tags_']) { + for (const [op, comparand] of TEXT_REFUSED) { + it(`${field} ${op}: 400 INVALID_FILTER, the operator and the field named to the author`, async () => { + const where = markFilterSubtreeProvenance({ [field]: { [op]: comparand } }, 'author') as FilterCondition; + const err = await refusalOf(() => driver.find(OBJECT, { where }, BYPASS)); + expect(err.code).toBe('INVALID_FILTER'); + expect(err.status).toBe(400); + expect(err.message).toContain(`Operator "${op}" on field "${field}" WAS NOT APPLIED`); + expect(err.message).toContain(`{ "${field}": { "$contains": "a" } }`); + }); + + it(`${field} ${op}: any other caller reads the equality family's withheld message, byte for byte`, async () => { + const where = { [field]: { [op]: comparand } } as FilterCondition; + for (const run of [ + () => driver.find(OBJECT, { where }, BYPASS), + () => driver.count(OBJECT, { where }, BYPASS), + ]) { + const err = await refusalOf(run); + expect(err.code).toBe('INVALID_FILTER'); + expect(err.status).toBe(400); + expect(err.message).toBe(jsonColumnOperatorRefusalText(field, '$in', false).message); + } + }); + } + } + + for (const [op, comparand, expected] of CONTROL) { + it(`control — label ${op} ${JSON.stringify(comparand)} answers ${JSON.stringify(expected)}`, async () => { + expect(await ids({ label: { [op]: comparand } } as FilterCondition)).toEqual(expected); + }); + } + + it('the membership pair still answers on the same columns', async () => { + expect(await ids({ owners: { $contains: 'u1' } })).toEqual(['1', '3']); + expect(await ids({ owners: { $notContains: 'u1' } })).toEqual(['2', '4']); + expect(await ids({ tags_: { $contains: 'red' } })).toEqual(['1']); + expect(await ids({ $or: [{ owners: { $contains: 'u3' } }, { owners: { $contains: 'u10' } }] })).toEqual(['2', '3']); + }); + }); +} diff --git a/packages/drivers/driver-sql/src/sql-driver-json-column-operator-refusal.test.ts b/packages/drivers/driver-sql/src/sql-driver-json-column-operator-refusal.test.ts index 8c96b08d996..3558985da3b 100644 --- a/packages/drivers/driver-sql/src/sql-driver-json-column-operator-refusal.test.ts +++ b/packages/drivers/driver-sql/src/sql-driver-json-column-operator-refusal.test.ts @@ -32,6 +32,13 @@ * sorts below `usr_…` on the leading `[`, so the ordering comparisons return a * lexicographic verdict over a *serialization*. * + * [#21009] The text family other than the membership pair answered the same + * way and is refused the same way now: on this fixture `$startsWith: '['` and + * `$endsWith: ']'` matched the row by the serialization's brackets, and + * `$icontains` by substring; on PostgreSQL all five (with the staged + * `$like` / `$ilike`) failed at query time with a 500. The dialect cells are + * `sql-driver-21009-json-column-text-operator-refusal.test.ts`. + * * ## What these tests pin * * 1. **The refusal**, per operator and per FACE. #6203's one-query-two-answers @@ -139,6 +146,21 @@ const REFUSED: ReadonlyArray = [ // compiling the compound would leave the same wrong answer alive at one more // spelling — the reasoning #5234 already applied in this file. ['$between', [U1, U2]], + // [#21009] The text family other than the membership pair. None has a + // membership reading, so each matched the SERIALIZATION as text: on this + // fixture `$startsWith: '['` and `$endsWith: ']'` matched the row (every + // stored array opens and closes that way), and on live PostgreSQL each failed + // at query time with a 500, a `json` column having no `LIKE` operator. The + // comparands below are the ones that used to MATCH here, so the flip from a + // row to a refusal is the whole point of each line. + ['$startsWith', '['], + ['$endsWith', ']'], + ['$icontains', U1], + // [#21009] The staged pattern pair — answered by this driver ahead of + // `FILTER_OPERATORS` (so outside the closed-world sweep below), and refused + // here for the same reason. + ['$like', `%${U1}%`], + ['$ilike', `%${U1.toUpperCase()}%`], ]; /** @@ -159,9 +181,9 @@ const REFUSED: ReadonlyArray = [ const KEPT: ReadonlyArray = [ ['$contains', U1], ['$notContains', 'nobody'], - ['$startsWith', '['], - ['$endsWith', ']'], - ['$icontains', U1], + // [#21009] `$startsWith` / `$endsWith` / `$icontains` stood here, with the + // very comparands that now head the refused rows above: they "worked" only + // by reading the serialization, which is the wrong answer #21009 refuses. ['$null', false], ['$exists', true], // [#20446] `$empty` joined `FILTER_OPERATORS`. It asks the question this @@ -434,7 +456,11 @@ describe('[#7398] SqlDriver refuses scalar-comparison operators on JSON/multi-va } // The partition is asserted whole, so an operator quietly moving from one // side to the other cannot pass as "still 16 operators". - expect(refused).toEqual(REFUSED.map(([op]) => op)); + // [#21009] REFUSED also carries the staged `$like` / `$ilike`, which this + // sweep never iterates — so its expectation is REFUSED's DECLARED members. + expect(refused).toEqual( + REFUSED.map(([op]) => op).filter((op) => (FILTER_OPERATORS as readonly string[]).includes(op)), + ); expect(compiled).toEqual(KEPT.map(([op]) => op)); }); From f3a3dcce15af1a8b2e89fc7713d6b7301872932d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 09:16:50 +0000 Subject: [PATCH 3/8] test(objectql,rest): the per-aggregation filter refuses the text family on a multi-valued field, as where does Claude-Session: https://claude.ai/code/session_01Ujdtvqs7ree7WyQmEDwEnG Co-authored-by: Claude --- ...gregate-filter-json-column-refusal.test.ts | 44 +++++++++++ ...egation-filter-json-column-refusal.test.ts | 73 +++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/packages/objectql/src/engine-aggregate-filter-json-column-refusal.test.ts b/packages/objectql/src/engine-aggregate-filter-json-column-refusal.test.ts index 10cdf6c51a5..dce46fe8c07 100644 --- a/packages/objectql/src/engine-aggregate-filter-json-column-refusal.test.ts +++ b/packages/objectql/src/engine-aggregate-filter-json-column-refusal.test.ts @@ -27,6 +27,13 @@ // the one face that evaluates `aggregations[i].filter`); the SQLite and // PostgreSQL cells, each beside its live `where` twin, are `packages/rest`'s // `aggregation-filter-json-column-refusal.test.ts`. +// +// [#21009] The text operators other than the membership pair joined the shared +// set, so this face refuses `$startsWith` / `$endsWith` / `$icontains` on a +// multi-valued field too — where it counted `m = 0` for every one of them, a +// `200` no stored array can support. (The staged `$like` / `$ilike` are refused +// here earlier, as operators this face does not evaluate; a structured-JSON +// field meets the text-operator declared-type door first, as before.) import { describe, it, expect, vi } from 'vitest'; import { lowerFilterCondition } from '@objectstack/spec/data'; @@ -122,6 +129,19 @@ function family(field: string): Array, ]; } +/** [#21009] field → [name, filter, operator] — the text family on a MULTI-VALUED field. */ +function textFamily(field: 'owners' | 'tags'): Array, string]> { + const [a] = VALUES[field]; + return [ + ['$startsWith', { [field]: { $startsWith: a } }, '$startsWith'], + ['$startsWith on the serialization', { [field]: { $startsWith: '[' } }, '$startsWith'], + ['$endsWith', { [field]: { $endsWith: a } }, '$endsWith'], + ['$icontains', { [field]: { $icontains: a.toUpperCase() } }, '$icontains'], + ['$icontains in an $or branch after one that holds', { $or: [{ title: 'x' }, { [field]: { $icontains: a } }] }, '$icontains'], + ['$startsWith under $not', { $not: { [field]: { $startsWith: a } } }, '$startsWith'], + ]; +} + async function refusalOf(run: () => Promise): Promise { try { await run(); @@ -170,6 +190,27 @@ describe('[#21007] engine.aggregate — a per-aggregation filter refuses a scala } } + for (const field of ['owners', 'tags'] as const) { + for (const [name, filter, op] of textFamily(field)) { + it(`[#21009] ${field} ${name}: 400 INVALID_FILTER in where's words, and no row is read`, async () => { + const { engine, driver, warn } = await makeEngine(ROWS); + const err = await refusalOf(() => engine.aggregate(OBJECT, perAggregation(filter))); + expectWhereRefusal(err, warn, field, op, false); + expect(driver.find).not.toHaveBeenCalled(); + }); + } + } + + it('[#21009] a structured-JSON field still meets the text-operator declared-type door first', async () => { + const { engine, warn } = await makeEngine(ROWS); + const err = await refusalOf(() => engine.aggregate(OBJECT, perAggregation({ meta: { $startsWith: 'a' } }))); + expect(err.code).toBe('INVALID_FILTER'); + expect(err.status).toBe(400); + expect(err.message).toContain("filter on 'meta' aims the text operator"); + expect(err.message).not.toContain('WAS NOT APPLIED'); + expect(warn.mock.calls.map((call: unknown[]) => String(call[0])).join('\n')).not.toContain('WAS NOT APPLIED'); + }); + it('the card on an EMPTY table: refused too — the verdict is the filter\'s, not the data\'s', async () => { for (const filter of [{ owners: { $in: ['u1', 'u9'] } }, { owners: { $nin: ['u1', 'u9'] } }]) { const { engine, warn } = await makeEngine([]); @@ -230,6 +271,9 @@ describe('[#21007] the per-row floor — a caller evaluating rows directly meets ['$eq', { tags: { $eq: 'red' } }], ['implicit equality', { tags: 'red' }], ['$between', { meta: { $between: ['a', 'b'] } }], + // [#21009] The text family reads the same set here: `$startsWith` on the + // stored array answered `false` for every row before. + ['$startsWith', { owners: { $startsWith: 'u1' } }], ] as const)('%s', (_name, filter) => { let thrown: (Error & { code?: string; status?: number }) | undefined; try { diff --git a/packages/rest/src/aggregation-filter-json-column-refusal.test.ts b/packages/rest/src/aggregation-filter-json-column-refusal.test.ts index 648ae70aad3..a36c29832b7 100644 --- a/packages/rest/src/aggregation-filter-json-column-refusal.test.ts +++ b/packages/rest/src/aggregation-filter-json-column-refusal.test.ts @@ -23,6 +23,13 @@ * presents) is `@objectstack/objectql`'s * `engine-aggregate-filter-json-column-refusal.test.ts`. * + * [#21009] The text operators other than the membership pair joined the shared + * set. Measured before (`origin/main` `7a606a9a3`) on the same table: `owners` + * / `tags` `$startsWith` / `$endsWith` / `$icontains` — `where` answered the + * serialization on SQLite (`$startsWith: '['` matched every row with a value) + * and `500` `DATABASE_ERROR` on PostgreSQL 16.14; the per-aggregation `filter` + * counted `m = 0` for each. Both faces now answer the `400` above. + * * ## The dialect axis of THIS file * * The SQLite cell always runs. The PostgreSQL and MySQL cells run where @@ -89,6 +96,30 @@ function family(field: string): Array] ]; } +/** + * [#21009] The text operators other than the membership pair, on a multi-valued + * field. Before, `where` answered the SERIALIZATION on SQLite (`$startsWith: '['` + * matched every row with a value) and a 500 `DATABASE_ERROR` on PostgreSQL, while + * the per-aggregation `filter` counted `m = 0` for each — three answers to one + * filter, none of them the caller's. + */ +function textFamily(field: 'owners' | 'tags'): Array]> { + const [a] = VALUES[field]; + return [ + ['$startsWith', { [field]: { $startsWith: a } }], + ['$startsWith on the serialization', { [field]: { $startsWith: '[' } }], + ['$endsWith', { [field]: { $endsWith: ']' } }], + ['$icontains', { [field]: { $icontains: a.toUpperCase() } }], + ]; +} + +/** [#21009] The same text operators on the scalar `title` column: unaffected. */ +const TEXT_CONTROLS: ReadonlyArray, number]> = [ + ['title $startsWith', { title: { $startsWith: 'u1' } }, 2], + ['title $endsWith', { title: { $endsWith: 'u10' } }, 1], + ['title $icontains', { title: { $icontains: 'U1' } }, 3], +]; + /** Controls on the scalar `title` column: the per-aggregation count equals the where twin's. */ const CONTROLS: ReadonlyArray]> = [ ['title $in', { title: { $in: ['u1', 'x'] } }], @@ -221,6 +252,48 @@ for (const cell of CELLS) { }, 60_000); } + for (const field of ['owners', 'tags'] as const) { + for (const [name, filter] of textFamily(field)) { + it(`[#21009] ${field} ${name}: 400 INVALID_FILTER on both faces, the where twin's very body`, async () => { + const twin = await post(whereTwin(filter)); + expect(twin.status, JSON.stringify(twin.json)).toBe(400); + expect(twin.json.code).toBe('INVALID_FILTER'); + warn.mockClear(); + const agg = await post(perAggregation(filter)); + expect(agg.status, JSON.stringify(agg.json)).toBe(400); + expect(agg.json.code).toBe('INVALID_FILTER'); + expect(agg.json.error).toBe(twin.json.error); + expect(agg.json.error).toContain('{ "FIELD": { "$contains": "a" } }'); + expect(agg.json.error).not.toContain(`"${field}"`); + const logged = warn.mock.calls.map((call: unknown[]) => String(call[0])).join('\n'); + expect(logged).toMatch(new RegExp(`Operator "\\$[A-Za-z]+" on field "${field}" WAS NOT APPLIED`)); + }, 60_000); + } + + it(`[#21009] ${field} $like: where refuses it as a JSON column, the per-aggregation filter as an operator it does not evaluate`, async () => { + const filter = { [field]: { $like: '%u1%' } }; + const twin = await post(whereTwin(filter)); + expect(twin.status, JSON.stringify(twin.json)).toBe(400); + expect(twin.json.code).toBe('INVALID_FILTER'); + expect(twin.json.error).toContain('WAS NOT APPLIED'); + const agg = await post(perAggregation(filter)); + expect(agg.status, JSON.stringify(agg.json)).toBe(400); + expect(agg.json.code).toBe('INVALID_FILTER'); + expect(agg.json.error).toContain("Unsupported operator '$like'"); + }, 60_000); + } + + for (const [name, filter, m] of TEXT_CONTROLS) { + it(`[#21009] control — ${name}: answered on both faces, m = ${m}`, async () => { + const twin = await post(whereTwin(filter)); + expect(twin.status, JSON.stringify(twin.json)).toBe(200); + expect(twin.json.records).toEqual([{ n: m }]); + const agg = await post(perAggregation(filter)); + expect(agg.status, JSON.stringify(agg.json)).toBe(200); + expect(agg.json.records).toEqual([{ n: 6, m }]); + }, 60_000); + } + it('an empty table refuses the card too — the verdict is the filter\'s', async () => { await boot(false); for (const filter of [{ owners: { $in: ['u1', 'u9'] } }, { owners: { $nin: ['u1', 'u9'] } }]) { From 77737f7d2f79679a3d97638d1ccd7cdc4e7f9cfa Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 09:36:23 +0000 Subject: [PATCH 4/8] test(driver-sql): flip the two compile pins that held the text family unmoved on a JSON column Claude-Session: https://claude.ai/code/session_01Ujdtvqs7ree7WyQmEDwEnG Co-authored-by: Claude --- ...43-multi-valued-boolean-membership.test.ts | 26 +++++++++++++--- ...river-17590-json-column-membership.test.ts | 31 +++++++++++++------ 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/packages/drivers/driver-sql/src/sql-driver-17343-multi-valued-boolean-membership.test.ts b/packages/drivers/driver-sql/src/sql-driver-17343-multi-valued-boolean-membership.test.ts index 9285be4d7d9..29e575d8890 100644 --- a/packages/drivers/driver-sql/src/sql-driver-17343-multi-valued-boolean-membership.test.ts +++ b/packages/drivers/driver-sql/src/sql-driver-17343-multi-valued-boolean-membership.test.ts @@ -270,13 +270,29 @@ describe('[#17343] the per-dialect construct, compiled — the registerExternalO const d = typed(config); for (const field of ['picks', 'refs']) { for (const op of POSITIVE_OPERATORS) { + // [#21009] Only `$contains` — the membership spelling — compiles on a + // JSON column now; the rest of the family is REFUSED there (`400`), + // ahead of both this card's declared-type gate and the emitter. Either + // way the gate this file is about does not fire: a refusal is not the + // `1 = 0` constant, and the SHAPE per operator is owned by #17590's + // and #21009's own files. + if (op !== '$contains') { + let refusal: (Error & { code?: string; status?: number }) | undefined; + try { + d.compileWhere({ [field]: { [op]: 'x' } } as FilterCondition); + } catch (e) { + refusal = e as Error & { code?: string; status?: number }; + } + expect(refusal?.code, `${op} over ${field}`).toBe('INVALID_FILTER'); + expect(refusal?.status, `${op} over ${field}`).toBe(400); + continue; + } const sql = d.compileWhere({ [field]: { [op]: 'x' } } as FilterCondition); expect(sql, `${op} over ${field}`).not.toMatch(/1 = 0|1 = 1/); - // [#17590] `$contains` compiles the MEMBERSHIP construct now and the - // rest of the family still compiles a pattern match. What this card - // is about is neither shape — it is that the declared-type gate does - // not fire — so this row asks for "a real predicate over the column", - // and the SHAPE per operator is owned by #17590's own file. + // [#17590] `$contains` compiles the MEMBERSHIP construct. What this + // card is about is not the shape — it is that the declared-type gate + // does not fire — so this row asks for "a real predicate over the + // column". expect(sql, `${op} over ${field}`).toMatch(REAL_PREDICATE[label]!); } } diff --git a/packages/drivers/driver-sql/src/sql-driver-17590-json-column-membership.test.ts b/packages/drivers/driver-sql/src/sql-driver-17590-json-column-membership.test.ts index 45a1a00c2ee..17ad08ac88c 100644 --- a/packages/drivers/driver-sql/src/sql-driver-17590-json-column-membership.test.ts +++ b/packages/drivers/driver-sql/src/sql-driver-17590-json-column-membership.test.ts @@ -306,18 +306,31 @@ describe('[#17590] the per-dialect membership construct, compiled', () => { }); /** - * The other text operators are NOT membership spellings and this card does - * not rule on them — they keep the text emitter's lowering (on SQLite, since - * #20024, `instr(` for `$icontains` and `substr(CAST(` for `$endsWith`). - * Pinned so a later widening is a deliberate edit here rather than a silent - * side effect. + * The other text operators are NOT membership spellings and this card did + * not rule on them — it pinned them unmoved "so a later widening is a + * deliberate edit here rather than a silent side effect". [#21009] is that + * edit: on a JSON column they matched the serialization (SQLite) or failed at + * query time (PostgreSQL), so they are now REFUSED there, in the equality + * family's `400`, before either emitter is reached — and no membership + * reading is invented for them. On the scalar string column they keep the + * text emitter's lowering (on SQLite, since #20024, `instr(` for + * `$icontains` and `substr(CAST(` for `$endsWith`). */ - it(`${label}: the rest of the text family is UNMOVED on a JSON column`, () => { + it(`${label}: [#21009] the rest of the text family is REFUSED on a JSON column, and unmoved on a scalar one`, () => { const d = new CompilerProbeDriver(config).declare(); for (const op of ['$startsWith', '$endsWith', '$icontains', '$like', '$ilike']) { - const sql = d.compileWhere({ tags_: { [op]: 'red' } } as FilterCondition); - expect(sql, `${op} on ${label}`).toMatch(/LIKE|GLOB|instr\(|substr\(CAST\(/); - expect(sql, `${op} on ${label}`).not.toMatch(CONSTRUCT[label]!); + let refusal: (Error & { code?: string; status?: number }) | undefined; + try { + d.compileWhere({ tags_: { [op]: 'red' } } as FilterCondition); + } catch (e) { + refusal = e as Error & { code?: string; status?: number }; + } + expect(refusal?.code, `${op} on ${label}`).toBe('INVALID_FILTER'); + expect(refusal?.status, `${op} on ${label}`).toBe(400); + expect(refusal?.message, `${op} on ${label}`).toContain('WAS NOT APPLIED'); + const sql = d.compileWhere({ label: { [op]: 'red' } } as FilterCondition); + expect(sql, `${op} on the scalar label, ${label}`).toMatch(/LIKE|GLOB|instr\(|substr\(CAST\(/); + expect(sql, `${op} on the scalar label, ${label}`).not.toMatch(CONSTRUCT[label]!); } }); } From 4fa4ef9c1bd459a421cc8b53ccce5aba61ca5f4e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 09:49:56 +0000 Subject: [PATCH 5/8] =?UTF-8?q?chore(changeset):=20core=20minor,=20BREAKIN?= =?UTF-8?q?G=20=E2=80=94=20the=20JSON-column=20refusal=20covers=20the=20te?= =?UTF-8?q?xt=20family=20other=20than=20membership?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Ujdtvqs7ree7WyQmEDwEnG Co-authored-by: Claude --- .../21009-json-column-text-operators.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .changeset/21009-json-column-text-operators.md diff --git a/.changeset/21009-json-column-text-operators.md b/.changeset/21009-json-column-text-operators.md new file mode 100644 index 00000000000..68d6a7b6ad2 --- /dev/null +++ b/.changeset/21009-json-column-text-operators.md @@ -0,0 +1,21 @@ +--- +"@objectstack/core": minor +--- + +fix(core)!: a filter that aims `$startsWith`, `$endsWith`, `$icontains`, `$like` or `$ilike` at a field stored as a JSON column is refused with `INVALID_FILTER` / 400, as `$eq` / `$in` / `$nin` already are, instead of matching the field's serialized text or failing at query time + +Clause-②: no (narrowing) + + + +**BREAKING**: this narrows which filters are answered on a field stored as a JSON column, on every face that reads `@objectstack/core`'s `JSON_COLUMN_INCOMPATIBLE_OPERATORS`: `driver-sql`'s `where` (and `driver-sqlite-wasm` and `driver-turso`'s local transport, which inherit it) on every read and write face that lowers a filter, and the engine's per-aggregation `filter`. It ships as `minor` under the launch-window convention for accept-set narrowings. + +**What is refused.** On a field declared multi-valued (an inherently multi-value option type such as `tags`, `multiselect` or `checkboxes`, or a `select`, `radio`, `lookup`, `user`, `file` or `image` field declared `multiple: true`) or structured-JSON (`json`, `address`, …), a filter using `$startsWith`, `$endsWith`, `$icontains`, or the staged pattern pair `$like` / `$ilike`, is refused with `INVALID_FILTER` / 400, at any depth under `$and` / `$or` / `$not`. The per-aggregation `filter` refuses the three declared ones; it already refused `$like` / `$ilike` as operators it does not evaluate. Through the engine, a structured-JSON field was already refused all seven text operators by the text-operator declared-type door, which still answers first there, in its own words; what moves for it is a direct driver call. + +**What an author sees.** The body the equality family already gets there, byte for byte: the filter WAS NOT APPLIED, and the spelling to use, `{ "FIELD": { "$contains": "a" } }` for membership or an `$or` of `$contains` for any-of. The field and the operator are withheld from the message and named in the server-log diagnostic; a filter positively marked as the caller's own reads them named. + +**Why a refusal.** Such a column stores the serialization `["u1","u2"]`, and none of these five operators has a membership reading. Measured through `POST /api/v1/data/:object/query` on a multi-value lookup and a `tags` field: on SQLite `$startsWith: "["` and `$endsWith: "]"` matched every row with a value, `$startsWith: "u1"` matched none of the rows holding `u1`, and `$icontains: "U1"` also matched the row holding only `u10`; on PostgreSQL 16 every one failed at query time with a `500` `DATABASE_ERROR`, a `json` column having no `LIKE` operator; the per-aggregation `filter` counted 0 for each. No membership reading is invented for a prefix, suffix or case-folded test. + +**Who is affected.** A saved filter, list view, dashboard widget, report or caller that aims one of these operators at a multi-valued or JSON-stored field. On SQLite it read rows that matched the stored brackets and quotes; it now gets the 400. On PostgreSQL it already failed, with a 500. Write `$contains` for "holds this member", an `$or` of `$contains` for "holds any of these", and `$not` around either for the exclusion. + +**Unchanged.** `$contains` and `$notContains` (membership on such a field), `$exists`, `$null` and `$empty`; every operator on a field that is not JSON-stored, the scalar text column included; `driver-memory`; and `driver-turso`'s remote transport, which compiles its own filters. From 9fa2d957bb576bda229d877ea5cc3c7142771f66 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 10:51:09 +0000 Subject: [PATCH 6/8] docs(core): drop a citation of a card that no longer resolves from the refusal set's docblock Claude-Session: https://claude.ai/code/session_01Ujdtvqs7ree7WyQmEDwEnG Co-authored-by: Claude --- packages/core/src/utils/json-column-operator-refusal.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/utils/json-column-operator-refusal.ts b/packages/core/src/utils/json-column-operator-refusal.ts index 8799778c586..7e2301c55ef 100644 --- a/packages/core/src/utils/json-column-operator-refusal.ts +++ b/packages/core/src/utils/json-column-operator-refusal.ts @@ -61,8 +61,8 @@ * membership pair (`$contains`, `$notContains`) and the null predicates * (`$null`, `$exists`, `$empty`). `$contains` is the ONLY working membership * spelling on a JSON-array column and downstream code depends on it (#7398's - * own tables) — `driver-sql` compiles it as a real per-dialect membership test - * (#17590), and `$notContains` as its exact complement — while `IS NULL` asks + * own tables) — `driver-sql` compiles it as a real per-dialect membership test, + * and `$notContains` as its exact complement — while `IS NULL` asks * about the column's presence, which is a well-formed question whatever the * column holds. * From 8c0212b27023f4292900f2538697e427b43d572e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 11:23:09 +0000 Subject: [PATCH 7/8] fix(objectql): $search matches a multi-valued field by $contains membership A field the object declares multi-valued is stored as a JSON array, where every operator but the membership pair is refused. The search expander emitted $in on a label match and $icontains otherwise, so one such field in the resolved set failed the whole search. A label term now becomes one $contains per matched option value, and any other term $contains of the term; scalar fields are unchanged. Claude-Session: https://claude.ai/code/session_01Ujdtvqs7ree7WyQmEDwEnG Co-authored-by: Claude --- .../21009-json-column-text-operators.md | 5 +- packages/objectql/src/search-filter.test.ts | 66 ++++++ packages/objectql/src/search-filter.ts | 36 ++++ .../dogfood/test/search-conformance.ledger.ts | 7 +- ...ata-search-multi-valued-membership.test.ts | 196 ++++++++++++++++++ 5 files changed, 308 insertions(+), 2 deletions(-) create mode 100644 packages/rest/src/data-search-multi-valued-membership.test.ts diff --git a/.changeset/21009-json-column-text-operators.md b/.changeset/21009-json-column-text-operators.md index 68d6a7b6ad2..a40f209bbc8 100644 --- a/.changeset/21009-json-column-text-operators.md +++ b/.changeset/21009-json-column-text-operators.md @@ -1,12 +1,13 @@ --- "@objectstack/core": minor +"@objectstack/objectql": minor --- fix(core)!: a filter that aims `$startsWith`, `$endsWith`, `$icontains`, `$like` or `$ilike` at a field stored as a JSON column is refused with `INVALID_FILTER` / 400, as `$eq` / `$in` / `$nin` already are, instead of matching the field's serialized text or failing at query time Clause-②: no (narrowing) - + **BREAKING**: this narrows which filters are answered on a field stored as a JSON column, on every face that reads `@objectstack/core`'s `JSON_COLUMN_INCOMPATIBLE_OPERATORS`: `driver-sql`'s `where` (and `driver-sqlite-wasm` and `driver-turso`'s local transport, which inherit it) on every read and write face that lowers a filter, and the engine's per-aggregation `filter`. It ships as `minor` under the launch-window convention for accept-set narrowings. @@ -18,4 +19,6 @@ Clause-②: no (narrowing) **Who is affected.** A saved filter, list view, dashboard widget, report or caller that aims one of these operators at a multi-valued or JSON-stored field. On SQLite it read rows that matched the stored brackets and quotes; it now gets the 400. On PostgreSQL it already failed, with a 500. Write `$contains` for "holds this member", an `$or` of `$contains` for "holds any of these", and `$not` around either for the exclusion. +**`@objectstack/objectql`: `$search` over a multi-valued field answers by membership.** The search expander (`$search` on `find`, `findOne` and `aggregate`, the REST `search` / `$search` parameter included) used to emit `$in` for a term matching a `select` option label and `$icontains` for any other term, against every field in the resolved search set. On a multi-valued field both are refused by the gate above, so one such field in the set failed the whole search: a label term answered 400 on every dialect, and any other term answered 500 on PostgreSQL and, with this change, 400 on SQLite. The auto-default set includes a `select` declared `multiple: true`, as in `examples/app-todo`'s `todo_task.tags`, and `searchableFields` may name a `tags` field or a multi-valued lookup. Such a field is now matched by membership: a term matching option labels becomes one `$contains` per matched option value, and any other term, or any term on a field with no options, becomes `$contains` of the term. No search answers 400 or 500 for it any more. **The visible cost:** to hit a multi-valued field, a term must now equal one of its members or match one of its option labels; SQLite used to match substrings of the stored array's serialized text as well, so a term like `wood` found a row tagged `redwood`, and it no longer does. Scalar fields are searched exactly as before. + **Unchanged.** `$contains` and `$notContains` (membership on such a field), `$exists`, `$null` and `$empty`; every operator on a field that is not JSON-stored, the scalar text column included; `driver-memory`; and `driver-turso`'s remote transport, which compiles its own filters. diff --git a/packages/objectql/src/search-filter.test.ts b/packages/objectql/src/search-filter.test.ts index 53b57ef5e37..48eb396fbff 100644 --- a/packages/objectql/src/search-filter.test.ts +++ b/packages/objectql/src/search-filter.test.ts @@ -180,4 +180,70 @@ describe('expandSearchToFilter', () => { ] }); }); }); + + /** + * [#21009] A field the object declares MULTI-VALUED is stored as a JSON array, + * where every operator but the membership pair is refused (`INVALID_FILTER` / + * 400). So it is searched by MEMBERSHIP: a label term becomes one `$contains` + * per matched option value, and a term matching no label — or a field with no + * options — becomes `$contains: term`. A scalar field keeps its clause. + */ + describe('[#21009] a multi-valued field is searched by membership', () => { + // app-todo's `todo_task.tags` shape: a `select` declared `multiple: true`, + // in the auto-default set (no `searchableFields`). + const taskFields: Record }> = { + subject: { type: 'text' }, + tags: { type: 'select', multiple: true, options: [ + { label: 'Important', value: 'important' }, + { label: 'Quick Win', value: 'quick_win' }, + { label: 'Quick Fix', value: 'quick_fix' }, + ] }, + status: { type: 'select', options: [{ label: 'Open', value: 'open' }] }, + }; + + it('the auto-default set includes the multi-valued select — the premise', () => { + expect(resolveSearchFields({ fields: taskFields })).toContain('tags'); + }); + + it('a label term: one $contains per matched option value, never $in', () => { + expect(expandSearchToFilter('important', { fields: taskFields })).toEqual({ $or: [ + { subject: { $icontains: 'important' } }, + { tags: { $contains: 'important' } }, + ] }); + // Two labels match → two membership clauses in the same $or (any-of). + expect(expandSearchToFilter('QUICK', { fields: taskFields, searchableFields: ['tags'] })).toEqual({ $or: [ + { tags: { $contains: 'quick_win' } }, + { tags: { $contains: 'quick_fix' } }, + ] }); + }); + + it('a term matching no label: $contains of the term, never $icontains', () => { + const f: SearchFilter | null = expandSearchToFilter('meeting', { fields: taskFields, searchableFields: ['tags'] }); + expect(f).toEqual({ $or: [{ tags: { $contains: 'meeting' } }] }); + expect(JSON.stringify(f)).not.toContain('$icontains'); + expect(JSON.stringify(f)).not.toContain('$in'); + }); + + it('an option-less multi-valued field (tags, a multi-valued lookup): $contains of the term', () => { + const noteFields = { + title: { type: 'text' }, + labels: { type: 'tags' }, + owners: { type: 'lookup', multiple: true }, + }; + expect(expandSearchToFilter('red', { fields: noteFields, searchableFields: ['title', 'labels', 'owners'] })).toEqual({ $or: [ + { title: { $icontains: 'red' } }, + { labels: { $contains: 'red' } }, + { owners: { $contains: 'red' } }, + ] }); + }); + + it('a scalar select and a scalar text field keep their clauses — the control', () => { + expect(expandSearchToFilter('open', { fields: taskFields, searchableFields: ['subject', 'status'] })).toEqual({ $or: [ + { subject: { $icontains: 'open' } }, + { status: { $in: ['open'] } }, + ] }); + expect(expandSearchToFilter('zzz', { fields: taskFields, searchableFields: ['status'] })) + .toEqual({ $or: [{ status: { $icontains: 'zzz' } }] }); + }); + }); }); diff --git a/packages/objectql/src/search-filter.ts b/packages/objectql/src/search-filter.ts index ae3676aaa59..44a09d96d53 100644 --- a/packages/objectql/src/search-filter.ts +++ b/packages/objectql/src/search-filter.ts @@ -41,9 +41,28 @@ * pinyin (`zhangwei`) and initials (`zw`) hit CJK names. Purely additive: * `resolveSearchFields` still returns only source fields (the companion is * invisible to `$searchFields` overrides and to clients). + * + * [#21009] A field the object declares MULTI-VALUED (`isMultiValueField`: a + * `tags` / `multiselect` / `checkboxes` field, or a `select` / `lookup` / + * `user` / … declared `multiple: true`) is matched by MEMBERSHIP, `$contains`, + * never by `$icontains` or `$in`. Such a field is stored as a JSON array, and + * every operator but the membership pair is refused on it with `INVALID_FILTER` + * / 400 (`@objectstack/core`'s `JSON_COLUMN_INCOMPATIBLE_OPERATORS`) — so one + * multi-valued field in the resolved set used to fail the WHOLE search: a label + * term's `$in` was refused on every dialect, and the raw `$icontains` was + * refused too once the text family joined that set (before it, PostgreSQL + * answered it with a 500 and SQLite matched substrings of the serialized + * array). The label → value mapping still applies: each matched option value + * becomes one `$contains` clause in the term's `$or`, and a term matching no + * label — or a field with no options, such as `tags` or a multi-valued lookup + * — becomes `$contains: term`. The visible cost: a term must EQUAL a member, + * or match an option label, to hit a multi-valued field. The declaration is + * read from the field map the engine already passes in (`fields`), whose + * entries are the object's full field definitions — `multiple` included. */ import { + isMultiValueField, resolveSearchFields, SEARCHABLE_ENUM_TYPES, type SearchFieldMeta, @@ -98,7 +117,24 @@ function optionValuesMatching(meta: SearchFieldMeta, term: string): unknown[] { return out; } +/** + * [#21009] Does the object declare this search field multi-valued? Asked of the + * field definition the engine handed in: `SearchFieldMeta` names only what the + * field RESOLUTION reads, but each entry of `fields` is the object's whole + * field definition, so its `multiple` flag is there to read. + */ +function isMultiValuedSearchField(meta: SearchFieldMeta): boolean { + const { type, multiple } = meta as SearchFieldMeta & { multiple?: unknown }; + return typeof type === 'string' && isMultiValueField({ type, multiple: multiple === true }); +} + function fieldClausesForTerm(field: string, term: string, meta: SearchFieldMeta): any[] { + // [#21009] Membership on a multi-valued field — see the module header. + if (isMultiValuedSearchField(meta)) { + const values = optionValuesMatching(meta, term).filter((v) => v !== null && v !== undefined); + if (values.length > 0) return values.map((v) => ({ [field]: { $contains: String(v) } })); + return [{ [field]: { $contains: term } }]; + } if (SEARCHABLE_ENUM_TYPES.has(meta?.type ?? '')) { const values = optionValuesMatching(meta, term); // The label→value path is already case-insensitive in JS (see diff --git a/packages/qa/dogfood/test/search-conformance.ledger.ts b/packages/qa/dogfood/test/search-conformance.ledger.ts index 8ebc8cebfcf..8d32763242c 100644 --- a/packages/qa/dogfood/test/search-conformance.ledger.ts +++ b/packages/qa/dogfood/test/search-conformance.ledger.ts @@ -25,7 +25,12 @@ export const SEARCH_SURFACE: ConformanceRow[] = [ // emits `$icontains` — the operator that actually folds — so the row names // it. Neither operator's own semantics moved; only what `$search` compiles // to did. - summary: '`$search` server-resolved cross-field executor (terms AND-ed, fields OR-ed, case-insensitive via `$icontains`)', + // [#21009] A MULTI-VALUED field (stored as a JSON array, where only the + // membership pair answers) is matched by `$contains` membership instead. + // No showcase object carries one in its search set, so that half's + // HTTP-level proof is `packages/rest`'s + // `data-search-multi-valued-membership.test.ts`, not this row's dogfood file. + summary: '`$search` server-resolved cross-field executor (terms AND-ed, fields OR-ed, case-insensitive via `$icontains`; a multi-valued field by `$contains` membership)', surface: 'spec/api/query.zod.ts:$search (QueryParams `search`)', state: 'enforced', enforcement: 'objectql/src/engine.ts (find AST expansion) → objectql/src/search-filter.ts expandSearchToFilter', diff --git a/packages/rest/src/data-search-multi-valued-membership.test.ts b/packages/rest/src/data-search-multi-valued-membership.test.ts new file mode 100644 index 00000000000..637a1449542 --- /dev/null +++ b/packages/rest/src/data-search-multi-valued-membership.test.ts @@ -0,0 +1,196 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21009] `$search` over a MULTI-VALUED field answers by membership, through + * the door a caller uses: `POST /api/v1/data/:object/query` with `search` → + * `RestServer` → `ObjectStackProtocolImplementation.findData` → `ObjectQL.find` + * (the search expander) → a real `SqlDriver`. + * + * Such a field is stored as a JSON array, and every operator but the membership + * pair is refused on it (`INVALID_FILTER` / 400). The expander used to emit `$in` + * for a label term and `$icontains` otherwise, so one multi-valued field in the + * resolved set failed the WHOLE search. Measured on `origin/main` `7a606a9a3` + * over the two objects below: + * + * | search | SQLite | PostgreSQL 16.14 | + * |:--|:--|:--| + * | task, a term matching no label | 200 (by subject) | 500 | + * | task, a term matching a `tags` label | 400 (the `$in`) | 400 | + * | note with a declared searchable `tags` | 200 (substrings of the serialized array) | 500 | + * + * With the text family refused on a JSON column too, every one of them answered + * 400 on both dialects until the expander emitted membership. Now a label term + * finds the rows HOLDING that option value, a raw member term finds the rows + * holding it, and a term that is no member finds none from that field — with no + * 400 and no 500 on any term. The scalar fields beside them are the control. + * + * ## The dialect axis of THIS file + * + * The SQLite cell always runs. The PostgreSQL and MySQL cells run where + * `OS_TEST_POSTGRES_URL` / `OS_TEST_MYSQL_URL` are set and are a named skip + * otherwise; each owns its tables, dropped before and after. + */ + +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { RestServer } from './rest-server'; + +/** `examples/app-todo`'s `todo_task.tags` shape: a `select` declared `multiple: true`, in the auto-default set. */ +const TASK = { + name: 'rest_search_task_21009', + label: 'Task 21009', + fields: { + subject: { name: 'subject', type: 'text' as const }, + tags: { + name: 'tags', type: 'select' as const, multiple: true, + options: [{ label: 'Important', value: 'important' }, { label: 'Quick Win', value: 'quick_win' }], + }, + status: { + name: 'status', type: 'select' as const, + options: [{ label: 'Open', value: 'open' }, { label: 'Done', value: 'done' }], + }, + }, +}; + +/** A declared searchable `tags` field (no options), beside a scalar text one. */ +const NOTE = { + name: 'rest_search_note_21009', + label: 'Note 21009', + searchableFields: ['title', 'labels'], + fields: { + title: { name: 'title', type: 'text' as const }, + labels: { name: 'labels', type: 'tags' as const }, + }, +}; + +const TASK_ROWS = [ + { id: 't1', subject: 'Write meeting notes', tags: ['important'], status: 'open' }, + { id: 't2', subject: 'Plan sprint', tags: ['quick_win'], status: 'done' }, + { id: 't3', subject: 'Call vendor', tags: ['important', 'quick_win'], status: 'open' }, + { id: 't4', subject: 'Archive', tags: [], status: 'done' }, + { id: 't5', subject: 'Misc', tags: null, status: null }, +]; + +const NOTE_ROWS = [ + { id: 'n1', title: 'Grocery list', labels: ['red', 'home'] }, + { id: 'n2', title: 'Red alert', labels: ['work'] }, + { id: 'n3', title: 'Blue sky', labels: ['redwood'] }, +]; + +/** [object, term, the ids the search answers, what the row pins] */ +const CASES: ReadonlyArray = [ + [TASK.name, 'Important', ['t1', 't3'], 'a label term finds the rows holding that value'], + [TASK.name, 'quick', ['t2', 't3'], 'a partial label term finds the rows holding the matched value'], + [TASK.name, 'quick_win', ['t2', 't3'], 'a raw member term (no label contains it) finds the rows holding it'], + [TASK.name, 'zebra', [], 'a term that is no member and no label finds none'], + [TASK.name, 'meeting', ['t1'], 'a term only the scalar subject holds still finds by the subject'], + [TASK.name, 'Open', ['t1', 't3'], 'control: the scalar select keeps its label → value match'], + [NOTE.name, 'red', ['n1', 'n2'], 'a raw member term finds the member row; the scalar title finds its own'], + [NOTE.name, 'redwood', ['n3'], 'a member term finds the row holding exactly that member'], + [NOTE.name, 'wood', [], 'a substring of a member is not a member'], + [NOTE.name, 'Blue', ['n3'], 'control: the scalar text field still folds case'], +]; + +interface Cell { + id: 'sqlite' | 'pg' | 'mysql'; + label: string; + env: string | null; + config: () => Record | null; +} + +const CELLS: readonly Cell[] = [ + { id: 'sqlite', label: 'sqlite', env: null, config: () => ({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) }, + { + id: 'pg', + label: 'live postgres', + env: 'OS_TEST_POSTGRES_URL', + config: () => (process.env.OS_TEST_POSTGRES_URL ? { client: 'pg', connection: process.env.OS_TEST_POSTGRES_URL } : null), + }, + { + id: 'mysql', + label: 'live mysql', + env: 'OS_TEST_MYSQL_URL', + config: () => (process.env.OS_TEST_MYSQL_URL ? { client: 'mysql2', connection: process.env.OS_TEST_MYSQL_URL } : null), + }, +]; + +function createMockServer() { + const noop = () => {}; + return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} }; +} + +function makeRes() { + const res: any = { + write: () => true, end: () => {}, + header: () => res, + status: (code: number) => { res._status = code; return res; }, + json: (body: any) => { res._json = body; return res; }, + }; + return res; +} + +for (const cell of CELLS) { + const config = cell.config(); + describe.skipIf(!config)( + `[#21009] POST /data/:object/query with search — a multi-valued field answers by membership — ${cell.label}${config ? '' : ` (skipped: set ${cell.env} to run this cell)`}`, + () => { + let engine: ObjectQL; + let driver: any; + let post: (object: string, body: Record) => Promise<{ status: number; json: any }>; + + const dropTables = async () => { + if (cell.id === 'sqlite') return; + for (const o of [TASK, NOTE]) await driver?.execute(`drop table if exists ${o.name}`).catch(() => {}); + }; + + beforeAll(async () => { + driver = new SqlDriver(config as any); + await dropTables(); + engine = new ObjectQL(); + engine.registerDriver(driver, true); + await engine.init(); + engine.registry.registerObject(TASK as any); + engine.registry.registerObject(NOTE as any); + await engine.syncSchemas(); + for (const row of TASK_ROWS) await engine.insert(TASK.name, { ...row } as any); + for (const row of NOTE_ROWS) await engine.insert(NOTE.name, { ...row } as any); + vi.spyOn((engine as any).logger, 'warn').mockImplementation(() => undefined); + + const protocol = new ObjectStackProtocolImplementation(engine as any); + const rest = new RestServer(createMockServer() as any, protocol as any, { api: { requireAuth: false } } as any); + (rest as any).resolveExecCtx = async () => ({ userId: 'test-user' }); + rest.registerRoutes(); + const route = rest.getRoutes().find((r: any) => r.method === 'POST' && r.path === '/api/v1/data/:object/query'); + expect(route).toBeDefined(); + post = async (object, body) => { + const res = makeRes(); + await route!.handler({ params: { object }, body: JSON.parse(JSON.stringify(body)) } as any, res); + return { status: res._status ?? 200, json: res._json }; + }; + }, 60_000); + + afterAll(async () => { + await dropTables(); + await engine?.destroy().catch(() => {}); + }, 60_000); + + it('stored every row — the premise of every answer below', async () => { + for (const [object, rows] of [[TASK.name, TASK_ROWS], [NOTE.name, NOTE_ROWS]] as const) { + const r = await post(object, {}); + expect(r.status, JSON.stringify(r.json)).toBe(200); + expect(r.json.records.map((x: any) => x.id).sort()).toEqual(rows.map((x) => x.id).sort()); + } + }, 60_000); + + for (const [object, term, ids, what] of CASES) { + it(`${object === TASK.name ? 'task' : 'note'} search "${term}": 200 ${JSON.stringify(ids)} — ${what}`, async () => { + const r = await post(object, { search: term }); + expect(r.status, JSON.stringify(r.json)).toBe(200); + expect(r.json.records.map((x: any) => x.id).sort()).toEqual(ids); + }, 60_000); + } + }, + ); +} From 75562d04b430e0be76275704e1bccacaf0e695f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 11:27:02 +0000 Subject: [PATCH 8/8] test(objectql): the auto-default search set keeps the scalar status clause beside the membership one Claude-Session: https://claude.ai/code/session_01Ujdtvqs7ree7WyQmEDwEnG Co-authored-by: Claude --- packages/objectql/src/search-filter.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/objectql/src/search-filter.test.ts b/packages/objectql/src/search-filter.test.ts index 48eb396fbff..7627d3a58f8 100644 --- a/packages/objectql/src/search-filter.test.ts +++ b/packages/objectql/src/search-filter.test.ts @@ -206,9 +206,11 @@ describe('expandSearchToFilter', () => { }); it('a label term: one $contains per matched option value, never $in', () => { + // The auto-default set: the scalar `status` beside it keeps its fallback. expect(expandSearchToFilter('important', { fields: taskFields })).toEqual({ $or: [ { subject: { $icontains: 'important' } }, { tags: { $contains: 'important' } }, + { status: { $icontains: 'important' } }, ] }); // Two labels match → two membership clauses in the same $or (any-of). expect(expandSearchToFilter('QUICK', { fields: taskFields, searchableFields: ['tags'] })).toEqual({ $or: [