From 1dba88b8c8f38a7d7de1f821715faef2747795c3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 06:39:34 +0000 Subject: [PATCH 1/5] fix(objectql)!: refuse a groupBy on a structured-JSON field at the engine's aggregate door A groupBy entry naming a json, composite, repeater, record, location, address or vector field is refused INVALID_FIELD / 400 before any driver is asked, in both entry spellings. Claude-Session: https://claude.ai/code/session_01DEvba2nBuD4tWzfq8r8NFY Co-authored-by: Claude --- packages/objectql/src/engine.ts | 9 ++ .../src/group-by-structured-json-door.ts | 139 ++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 packages/objectql/src/group-by-structured-json-door.ts diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 0bf48c44594..67198fc4a8f 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -195,6 +195,7 @@ import { EmptyCredentialWriteError, SECRET_MASK, } from './secret-fields.js'; +import { assertGroupByNamesNoStructuredJsonField } from './group-by-structured-json-door.js'; import { pluralToSingular, ExternalWriteForbiddenError } from '@objectstack/spec/shared'; import { SchemaRegistry, computeFQN, type ArtifactInstallScope } from './registry.js'; import { expandSearchToFilter } from './search-filter.js'; @@ -16281,6 +16282,14 @@ export class ObjectQL implements IObjectQLEngine { // the `where` doors above, before the AST is built and tokens resolve). query = this.expandSearchOnAggregateOptions(object, query); this.rejectCredentialAggregation(object, query); + // [#20783] …and a `groupBy` entry naming a structured-JSON field (`json`, + // `composite`, `address`, …) is refused `INVALID_FIELD` / 400 here, before + // any driver is asked: the drivers share no meaning for a JSON document as + // a group key (memory merged every row into one group, SQLite grouped each + // serialized document apart, PostgreSQL answered 500). After the + // credential refusal, which reads the same entries, so a protected field + // keeps that refusal's words. + assertGroupByNamesNoStructuredJsonField(object, this._registry.getObject(object), query.groupBy); // [#10576] The per-aggregation `filter` (`AggregationNodeSchema.filter`, // the contract half of #10413) is a second filter position on this verb, // so it walks through the same refusal doors `where` does at this seam: diff --git a/packages/objectql/src/group-by-structured-json-door.ts b/packages/objectql/src/group-by-structured-json-door.ts new file mode 100644 index 00000000000..4987e17655b --- /dev/null +++ b/packages/objectql/src/group-by-structured-json-door.ts @@ -0,0 +1,139 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20783] A `groupBy` that names a STRUCTURED-JSON field — `json`, + * `composite`, `repeater`, `record`, `location`, `address`, `vector` — is + * refused with `INVALID_FIELD` / 400 by the engine's `aggregate`, naming the + * field, its declared type and the position, before any driver is asked. + * + * ## What ran before this door, measured on `origin/main` `7a09eee1b1` + * + * Through `POST /api/v1/data/:object/query` (`{ groupBy: [FIELD], + * aggregations: [{ function: 'count', alias: 'n' }] }`) and `engine.aggregate` + * alike, three rows whose values under the grouped field differ: + * + * | `groupBy` | InMemoryDriver | SqlDriver, SQLite | SqlDriver, PostgreSQL 16 | + * |:--|:--|:--|:--| + * | a `text` field (the control) | 200, one group per value | same | same | + * | a `json` field, and its `composite`, `repeater`, `record`, `location`, `address` twins | 200, **one group** holding every row | 200, one group per serialized document | **500 `DATABASE_ERROR`** | + * | a `vector` field | 200, one group per array | 200, one group per serialized array | 500 | + * | `{ field: 'meta', dateGranularity: 'month' }` over a `json` field | 200, one `null` bucket | 200, one `null` bucket | 500 | + * + * One query, three answers, one of them a server error. No member of the class + * answers one way on the three drivers, so the class is refused, not its + * `json` member alone. + * + * ## Refuse, don't define + * + * The triage direction on the card: grouping by a JSON document has no meaning + * the drivers share, no producer that groups by one was measured (no dataset, + * cube, view grouping or `groupBy` in `examples/` names a structured-JSON + * field), and #20745 refused the JSON object comparand at this same door. So + * the engine refuses it rather than choosing a serialization for every driver + * to agree on. ⛔ No per-driver serialization rule. A real producer that needs + * a meaning proposes one on a card of its own. + * + * ## Where it stands, and what it judges + * + * `aggregate` is the one engine verb that takes `groupBy` (`find` refuses the + * key: `ENGINE_FIND_OPTION_KEYS`), and this door runs at its entry, beside the + * credential refusal that reads the same `groupBy` entries, so it holds for + * every caller that reaches the engine: the REST query door, a flow, a hook, + * and the analytics strategy that lowers a cube query onto `engine.aggregate`. + * Both spellings of an entry are judged — a field name, and the `{ field }` + * object (a date bucket included: no granularity makes a JSON document a + * date). + * + * The class is `@objectstack/spec/data`'s {@link STRUCTURED_JSON_TYPES}, the + * set #20745's JSON arm judges too, never a list minted here. **Not judged:** + * an undeclared name (the engine's registry-less tolerance — the ingress door + * answers an unknown one `INVALID_FIELD` first), a registry-less host (no + * field map, no verdict), and every other type, `multiple: true` lists and + * file fields included: those are not this card's class. + * + * `INVALID_FIELD`, not a new code: the verdict is about the NAMED field's + * type at a position, the question the ingress door answers with + * `INVALID_FIELD` for an unknown `groupBy` name and the search axis answers + * with `INVALID_FIELD` for a field whose type it cannot scan. + * + * @see https://github.com/objectstack-ai/objectstack/issues/20783 + */ + +import { StandardErrorCode } from '@objectstack/spec/api'; +import { STRUCTURED_JSON_TYPES } from '@objectstack/spec/data'; + +/** One `groupBy` entry that names a structured-JSON field. */ +interface StructuredJsonGroupTarget { + readonly field: string; + readonly type: string; + /** `groupBy[i]` for a name, `groupBy[i].field` for the object form. */ + readonly position: string; +} + +/** + * The entries of `groupBy` that name a declared structured-JSON field, in + * order. An entry that names no field, a field the map does not declare, or a + * field of any other type is not collected. + */ +function structuredJsonGroupTargets( + fields: Record, + groupBy: readonly unknown[], +): StructuredJsonGroupTarget[] { + const hits: StructuredJsonGroupTarget[] = []; + for (const [i, entry] of groupBy.entries()) { + const objectForm = entry !== null && typeof entry === 'object' && !Array.isArray(entry); + const field = typeof entry === 'string' + ? entry + : objectForm ? (entry as { field?: unknown }).field : undefined; + if (typeof field !== 'string') continue; + if (!Object.prototype.hasOwnProperty.call(fields, field)) continue; + const type = (fields[field] as { type?: unknown } | undefined)?.type; + if (typeof type !== 'string' || !STRUCTURED_JSON_TYPES.has(type)) continue; + hits.push({ field, type, position: objectForm ? `groupBy[${i}].field` : `groupBy[${i}]` }); + } + return hits; +} + +/** + * Refuse a `groupBy` entry that names a declared structured-JSON field — + * `INVALID_FIELD` / 400, before any driver is asked. See the module header. + * + * The words put the position and the verdict first, then that the query did + * not run, then the route, then the reason: the REST door keeps the first 500 + * characters of a 4xx message (`CLIENT_MESSAGE_MAX`), and the route must be + * inside them. + */ +export function assertGroupByNamesNoStructuredJsonField( + object: string, + schema: unknown, + groupBy: unknown, +): void { + if (!Array.isArray(groupBy) || groupBy.length === 0) return; + const fields = (schema as { fields?: unknown } | undefined)?.fields; + if (!fields || typeof fields !== 'object') return; + const hits = structuredJsonGroupTargets(fields as Record, groupBy); + if (hits.length === 0) return; + const [first] = hits; + const err = new Error( + `aggregate('${object}'): ${first.position} names '${first.field}', a declared ${first.type} field ` + + '— a structured-JSON value, which the engine does not group by' + + (hits.length > 1 ? ` (also: ${hits.slice(1).map((h) => `'${h.field}'`).join(', ')})` : '') + + '. The query was NOT run. Group by a field that stores one scalar value: store the part you ' + + 'group on in a field of its own and group by that field. A JSON document is no group key the ' + + 'drivers share: one merged every row into a single group, one grouped each serialized ' + + 'document apart, one refused the statement.', + ) as Error & { + code?: string; status?: number; httpStatus?: number; + field?: string; fields?: string[]; object?: string; param?: string; + }; + err.code = StandardErrorCode.enum.INVALID_FIELD; + err.status = 400; + // …and `httpStatus`, the same number under ADR-0112 D5's spelling — what a + // consumer holding the THROWN error reads; `status` stays for the HTTP doors. + err.httpStatus = 400; + err.field = first.field; + err.fields = hits.map((h) => h.field); + err.object = object; + err.param = 'groupBy'; + throw err; +} From 8af7b08c00bfe1eb27fe875f96554856215a495b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 06:40:52 +0000 Subject: [PATCH 2/5] test(objectql): pin the structured-JSON groupBy refusal at the engine door Claude-Session: https://claude.ai/code/session_01DEvba2nBuD4tWzfq8r8NFY Co-authored-by: Claude --- .../src/engine-group-by-json-door.test.ts | 196 ++++++++++++++++++ .../src/engine-nested-object-door.test.ts | 4 +- .../number-comparand-declared-type-door.ts | 4 +- 3 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 packages/objectql/src/engine-group-by-json-door.test.ts diff --git a/packages/objectql/src/engine-group-by-json-door.test.ts b/packages/objectql/src/engine-group-by-json-door.test.ts new file mode 100644 index 00000000000..ee8a3571ee9 --- /dev/null +++ b/packages/objectql/src/engine-group-by-json-door.test.ts @@ -0,0 +1,196 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20783] A `groupBy` entry naming a STRUCTURED-JSON field (`json`, + * `composite`, `repeater`, `record`, `location`, `address`, `vector`) is + * refused `INVALID_FIELD` / 400 by `engine.aggregate`, naming the field, its + * declared type and the position, before any driver is asked + * (`group-by-structured-json-door.ts`). + * + * Measured on the base (`origin/main` `7a09eee1b1`) through + * `POST /api/v1/data/:object/query`, `{ groupBy: [FIELD], aggregations: + * [{ function: 'count', alias: 'n' }] }` over three rows: + * + * | `groupBy` | InMemoryDriver | SqlDriver, SQLite | SqlDriver, PostgreSQL 16 | + * |:--|:--|:--|:--| + * | `text` (the control) | 200, one group per value | same | same | + * | `json`, and its `composite`, `repeater`, `record`, `location`, `address` twins | 200, one group holding every row | 200, one group per serialized document | 500 `DATABASE_ERROR` | + * | `vector` | 200, one group per array | 200, one group per serialized array | 500 | + * | `{ field: 'meta', dateGranularity: 'month' }` (json) | 200, one `null` bucket | 200, one `null` bucket | 500 | + * + * The InMemoryDriver cell is this suite's recording driver by construction: + * the door answers before a driver is resolved, so no read runs. The SQL cells + * over a real driver live in `@objectstack/rest`'s + * `data-group-by-json-door.test.ts`; that driver's test consumers are a ruled, + * closed census (`check:driver-memory-census`), so no new suite of it here. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { FieldType, STRUCTURED_JSON_TYPES, type EngineAggregateOptions } from '@objectstack/spec/data'; +import { ObjectQL } from './engine.js'; +import { assertGroupByNamesNoStructuredJsonField } from './group-by-structured-json-door.js'; + +const OBJECT = 'group_by_json_probe'; + +/** field · declared type — one field of every structured-JSON type. */ +const JSONS: ReadonlyArray = [ + ['meta', 'json'], + ['spec', 'composite'], + ['rep', 'repeater'], + ['rec', 'record'], + ['loc', 'location'], + ['ship_to', 'address'], + ['vec', 'vector'], +]; + +const PROBE = { + name: OBJECT, + label: 'Group-by JSON probe', + fields: { + title: { name: 'title', type: 'text' }, + amount: { name: 'amount', type: 'number' }, + tags: { name: 'tags', type: 'select', multiple: true, options: [{ label: 'A', value: 'a' }] }, + photo: { name: 'photo', type: 'image' }, + ...Object.fromEntries(JSONS.map(([name, type]) => [name, { name, type }])), + }, +}; + +const COUNT = [{ function: 'count', alias: 'n' }] as EngineAggregateOptions['aggregations']; + +interface SeenRead { ast: any } + +/** Minimal recording driver — the same witness shape as the sibling door suites. */ +function makeRecordingDriver() { + const rows = new Map>(); + const reads: SeenRead[] = []; + const run = (_ast: any) => [...rows.values()]; + const driver: any = { + name: 'recording', version: '0.0.0', supports: {}, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, + async find(_o: string, ast: any) { reads.push({ ast }); return run(ast); }, + async findOne(_o: string, ast: any) { reads.push({ ast }); return run(ast)[0] ?? null; }, + async count(_o: string, ast: any) { reads.push({ ast }); return run(ast).length; }, + async create(_o: string, data: Record) { + const id = (data.id as string) ?? `r_${rows.size + 1}`; + const row = { ...data, id }; rows.set(id, row); return row; + }, + async update(_o: string, id: string, data: Record) { + const up = { ...(rows.get(id) ?? {}), ...data, id }; rows.set(id, up); return up; + }, + async delete(_o: string, id: string) { return rows.delete(id); }, + async beginTransaction() { return { commit: async () => {}, rollback: async () => {} }; }, + async commit() {}, async rollback() {}, + }; + return { driver, reads }; +} + +type Thrown = (Error & { + code?: string; status?: number; httpStatus?: number; field?: string; fields?: string[]; object?: string; param?: string; +}) | null; + +const refusalOf = async (p: Promise): Promise => p.then(() => null, (e: any) => e); + +const ENVELOPE = { code: 'INVALID_FIELD', status: 400, httpStatus: 400 }; +const envelopeOf = (err: Thrown) => ({ code: err?.code, status: err?.status, httpStatus: err?.httpStatus }); + +/** The door's own words, in every refusal it raises — a control must never be answered in them. */ +const DOOR_WORDS = 'which the engine does not group by'; + +describe('[#20783] a groupBy on a structured-JSON field, at the engine\'s aggregate door', () => { + let engine: ObjectQL; + let reads: SeenRead[]; + + beforeEach(async () => { + const rec = makeRecordingDriver(); + reads = rec.reads; + engine = new ObjectQL(); + engine.registerDriver(rec.driver, true); + await engine.init(); + engine.registry.registerObject(PROBE as any, 'test'); + reads.length = 0; + }); + + it('refuses a groupBy on every structured-JSON type with INVALID_FIELD / 400, naming the field, its type and the position — no read', async () => { + for (const [field, type] of JSONS) { + const err = await refusalOf(engine.aggregate(OBJECT, { groupBy: [field], aggregations: COUNT })); + expect(envelopeOf(err), field).toEqual(ENVELOPE); + expect({ field: err?.field, fields: err?.fields, object: err?.object, param: err?.param }, field) + .toEqual({ field, fields: [field], object: OBJECT, param: 'groupBy' }); + expect(err!.message, field).toMatch( + new RegExp(`^aggregate\\('${OBJECT}'\\): groupBy\\[0\\] names '${field}', a declared ${type} field `), + ); + expect(err!.message, field).toContain('The query was NOT run.'); + } + expect(reads, 'every refusal precedes the driver').toHaveLength(0); + }); + + it('refuses the { field } object form, a date bucket over it, and an entry after a scalar one, at the entry\'s own position — no read', async () => { + const cases: ReadonlyArray = [ + [[{ field: 'meta' }], 'groupBy[0].field', ['meta']], + [[{ field: 'meta', dateGranularity: 'month' }], 'groupBy[0].field', ['meta']], + [['title', 'ship_to'], 'groupBy[1]', ['ship_to']], + [['meta', 'title', { field: 'loc' }], 'groupBy[0]', ['meta', 'loc']], + ] as ReadonlyArray; + for (const [groupBy, position, fields] of cases) { + const label = JSON.stringify(groupBy); + const err = await refusalOf(engine.aggregate(OBJECT, { groupBy, aggregations: COUNT })); + expect(envelopeOf(err), label).toEqual(ENVELOPE); + expect(err?.fields, label).toEqual(fields); + expect(err!.message, label).toContain(`): ${position} names '${fields[0]}',`); + } + expect(reads).toHaveLength(0); + }); + + it('CONTROL a text, number, multi-value select, file or undeclared groupBy reaches the driver, never this refusal', async () => { + for (const field of ['title', 'amount', 'tags', 'photo', 'not_declared']) { + const before = reads.length; + const out = await engine.aggregate(OBJECT, { groupBy: [field], aggregations: COUNT }).then( + () => null, + (e: Error) => e.message, + ); + expect(out ?? '', field).not.toContain(DOOR_WORDS); + expect(reads.length - before, `${field}: the driver was asked`).toBe(1); + } + // A structured-JSON field as an AGGREGATED column is not this door's. + await expect(engine.aggregate(OBJECT, { + groupBy: ['title'], + aggregations: [{ function: 'count', field: 'meta', alias: 'n' }], + } as EngineAggregateOptions)).resolves.toBeDefined(); + }); + + it('the REST door into findData answers the same refusal — no read', async () => { + const protocol = new ObjectStackProtocolImplementation(engine); + const err = await refusalOf(protocol.findData({ + object: OBJECT, + query: { groupBy: ['meta'], aggregations: COUNT }, + } as any)); + expect(envelopeOf(err)).toEqual(ENVELOPE); + expect(err!.message).toContain(`groupBy[0] names 'meta', a declared json field`); + expect(reads).toHaveLength(0); + }); + + it('GUARD the judged types are exactly the spec\'s STRUCTURED_JSON_TYPES, over every FieldType', () => { + for (const type of FieldType.options) { + const thrown = (() => { + try { + assertGroupByNamesNoStructuredJsonField(OBJECT, { fields: { f: { type } } }, ['f']); + return null; + } catch (e) { + return e as Thrown; + } + })(); + expect(thrown === null ? null : envelopeOf(thrown), type) + .toEqual(STRUCTURED_JSON_TYPES.has(type) ? ENVELOPE : null); + } + }); + + it('GUARD no verdict without a field map, for an undeclared name, or for an entry that names no field', () => { + const judge = (schema: unknown, groupBy: unknown) => () => assertGroupByNamesNoStructuredJsonField(OBJECT, schema, groupBy); + expect(judge(undefined, ['meta'])).not.toThrow(); + expect(judge({}, ['meta'])).not.toThrow(); + expect(judge(PROBE, ['nope', { field: 'nope' }, { dateGranularity: 'month' }, 7, null])).not.toThrow(); + expect(judge(PROBE, 'meta')).not.toThrow(); + expect(judge(PROBE, [])).not.toThrow(); + }); +}); diff --git a/packages/objectql/src/engine-nested-object-door.test.ts b/packages/objectql/src/engine-nested-object-door.test.ts index 4251b5f963f..603b3df64b4 100644 --- a/packages/objectql/src/engine-nested-object-door.test.ts +++ b/packages/objectql/src/engine-nested-object-door.test.ts @@ -294,7 +294,9 @@ describe('[#20745] a no-operator object beneath a relation, structured-JSON or p const cases: ReadonlyArray = [ [{ groupBy: ['owner'], aggregations: [{ function: 'count', alias: 'n' }], having: { owner: { region: 'NA' } } }, 'owner', 'lookup', 'nested-relation form'], [{ groupBy: ['title'], aggregations: [{ function: 'max', field: 'boss', alias: 'top' }], having: { top: { region: 'NA' } } }, 'top', 'master_detail', 'nested-relation form'], - [{ groupBy: ['meta'], aggregations: [{ function: 'count', alias: 'n' }], having: { meta: { a: 1 } } }, 'meta', 'json', 'whole-value match'], + // [#20783] A JSON column reaches `having` as a `max` of a json field: a + // json GROUPBY is refused one door earlier (`engine-group-by-json-door.test.ts`). + [{ groupBy: ['title'], aggregations: [{ function: 'max', field: 'meta', alias: 'top_meta' }], having: { top_meta: { a: 1 } } }, 'top_meta', 'json', 'whole-value match'], ] as ReadonlyArray; for (const [query, column, type, words] of cases) { reads.length = 0; diff --git a/packages/objectql/src/number-comparand-declared-type-door.ts b/packages/objectql/src/number-comparand-declared-type-door.ts index ea19d195925..6c489c283cb 100644 --- a/packages/objectql/src/number-comparand-declared-type-door.ts +++ b/packages/objectql/src/number-comparand-declared-type-door.ts @@ -507,7 +507,9 @@ export function narrowNumberComparands( * A `json` or `lookup` groupBy (or a `min` / `max` of one) is judged now, by * that same type: the engine evaluates `having` itself, and a relation column * there carries the related record's id, never the record — measured, a - * nested-relation `having` kept no group on every driver. + * nested-relation `having` kept no group on every driver. [#20783] A `json` + * GROUPBY no longer reaches here: `aggregate` refuses it at its entry + * (`group-by-structured-json-door.ts`); a `min` / `max` of a json field does. */ export function narrowHavingNumberComparands( object: string, From a5a35834de483ccd8f7f02f9da21ce351e27d685 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 06:41:57 +0000 Subject: [PATCH 3/5] test(rest): pin the structured-JSON groupBy refusal at the query door on SQLite and live SQL Claude-Session: https://claude.ai/code/session_01DEvba2nBuD4tWzfq8r8NFY Co-authored-by: Claude --- .../rest/src/data-group-by-json-door.test.ts | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 packages/rest/src/data-group-by-json-door.test.ts diff --git a/packages/rest/src/data-group-by-json-door.test.ts b/packages/rest/src/data-group-by-json-door.test.ts new file mode 100644 index 00000000000..46185cb8cc3 --- /dev/null +++ b/packages/rest/src/data-group-by-json-door.test.ts @@ -0,0 +1,204 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20783] A `groupBy` on a structured-JSON field is refused at the public + * door — `POST /api/v1/data/:object/query` answers `400 INVALID_FIELD` in the + * engine's words, naming the field, its type and the position, before any + * read — over a real `SqlDriver`; and a `text` field's `groupBy` (the control) + * is served unchanged. + * + * Measured on the base (`origin/main` `7a09eee1b1`) through this door, three + * rows, `{ groupBy: [FIELD], aggregations: [{ function: 'count', alias: 'n' }] }`: + * + * | `groupBy` | InMemoryDriver | SQLite | PostgreSQL 16 | + * |:--|:--|:--|:--| + * | `title` (text, the control) | 200, `x` 2 · `y` 1 | same | same | + * | `meta` (json; composite, repeater, record, location, address alike) | 200, one group, `n` 3 | 200, one group per serialized document | 500 `DATABASE_ERROR` | + * | `vec` (vector) | 200, one group per array | 200, one group per serialized array | 500 | + * | `{ field: 'meta', dateGranularity: 'month' }` | 200, one `null` bucket | 200, one `null` bucket | 500 | + * + * The refusal sits in the engine, in front of every driver, so one verdict + * holds on each cell. InMemoryDriver's row is `@objectstack/objectql`'s + * `engine-group-by-json-door.test.ts` by construction (the door answers before + * a driver is resolved); this package does not depend on the in-memory + * driver, and that driver's test consumers are a ruled, closed census + * (`check:driver-memory-census`). + * + * ## The dialect axis of THIS file + * + * The SQLite cell always runs. The PostgreSQL and MySQL cells run where + * `OS_TEST_POSTGRES_URL` / `OS_TEST_MYSQL_URL` are set and are a named skip + * otherwise. ⚠️ No CI job provisions those variables for this package, so the + * live cells are red-capable and un-run in CI; the PR that landed this file + * carries their local PostgreSQL run. Each live cell owns its table, dropped + * before and after. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { RestServer } from './rest-server'; + +const OBJECT = 'rest_group_by_json_20783'; + +/** field · declared type — one field of every structured-JSON type. */ +const JSONS: ReadonlyArray = [ + ['meta', 'json'], + ['spec', 'composite'], + ['rep', 'repeater'], + ['rec', 'record'], + ['loc', 'location'], + ['ship_to', 'address'], + ['vec', 'vector'], +]; + +const LEDGER = { + name: OBJECT, + label: 'Ledger 20783', + fields: { + title: { name: 'title', type: 'text' as const }, + ...Object.fromEntries(JSONS.map(([name, type]) => [name, { name, type }])), + }, +}; + +const ROWS = [ + { id: 'd1', title: 'x', meta: { a: 1 }, spec: { k: 1 }, rep: [{ q: 1 }], rec: { r: 1 }, loc: { lat: 1, lng: 2 }, ship_to: { city: 'Paris' }, vec: [1, 2] }, + { id: 'd2', title: 'x', meta: { a: 2 }, spec: { k: 2 }, rep: [{ q: 2 }], rec: { r: 2 }, loc: { lat: 3, lng: 4 }, ship_to: { city: 'Rome' }, vec: [3, 4] }, + { id: 'd3', title: 'y', meta: { b: 1 }, spec: { k: 1 }, rep: [{ q: 1 }], rec: { r: 1 }, loc: { lat: 1, lng: 2 }, ship_to: { city: 'Paris' }, vec: [1, 2] }, +]; + +const COUNT = [{ function: 'count', alias: 'n' }]; + +/** The route the refusal names — asserted on the REST body, so it must land inside the door's 500-character bound. */ +const ROUTE = 'Group by a field that stores one scalar value: store the part you group on in a field of its own and group by that field.'; + +interface Cell { + id: 'sqlite' | 'pg' | 'mysql'; + label: string; + env: string | null; + config: () => Record | null; +} + +const CELLS: readonly Cell[] = [ + { id: 'sqlite', label: 'sqlite', env: null, config: () => ({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) }, + { + id: 'pg', + label: 'live postgres', + env: 'OS_TEST_POSTGRES_URL', + config: () => (process.env.OS_TEST_POSTGRES_URL ? { client: 'pg', connection: process.env.OS_TEST_POSTGRES_URL } : null), + }, + { + id: 'mysql', + label: 'live mysql', + env: 'OS_TEST_MYSQL_URL', + config: () => (process.env.OS_TEST_MYSQL_URL ? { client: 'mysql2', connection: process.env.OS_TEST_MYSQL_URL } : null), + }, +]; + +function createMockServer() { + const noop = () => {}; + return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} }; +} + +function makeRes() { + const res: any = { + write: () => true, end: () => {}, + header: () => res, + status: (code: number) => { res._status = code; return res; }, + json: (body: any) => { res._json = body; return res; }, + }; + return res; +} + +for (const cell of CELLS) { + const config = cell.config(); + describe.skipIf(!config)( + `[#20783] a groupBy on a structured-JSON field at the public door — ${cell.label}${config ? '' : ` (skipped: set ${cell.env} to run this cell)`}`, + () => { + let engine: ObjectQL; + let driver: any; + const reads = { n: 0 }; + let query: (body: Record) => Promise<{ status: number; body: any }>; + + const dropTables = async () => { + if (cell.id === 'sqlite') return; + await driver?.execute(`drop table if exists ${OBJECT}`).catch(() => {}); + }; + + beforeAll(async () => { + driver = new SqlDriver(config as any); + await dropTables(); + engine = new ObjectQL(); + engine.registerDriver(driver, true); + await engine.init(); + engine.registry.registerObject(LEDGER as any); + await engine.syncSchemas(); + for (const row of ROWS) await engine.insert(OBJECT, { ...row } as any); + + // Reads of THIS object — the protocol's own metadata traffic is not the question. + for (const verb of ['find', 'findOne', 'count', 'aggregate'] as const) { + const real = driver[verb].bind(driver); + driver[verb] = (o: string, ...rest: unknown[]) => { if (o === OBJECT) reads.n += 1; return real(o, ...rest); }; + } + + const protocol = new ObjectStackProtocolImplementation(engine as any); + const rest = new RestServer(createMockServer() as any, protocol as any, { api: { requireAuth: false } } as any); + (rest as any).resolveExecCtx = async () => ({ userId: 'test-user' }); + rest.registerRoutes(); + const route = rest.getRoutes().find((r: any) => r.method === 'POST' && r.path === '/api/v1/data/:object/query'); + expect(route).toBeDefined(); + query = async (body) => { + const res = makeRes(); + // What the wire carries: JSON. + await route!.handler({ params: { object: OBJECT }, body: JSON.parse(JSON.stringify(body)), query: {}, headers: {} } as any, res); + return { status: res._status ?? 200, body: res._json }; + }; + }); + + afterAll(async () => { + await dropTables(); + try { await engine?.destroy(); } catch { /* noop */ } + }); + + it('every structured-JSON type answers 400 INVALID_FIELD in the engine\'s words, naming the field, its type and the route — no read', async () => { + const before = reads.n; + for (const [field, type] of JSONS) { + const res = await query({ groupBy: [field], aggregations: COUNT }); + expect(res.status, `${field}: ${JSON.stringify(res.body)}`).toBe(400); + expect(res.body.code, field).toBe('INVALID_FIELD'); + expect(res.body.error, field).toContain(`groupBy[0] names '${field}', a declared ${type} field`); + expect(res.body.error, field).toContain(ROUTE); + const err = await engine.aggregate(OBJECT, { groupBy: [field], aggregations: COUNT } as any).then(() => null, (e: any) => e); + expect({ code: err?.code, status: err?.status }, `engine.aggregate, ${field}`).toEqual({ code: 'INVALID_FIELD', status: 400 }); + } + expect(reads.n - before, 'no read of the object — every refusal precedes the driver').toBe(0); + }); + + it('the { field } object form and a date bucket over a json field answer the same 400 at groupBy[0].field — no read', async () => { + const before = reads.n; + for (const entry of [{ field: 'meta' }, { field: 'meta', dateGranularity: 'month' }]) { + const res = await query({ groupBy: [entry], aggregations: COUNT }); + expect(res.status, `${JSON.stringify(entry)}: ${JSON.stringify(res.body)}`).toBe(400); + expect(res.body.code).toBe('INVALID_FIELD'); + expect(res.body.error).toContain(`groupBy[0].field names 'meta', a declared json field`); + } + const mixed = await query({ groupBy: ['title', 'meta'], aggregations: COUNT }); + expect(mixed.status, JSON.stringify(mixed.body)).toBe(400); + expect(mixed.body.error).toContain(`groupBy[1] names 'meta'`); + expect(reads.n - before, 'no read of the object').toBe(0); + }); + + it('CONTROL a text field\'s groupBy is served unchanged: one group per value, counted, from the driver', async () => { + const before = reads.n; + const res = await query({ groupBy: ['title'], aggregations: COUNT }); + expect(res.status, JSON.stringify(res.body)).toBe(200); + const groups = (res.body.records as Array<{ title: string; n: number | string }>) + .map((r) => [r.title, Number(r.n)] as const) + .sort(([a], [b]) => a.localeCompare(b)); + expect(groups).toEqual([['x', 2], ['y', 1]]); + expect(reads.n - before, 'the driver was asked').toBe(1); + }); + }, + ); +} From 1eacad5296e9b4389add66f453c6399686c6e941 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 06:43:15 +0000 Subject: [PATCH 4/5] chore(changeset): objectql minor for the structured-JSON groupBy refusal Claude-Session: https://claude.ai/code/session_01DEvba2nBuD4tWzfq8r8NFY Co-authored-by: Claude --- .../20783-groupby-structured-json-refused.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .changeset/20783-groupby-structured-json-refused.md diff --git a/.changeset/20783-groupby-structured-json-refused.md b/.changeset/20783-groupby-structured-json-refused.md new file mode 100644 index 00000000000..77c31eee744 --- /dev/null +++ b/.changeset/20783-groupby-structured-json-refused.md @@ -0,0 +1,19 @@ +--- +"@objectstack/objectql": minor +--- + +fix(objectql)!: a `groupBy` on a structured-JSON field is refused with `INVALID_FIELD` / 400 at the engine's `aggregate`, on every driver + +Clause-②: no (narrowing) + + + +**BREAKING**: this narrows what `aggregate` accepts as a grouping target. A `groupBy` entry that names a declared field of the structured-JSON class (`json`, `composite`, `repeater`, `record`, `location`, `address`, `vector`) is refused by the engine before any driver is asked. Both entry spellings are judged, the field name and the `{ field }` object, a `dateGranularity` bucket included. It ships as `minor` under the launch-window convention for accept-set narrowings. No export or published type changes. + +**What an author sees now.** `400 INVALID_FIELD`, naming the position (`groupBy[0]`, or `groupBy[0].field` for the object form), the field and its declared type, saying the query was not run, and naming the route inside the first 500 characters the REST door keeps: group by a field that stores one scalar value, storing the part of the document you group on in a field of its own. The thrown error carries `field`, `fields`, `object` and `param: 'groupBy'`. + +**Why a refusal.** The drivers share no meaning for a JSON document as a group key. Measured through `POST /api/v1/data/:object/query` over three rows with different documents under the grouped field: the in-memory driver answered 200 with one group holding every row, SQLite answered 200 with one group per serialized document, and PostgreSQL answered 500 `DATABASE_ERROR`. A `vector` field split the same three ways, and a date bucket over a `json` field answered one `null` bucket on memory and SQLite and 500 on PostgreSQL. No producer that groups by a structured-JSON field was found (no dataset, cube, view grouping or `groupBy` in the example apps names one), so no meaning is defined for it here. + +**Who is affected.** A caller of `engine.aggregate` or of the REST query door that grouped by such a field on the in-memory driver or on SQLite and read the merged or per-serialization groups as real ones. On PostgreSQL the same query was already a 500. The analytics service's aggregate path (a cube query the native-SQL strategy declines, such as a time dimension with a granularity, or any cube query on the in-memory driver) reaches the engine and answers this refusal too. + +**Unchanged.** A `groupBy` on any other type (`text`, `number`, a `multiple: true` select, a file field), a structured-JSON field as an AGGREGATED column (`count`, `count_distinct`, `min`, `max`), and an undeclared name, which the REST door answers `INVALID_FIELD` as unknown before the engine is reached. From cafaf885d8100ef9617431e4960ac84bf3ff872e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 07:04:28 +0000 Subject: [PATCH 5/5] test(rest): type the aggregate options in the groupBy door pin instead of erasing them Claude-Session: https://claude.ai/code/session_01DEvba2nBuD4tWzfq8r8NFY Co-authored-by: Claude --- packages/rest/src/data-group-by-json-door.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/rest/src/data-group-by-json-door.test.ts b/packages/rest/src/data-group-by-json-door.test.ts index 46185cb8cc3..d97e6b141da 100644 --- a/packages/rest/src/data-group-by-json-door.test.ts +++ b/packages/rest/src/data-group-by-json-door.test.ts @@ -35,6 +35,7 @@ */ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import type { EngineAggregateOptions } from '@objectstack/spec/data'; import { ObjectQL } from '@objectstack/objectql'; import { SqlDriver } from '@objectstack/driver-sql'; import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; @@ -68,7 +69,7 @@ const ROWS = [ { id: 'd3', title: 'y', meta: { b: 1 }, spec: { k: 1 }, rep: [{ q: 1 }], rec: { r: 1 }, loc: { lat: 1, lng: 2 }, ship_to: { city: 'Paris' }, vec: [1, 2] }, ]; -const COUNT = [{ function: 'count', alias: 'n' }]; +const COUNT: EngineAggregateOptions['aggregations'] = [{ function: 'count', alias: 'n' }]; /** The route the refusal names — asserted on the REST body, so it must land inside the door's 500-character bound. */ const ROUTE = 'Group by a field that stores one scalar value: store the part you group on in a field of its own and group by that field.'; @@ -169,7 +170,7 @@ for (const cell of CELLS) { expect(res.body.code, field).toBe('INVALID_FIELD'); expect(res.body.error, field).toContain(`groupBy[0] names '${field}', a declared ${type} field`); expect(res.body.error, field).toContain(ROUTE); - const err = await engine.aggregate(OBJECT, { groupBy: [field], aggregations: COUNT } as any).then(() => null, (e: any) => e); + const err = await engine.aggregate(OBJECT, { groupBy: [field], aggregations: COUNT }).then(() => null, (e: any) => e); expect({ code: err?.code, status: err?.status }, `engine.aggregate, ${field}`).toEqual({ code: 'INVALID_FIELD', status: 400 }); } expect(reads.n - before, 'no read of the object — every refusal precedes the driver').toBe(0);