From 24706c959fd73b5c672f01fc6a5c52e0c8ea0477 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 00:53:00 +0000 Subject: [PATCH 1/3] fix(objectql): rows-path sum / avg add with Kahan-Babuska-Neumaier compensation, as SQLite does The engine's rows path (in-memory-aggregation.ts) folded sum and avg naively, while SQLite 3.43+ compensates, so one query answered two doubles on SQLite depending on the path engine.aggregate took (0.1 + 0.2 + 0.3: native 0.6, rows 0.6000000000000001). Both arms now add through one compensated fold transcribed from SQLite's kahanBabuskaNeumaierStep and its finalizers' overflow guard. Claude-Session: https://claude.ai/code/session_01N8TPEsoJxPsdSdNKGnNGEN Co-authored-by: Claude --- ...memory-aggregation-compensated-sum.test.ts | 148 ++++++++++++++ .../objectql/src/in-memory-aggregation.ts | 43 +++- .../rest-aggregate-compensated-sum.test.ts | 183 ++++++++++++++++++ 3 files changed, 372 insertions(+), 2 deletions(-) create mode 100644 packages/objectql/src/in-memory-aggregation-compensated-sum.test.ts create mode 100644 packages/rest/src/rest-aggregate-compensated-sum.test.ts diff --git a/packages/objectql/src/in-memory-aggregation-compensated-sum.test.ts b/packages/objectql/src/in-memory-aggregation-compensated-sum.test.ts new file mode 100644 index 00000000000..977d01b7f23 --- /dev/null +++ b/packages/objectql/src/in-memory-aggregation-compensated-sum.test.ts @@ -0,0 +1,148 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20489] The rows path's `sum` / `avg` add with Kahan-Babuska-Neumaier + * compensation (`in-memory-aggregation.ts`, `compensatedSum`), the summation + * SQLite 3.43+ uses for its own `sum` / `avg`. + * + * The expected values below are SQLite's own answers, measured with + * better-sqlite3 (SQLite 3.53.4), sql.js (3.49.1) and @libsql/client (3.45.1) + * over a `REAL` and a `NUMERIC` column holding the same values. All three + * engines agreed on every fixture. The naive fold the rows path used before is + * shown beside each one: + * + * | fixture | naive `sum` / `avg` (before) | SQLite native = compensated (after) | + * |:--|:--|:--| + * | `0.1, 0.2, 0.3` | `0.6000000000000001` / `0.20000000000000004` | `0.6` / `0.19999999999999998` | + * | `1e16, 1, -1e16` | `0` / `0` | `1` / `0.3333333333333333` | + * | `1e16, 0.5, -1e16` | `0` / `0` | `0.5` / `0.16666666666666666` | + * | `0.1, 0.2` (two addends) | `0.30000000000000004` / `0.15000000000000002` | the same | + * | `1, 2, 3, 40, 500` (integers) | `546` / `109.2` | the same | + * + * `engine.aggregate` and REST on SQLite are pinned in `packages/rest` + * (`rest-aggregate-compensated-sum.test.ts`): the rows path and the native + * path answer the same double there. + */ + +import { describe, it, expect } from 'vitest'; +import { applyInMemoryAggregation } from './in-memory-aggregation.js'; + +/** The fold the rows path used before #20489: in order, one addition at a time. */ +const naiveSum = (xs: readonly number[]) => xs.reduce((a, b) => a + b, 0); + +/** `sum` and `avg` of `w` over `values`, one group, through the rows path. */ +function sumAvg(values: readonly unknown[]): { s: unknown; a: unknown } { + const rows = values.map((w, i) => ({ id: `r${i}`, w })); + const [out] = applyInMemoryAggregation(rows, { + aggregations: [ + { function: 'sum', field: 'w', alias: 's' }, + { function: 'avg', field: 'w', alias: 'a' }, + ], + }); + return out as { s: unknown; a: unknown }; +} + +describe('[#20489] rows path — sum / avg add with compensation, as SQLite does', () => { + it("the card's fixture: 0.1 + 0.2 + 0.3 answers SQLite's 0.6, and avg its 0.19999999999999998", () => { + const values = [0.1, 0.2, 0.3]; + // The fixture discriminates: the naive fold answers another double. + expect(naiveSum(values)).toBe(0.6000000000000001); + expect(sumAvg(values)).toStrictEqual({ s: 0.6, a: 0.19999999999999998 }); + // So `having { s: { $eq: 0.6 } }` now keeps the group on this path too. + expect(sumAvg(values).s === 0.6).toBe(true); + }); + + it('a mixed-sign set with large cancellation keeps the small addend', () => { + expect(naiveSum([1e16, 1, -1e16])).toBe(0); + expect(sumAvg([1e16, 1, -1e16])).toStrictEqual({ s: 1, a: 1 / 3 }); + expect(naiveSum([1e16, 0.5, -1e16])).toBe(0); + expect(sumAvg([1e16, 0.5, -1e16])).toStrictEqual({ s: 0.5, a: 0.5 / 3 }); + }); + + it('two addends are unchanged: the compensated a + b is the naive one', () => { + for (const pair of [[0.1, 0.2], [0.7, 0.1], [1e16, 1], [-0.3, 0.1]]) { + const n = naiveSum(pair); + expect(sumAvg(pair), `${pair}`).toStrictEqual({ s: n, a: n / 2 }); + } + expect(sumAvg([0.1, 0.2])).toStrictEqual({ s: 0.30000000000000004, a: 0.15000000000000002 }); + }); + + it('integers whose partial sums stay within 2^53 are unchanged, and stay integers', () => { + const values = [1, 2, 3, 40, 500]; + const { s, a } = sumAvg(values); + expect(s).toBe(naiveSum(values)); + expect(s).toBe(546); + expect(Number.isInteger(s)).toBe(true); + expect(a).toBe(109.2); + expect(sumAvg([-7, 3, 12, 0, 9_000_000_000])).toStrictEqual({ s: 9_000_000_008, a: 9_000_000_008 / 5 }); + }); + + it('above 2^53 the compensated total is the exact one SQLite answers, where the naive fold lost the 1s', () => { + // SQLite 3.53.4 answers 9007199254740994 over a REAL and over a NUMERIC column. + expect(naiveSum([2 ** 53, 1, 1])).toBe(2 ** 53); + expect(sumAvg([2 ** 53, 1, 1])).toStrictEqual({ s: 9007199254740994, a: 3002399751580331.5 }); + }); + + it('under groupBy and a per-aggregation filter, every bucket takes the same fold', () => { + const rows = [ + { g: 'x', k: 'in', w: 0.1 }, + { g: 'x', k: 'in', w: 0.2 }, + { g: 'x', k: 'out', w: 100 }, + { g: 'x', k: 'in', w: 0.3 }, + { g: 'y', k: 'in', w: 1e16 }, + { g: 'y', k: 'in', w: 1 }, + { g: 'y', k: 'in', w: -1e16 }, + ]; + const out = applyInMemoryAggregation(rows, { + groupBy: ['g'], + aggregations: [ + { function: 'sum', field: 'w', alias: 's', filter: { k: 'in' } }, + { function: 'avg', field: 'w', alias: 'a', filter: { k: 'in' } }, + { function: 'sum', field: 'w', alias: 'all' }, + ], + } as any).sort((p, q) => String(p.g).localeCompare(String(q.g))); + expect(out).toStrictEqual([ + { g: 'x', s: 0.6, a: 0.19999999999999998, all: 100.6 }, + { g: 'y', s: 1, a: 1 / 3, all: 1 }, + ]); + }); +}); + +describe('[#20489] rows path — what the compensated fold leaves as it was', () => { + it('null: sum adds nothing for it, avg leaves it out of the count', () => { + expect(sumAvg([0.1, null, 0.2, undefined, 0.3])).toStrictEqual({ s: 0.6, a: 0.19999999999999998 }); + expect(sumAvg([10, null, 20])).toStrictEqual({ s: 30, a: 15 }); + }); + + it('a non-numeric cell reads as 0 in both, and counts in avg; a numeric string reads as its number', () => { + expect(sumAvg([10, 'abc', 20])).toStrictEqual({ s: 30, a: 10 }); + expect(sumAvg(['0.1', '0.2', '0.3'])).toStrictEqual({ s: 0.6, a: 0.19999999999999998 }); + expect(sumAvg([true, false, true])).toStrictEqual({ s: 2, a: 2 / 3 }); + }); + + it('an empty group: sum 0, avg null — with no rows, and with only nulls', () => { + expect(sumAvg([])).toStrictEqual({ s: 0, a: null }); + expect(sumAvg([null, undefined])).toStrictEqual({ s: 0, a: null }); + }); + + it('a non-finite total is the naive one, as SQLite returns its running sum when the error term overflows', () => { + for (const values of [ + [Infinity, 1, 2], + [1, -Infinity, 0.3], + [1e308, 1e308, -1e308], + [Infinity, -Infinity, 1], + [NaN, 0.1, 0.2], + ]) { + const n = naiveSum(values); + const { s, a } = sumAvg(values); + expect(Object.is(s, n), `sum of ${values}: ${s} vs naive ${n}`).toBe(true); + expect(Object.is(a, n / values.length), `avg of ${values}`).toBe(true); + } + }); + + it('the answer is a JS number on every arm, never a string', () => { + const { s, a } = sumAvg(['0.1', 0.2, '0.3']); + expect(typeof s).toBe('number'); + expect(typeof a).toBe('number'); + }); +}); diff --git a/packages/objectql/src/in-memory-aggregation.ts b/packages/objectql/src/in-memory-aggregation.ts index b1ebc47def0..ff8d2385a69 100644 --- a/packages/objectql/src/in-memory-aggregation.ts +++ b/packages/objectql/src/in-memory-aggregation.ts @@ -228,12 +228,15 @@ function aggregateBucket( case 'count_distinct': out[alias] = new Set(values.filter((v) => v != null)).size; break; + // [#20489] Both arms add through ONE compensated fold + // ({@link compensatedSum}) — the summation SQLite's own `sum` / `avg` + // use — so the rows path and SQLite's native path answer the same double. case 'sum': - out[alias] = values.reduce((a, b) => a + toNumber(b), 0); + out[alias] = compensatedSum(values.map(toNumber)); break; case 'avg': { const nums = values.filter((v) => v != null).map(toNumber); - out[alias] = nums.length === 0 ? null : nums.reduce((a, b) => a + b, 0) / nums.length; + out[alias] = nums.length === 0 ? null : compensatedSum(nums) / nums.length; break; } // [#11152] `min`/`max` read a BOOLEAN as the number it is worth (0/1) — @@ -284,6 +287,42 @@ function toNumber(v: any): number { return Number.isFinite(n) ? n : 0; } +/** + * [#20489] The sum of `nums`, added in order with Kahan-Babuska-Neumaier + * compensation — the summation SQLite (3.43 and later) uses for its own `sum` + * and `avg`, transcribed from its `kahanBabuskaNeumaierStep` and the + * finalizers' overflow guard. + * + * Why: this fold used to add naively (`reduce((a, b) => a + b, 0)`), so one + * query answered two doubles on SQLite depending on the path `engine.aggregate` + * took. A `number` column holding `0.1`, `0.2` and `0.3` summed to `0.6` + * natively and to `0.6000000000000001` here (`avg` `0.19999999999999998` + * against `0.20000000000000004`), and `having { s: { $eq: 0.6 } }` kept the + * group on the native path alone. Compensated, the two paths agree, and the + * answer is the more accurate one (`1e16 + 1 - 1e16` is `1`, not `0`). + * + * What does not move: two addends (the compensated `a + b` IS the naive one), + * integers whose partial sums stay within 2^53 (every addition is exact), and + * a non-finite total. `s` below is exactly the naive running sum; once it + * overflows or meets a NaN, the error term is non-finite and the naive answer + * is returned as it was, which is SQLite's rule too. + * + * ⚠️ Residual, stated: PostgreSQL and MySQL add their doubles natively without + * compensation, so over three or more fractions their native path can still + * differ from this one in the last place. An exact `$eq` on a fractional sum + * compares doubles; compare with a range. + */ +function compensatedSum(nums: readonly number[]): number { + let s = 0; + let c = 0; + for (const r of nums) { + const t = s + r; + c += Math.abs(s) > Math.abs(r) ? (s - t) + r : (r - t) + s; + s = t; + } + return Number.isFinite(c) ? s + c : s; +} + /** * Bucket a date-like value into an ISO-formatted period label. Weeks start * Monday and use ISO week numbering. diff --git a/packages/rest/src/rest-aggregate-compensated-sum.test.ts b/packages/rest/src/rest-aggregate-compensated-sum.test.ts new file mode 100644 index 00000000000..eb34a866cbf --- /dev/null +++ b/packages/rest/src/rest-aggregate-compensated-sum.test.ts @@ -0,0 +1,183 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20489] On SQLite, `sum` / `avg` answer ONE double on both of + * `engine.aggregate`'s paths — the native `SqlDriver.aggregate` and the rows + * path (`objectql`'s `in-memory-aggregation.ts`, which a filtered sibling + * aggregation forces) — through `engine.aggregate` and + * `POST /api/v1/data/:object/query`, over a real `SqlDriver`. + * + * SQLite 3.43+ sums with Kahan-Babuska-Neumaier compensation; the rows path + * used to add naively. Measured on the base (`b2b6a0643`), SQLite 3.53.4, a + * `number` column, same readings through the engine and REST: + * + * | group | native `sum` / `avg` | rows path `sum` / `avg` (base) | + * |:--|:--|:--| + * | `0.1, 0.2, 0.3` | `0.6` / `0.19999999999999998` | `0.6000000000000001` / `0.20000000000000004` | + * | `1e16, 1, -1e16` | `1` / `0.3333333333333333` | `0` / `0` | + * | `1e16, 0.5, -1e16` | `0.5` / `0.16666666666666666` | `0` / `0` | + * | `0.1, 0.2` | `0.30000000000000004` / `0.15000000000000002` | the same | + * | `1, 2, 3, 40, 500` | `546` / `109.2` | the same | + * + * `having { s: { $eq: 0.6 } }` kept the first group on the native path and no + * group on the rows path. The rows path now adds with the same compensation, + * so every row of that table reads as its native column on both paths. + * + * ## The dialect axis of THIS file + * + * SQLite only, and deliberately. PostgreSQL and MySQL add their doubles + * natively without compensation (`sql-driver.ts`, `AGGREGATE_ACCUMULATION`), so + * over three or more fractions their native path can differ from the rows path + * in the last place: that is the residual #20489 states, not a defect a pin + * here should hold red. An exact `$eq` on a fractional sum compares doubles. + */ + +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import type { EngineAggregateOptions, FilterCondition } from '@objectstack/spec/data'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { RestServer } from './rest-server'; + +const OBJECT = 'rest_agg_20489'; + +const LEDGER = { + name: OBJECT, + label: 'Ledger 20489', + fields: { + g: { name: 'g', type: 'text' as const }, + w: { name: 'w', type: 'number' as const }, + }, +}; + +/** group → its values, and SQLite's native `sum` / `avg` over them. */ +const GROUPS: Record = { + card: { values: [0.1, 0.2, 0.3], s: 0.6, a: 0.19999999999999998 }, + cancel: { values: [1e16, 1, -1e16], s: 1, a: 0.3333333333333333 }, + cancel_half: { values: [1e16, 0.5, -1e16], s: 0.5, a: 0.16666666666666666 }, + two: { values: [0.1, 0.2], s: 0.30000000000000004, a: 0.15000000000000002 }, + ints: { values: [1, 2, 3, 40, 500], s: 546, a: 109.2 }, +}; + +function createMockServer() { + const noop = () => {}; + return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} }; +} + +function makeRes() { + const res: any = { + write: () => true, end: () => {}, + header: () => res, + status: (code: number) => { res._status = code; return res; }, + json: (body: any) => { res._json = body; return res; }, + }; + return res; +} + +type Path = 'native' | 'rows'; + +/** + * `native`: SqlDriver aggregates and the engine applies `having` to its + * answer. `rows`: a filtered aggregation sends the engine to the rows path, + * where it aggregates `find()` rows itself and then applies `having`. + */ +function grouped(path: Path, having?: Record): EngineAggregateOptions { + const aggregations: NonNullable = [ + { function: 'sum', field: 'w', alias: 's' }, + { function: 'avg', field: 'w', alias: 'a' }, + ]; + if (path === 'rows') aggregations.push({ function: 'count', alias: 'fb', filter: { g: { $ne: '' } } }); + return { groupBy: ['g'], aggregations, ...(having ? { having: having as FilterCondition } : {}) }; +} + +const byGroup = (rows: any[]) => Object.fromEntries(rows.map((r) => [r.g, { s: r.s, a: r.a }])); +const groupsOf = (rows: any[]) => rows.map((r) => r.g).sort(); + +describe('[#20489] sum / avg — one double on both SQLite paths, engine and REST', () => { + let engine: ObjectQL; + let driver: SqlDriver; + let post: (body: Record) => Promise; + + beforeAll(async () => { + driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true } as never); + engine = new ObjectQL(); + engine.registerDriver(driver, true); + await engine.init(); + engine.registry.registerObject(LEDGER as any); + await engine.syncSchemas(); + let i = 0; + for (const [g, { values }] of Object.entries(GROUPS)) { + for (const w of values) await engine.insert(OBJECT, { id: `r${i++}`, g, w } as any); + } + + const protocol = new ObjectStackProtocolImplementation(engine as any); + const rest = new RestServer(createMockServer() as any, protocol as any, { api: { requireAuth: false } } as any); + (rest as any).resolveExecCtx = async () => ({ userId: 'test-user' }); + rest.registerRoutes(); + const route = rest.getRoutes().find((r: any) => r.method === 'POST' && r.path === '/api/v1/data/:object/query'); + expect(route).toBeDefined(); + post = async (body) => { + const res = makeRes(); + // What the wire carries: JSON, both ways. + await route!.handler({ params: { object: OBJECT }, body: JSON.parse(JSON.stringify(body)) } as any, res); + if (res._json !== undefined) res._json = JSON.parse(JSON.stringify(res._json)); + return res; + }; + }); + + afterAll(async () => { + try { await engine?.destroy(); } catch { /* noop */ } + }); + + it('each path is the one it names: native asks driver.aggregate, rows asks driver.find alone', async () => { + const spy = vi.spyOn(driver, 'aggregate'); + try { + await engine.aggregate(OBJECT, grouped('native')); + expect(spy, 'native').toHaveBeenCalledTimes(1); + spy.mockClear(); + await engine.aggregate(OBJECT, grouped('rows')); + expect(spy, 'rows').not.toHaveBeenCalled(); + } finally { + spy.mockRestore(); + } + }); + + it("find() reads back the doubles written, so both paths add the same operands", async () => { + const rows = (await engine.find(OBJECT, {})) as Array<{ g: string; w: number }>; + for (const [g, { values }] of Object.entries(GROUPS)) { + expect(rows.filter((r) => r.g === g).map((r) => r.w).sort((x, y) => x - y), g) + .toStrictEqual([...values].sort((x, y) => x - y)); + } + }); + + it('sum / avg: SQLite native answers, equal on both paths, through the engine and REST', async () => { + const expected = Object.fromEntries(Object.entries(GROUPS).map(([g, { s, a }]) => [g, { s, a }])); + for (const path of ['native', 'rows'] as const) { + expect(byGroup(await engine.aggregate(OBJECT, grouped(path))), `engine, ${path}`).toStrictEqual(expected); + const res = await post(grouped(path) as Record); + expect(res._status ?? 200, JSON.stringify(res._json)).toBe(200); + expect(byGroup(res._json.records), `REST, ${path}`).toStrictEqual(expected); + } + }); + + it('having $eq on the fractional sum / avg keeps the same group on both paths', async () => { + const KEPT: ReadonlyArray, string[]]> = [ + [{ s: { $eq: 0.6 } }, ['card']], + [{ a: { $eq: 0.19999999999999998 } }, ['card']], + [{ s: { $eq: 1 } }, ['cancel']], + [{ s: { $in: [0.5, 546] } }, ['cancel_half', 'ints']], + // The residual, as a double: `0.1 + 0.2` is not `0.3` on any path. + [{ s: { $eq: 0.3 } }, []], + [{ s: { $eq: 0.30000000000000004 } }, ['two']], + ]; + for (const [having, kept] of KEPT) { + for (const path of ['native', 'rows'] as const) { + expect(groupsOf(await engine.aggregate(OBJECT, grouped(path, having))), `engine, ${path}, ${JSON.stringify(having)}`) + .toEqual(kept); + const res = await post(grouped(path, having) as Record); + expect(res._status ?? 200, JSON.stringify(res._json)).toBe(200); + expect(groupsOf(res._json.records), `REST, ${path}, ${JSON.stringify(having)}`).toEqual(kept); + } + } + }); +}); From db9d272301694c488378fdc576b2d7a10ea9b5d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 00:58:48 +0000 Subject: [PATCH 2/3] docs(objectql): changeset for the compensated rows-path fold, and the residual it leaves Claude-Session: https://claude.ai/code/session_01N8TPEsoJxPsdSdNKGnNGEN Co-authored-by: Claude --- .changeset/20489-rows-path-compensated-sum.md | 40 +++++++++++++++++++ .../objectql/src/in-memory-aggregation.ts | 6 ++- 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 .changeset/20489-rows-path-compensated-sum.md diff --git a/.changeset/20489-rows-path-compensated-sum.md b/.changeset/20489-rows-path-compensated-sum.md new file mode 100644 index 00000000000..a05c0983b23 --- /dev/null +++ b/.changeset/20489-rows-path-compensated-sum.md @@ -0,0 +1,40 @@ +--- +'@objectstack/objectql': patch +--- + +fix(objectql): the engine's rows path adds `sum` / `avg` with compensated summation, as SQLite does + +Clause-②: no + +`engine.aggregate` answers a `sum` / `avg` on one of two paths: the driver's own aggregate, or +the rows path (`applyInMemoryAggregation`), which aggregates `find()` rows in JavaScript and is +taken for a per-aggregation `filter`, a non-UTC date bucket, or a driver without native +aggregation. SQLite 3.43 and later adds with Kahan-Babuska-Neumaier compensation; the rows path +added naively. So on SQLite one query answered two doubles depending on the path. A `number` +column holding `0.1`, `0.2` and `0.3` in one group: + +| | `sum` | `avg` | `having { s: { $eq: 0.6 } }` | +|:--|:--|:--|:--| +| SQLite native | `0.6` | `0.19999999999999998` | keeps the group | +| rows path, before | `0.6000000000000001` | `0.20000000000000004` | keeps no group | +| rows path, after | `0.6` | `0.19999999999999998` | keeps the group | + +The rows path now adds with the same compensation, transcribed from SQLite's own, so on SQLite +both paths answer the same double, through `engine.aggregate` and +`POST /api/v1/data/:object/query` alike. It is also the more accurate sum: `1e16 + 1 - 1e16` is +`1`, where the naive fold answered `0`. + +Unchanged: two addends (the compensated `a + b` is the naive one, so `0.1 + 0.2` is still +`0.30000000000000004`), integers whose running total stays within 2^53, a non-finite total, +`null` and non-numeric cells, and the empty group (`sum` `0`, `avg` `null`). The answer is still a +JS number. + +**Residual, stated.** PostgreSQL and MySQL add `sum` / `avg` natively in double without +compensation, and that arithmetic is the database's own. So over three or more fractions their +native path can still differ from the rows path in the last place (`0.1 + 0.2 + 0.3`: native +`0.6000000000000001`, rows path `0.6`). This replaces the residual the `@objectstack/driver-sql` +entry for the double accumulation states: that difference is no longer SQLite's native path +against every other face; it is now PostgreSQL / MySQL native against the rows path. The +in-memory driver (`@objectstack/driver-memory`) still adds naively in its own `aggregate`, so on +that driver the two paths can now differ in the same last place. An exact `$eq` on a fractional +sum compares doubles: compare with a range. diff --git a/packages/objectql/src/in-memory-aggregation.ts b/packages/objectql/src/in-memory-aggregation.ts index ff8d2385a69..4fc9005fc39 100644 --- a/packages/objectql/src/in-memory-aggregation.ts +++ b/packages/objectql/src/in-memory-aggregation.ts @@ -309,8 +309,10 @@ function toNumber(v: any): number { * * ⚠️ Residual, stated: PostgreSQL and MySQL add their doubles natively without * compensation, so over three or more fractions their native path can still - * differ from this one in the last place. An exact `$eq` on a fractional sum - * compares doubles; compare with a range. + * differ from this one in the last place. So can the folds that do not call + * this one: `driver-memory`'s `aggregate` and analytics faces, and + * `service-analytics`' draft preview (`preview-evaluator.ts`). An exact `$eq` + * on a fractional sum compares doubles; compare with a range. */ function compensatedSum(nums: readonly number[]): number { let s = 0; From 1c476788382b5de4e5aec4c5cebb71fe14452f53 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 01:41:20 +0000 Subject: [PATCH 3/3] docs: state the moved sum / avg residual in the pending driver-sql note and its comments The rows path now adds with compensated summation, so the residual the pending driver-sql double-accumulation note, the AGGREGATE_ACCUMULATION comment and the driver-sql test header stated ("SQLite's native face alone", "every other face 0.6000000000000001") no longer held. Each now states it as PostgreSQL / MySQL native against SQLite and the rows path. Text only; no logic moves. Claude-Session: https://claude.ai/code/session_01N8TPEsoJxPsdSdNKGnNGEN Co-authored-by: Claude --- .changeset/20387-aggregate-one-double.md | 11 ++++++----- .changeset/20489-rows-path-compensated-sum.md | 6 +++--- ...river-20387-aggregate-double-accumulation.test.ts | 10 ++++++---- packages/drivers/driver-sql/src/sql-driver.ts | 12 +++++++----- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.changeset/20387-aggregate-one-double.md b/.changeset/20387-aggregate-one-double.md index 7194e30ee2e..691961c7a1a 100644 --- a/.changeset/20387-aggregate-one-double.md +++ b/.changeset/20387-aggregate-one-double.md @@ -39,11 +39,12 @@ the rows path add exact decimals and round once. It was rejected because SQLite' adds the stored doubles (`0.30000000000000004`), so the rows path would then disagree with SQLite for exactly `0.1 + 0.2`. -**Residual, stated.** Double sums are added in row order, one after another, as the rows path -adds them. SQLite 3.43 and later adds with compensated summation. So for a group of three or -more fractions, SQLite's native answer can still differ from every other face in the last -place (`0.1 + 0.2 + 0.3`: SQLite `0.6`, every other face `0.6000000000000001`). This was -already true of SQLite's two paths before this change. Two addends cannot differ. +**Residual, stated.** On PostgreSQL and MySQL the double sums are added in row order, one after +another, without compensation. SQLite 3.43 and later adds with compensated summation, and since +#20489 so does the engine's rows path. So for a group of three or more fractions, the PostgreSQL +and MySQL native answer can still differ from SQLite's and the rows path's in the last place +(`0.1 + 0.2 + 0.3`: PostgreSQL / MySQL native `0.6000000000000001`, SQLite and the rows path +`0.6`). Before #20489, SQLite's own two paths differed there too. Two addends cannot differ. A consumer that compared `sum` / `avg` over a fractional column with a decimal literal on PostgreSQL or MySQL (`$eq: 0.3`) now gets the answer SQLite and the rows path already gave: diff --git a/.changeset/20489-rows-path-compensated-sum.md b/.changeset/20489-rows-path-compensated-sum.md index a05c0983b23..dd0d38b5422 100644 --- a/.changeset/20489-rows-path-compensated-sum.md +++ b/.changeset/20489-rows-path-compensated-sum.md @@ -32,9 +32,9 @@ JS number. **Residual, stated.** PostgreSQL and MySQL add `sum` / `avg` natively in double without compensation, and that arithmetic is the database's own. So over three or more fractions their native path can still differ from the rows path in the last place (`0.1 + 0.2 + 0.3`: native -`0.6000000000000001`, rows path `0.6`). This replaces the residual the `@objectstack/driver-sql` -entry for the double accumulation states: that difference is no longer SQLite's native path -against every other face; it is now PostgreSQL / MySQL native against the rows path. The +`0.6000000000000001`, rows path `0.6`). The `@objectstack/driver-sql` entry for the double +accumulation states the same residual: the difference is no longer SQLite's native path against +every other face; it is PostgreSQL / MySQL native against SQLite and the rows path. The in-memory driver (`@objectstack/driver-memory`) still adds naively in its own `aggregate`, so on that driver the two paths can now differ in the same last place. An exact `$eq` on a fractional sum compares doubles: compare with a range. diff --git a/packages/drivers/driver-sql/src/sql-driver-20387-aggregate-double-accumulation.test.ts b/packages/drivers/driver-sql/src/sql-driver-20387-aggregate-double-accumulation.test.ts index 53e0b007f92..9883135dd23 100644 --- a/packages/drivers/driver-sql/src/sql-driver-20387-aggregate-double-accumulation.test.ts +++ b/packages/drivers/driver-sql/src/sql-driver-20387-aggregate-double-accumulation.test.ts @@ -25,9 +25,11 @@ * `having` is evaluated by the engine, over the value this door answers * (`having-filter.ts`: `$eq` compares the value with `==`), so the value pinned * here is what decides `having` `$eq` / `$in` on every face. The expected - * values are computed from `find()`'s own rows with the rows path's arithmetic + * values are computed from `find()`'s own rows, added in row order * ({@link rowsPathSum}), plus the literal the triage named, so a decimal - * answer, a compensated sum or a string each fail. + * answer or a string each fail. Over these fixtures — two addends, and + * integer-valued columns within 2^53 — that sum is also the rows path's, which + * adds with compensation since #20489: the two folds cannot differ there. * * What stays as it was, pinned too: `count` / `min` / `max`, and a `sum` over * an integer-valued column, which keeps the database's exact total (above 2^53 @@ -49,10 +51,10 @@ function toNumber(v: unknown): number { return Number.isFinite(n) ? n : 0; } -/** The rows path's `sum`: `values.reduce((a, b) => a + toNumber(b), 0)`, in row order. */ +/** `values.reduce((a, b) => a + toNumber(b), 0)`, in row order: the rows path's `sum` over this file's fixtures (see the header). */ const rowsPathSum = (values: readonly unknown[]) => values.reduce((a, b) => a + toNumber(b), 0); -/** The rows path's `avg`: the non-null values' `sum`, divided by how many there are. */ +/** The non-null values' {@link rowsPathSum}, divided by how many there are: the rows path's `avg` over this file's fixtures. */ function rowsPathAvg(values: readonly unknown[]): number | null { const defined = values.filter((v) => v != null); return defined.length === 0 ? null : rowsPathSum(defined) / defined.length; diff --git a/packages/drivers/driver-sql/src/sql-driver.ts b/packages/drivers/driver-sql/src/sql-driver.ts index 6c8223ab802..51193494fd7 100644 --- a/packages/drivers/driver-sql/src/sql-driver.ts +++ b/packages/drivers/driver-sql/src/sql-driver.ts @@ -1594,11 +1594,13 @@ const AGGREGATE_ANSWER_KIND: Readonly