From 7bd081f94334ba2e0cb2813a294b889d0b84ba23 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:41:18 -0500 Subject: [PATCH 1/2] fix: correct overview costs, SLO coverage, and freshness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use producer-normalized total throughput once, minimize cost across eligible configurations, and retain measured endpoints that meet the minimum SLO. Separate measurement, run-attempt, and curve dates; expire client cache entries and guard refresh races. Correct physical-chip ingestion and add a conservative, dry-run-first topology repair command. Align dependent run/rankings pages. 中文:修复概览成本计算、SLO 数据覆盖与刷新问题。 直接使用生产端已按全部物理芯片归一化的总吞吐量,从满足最低 SLO 的有效配置中 选择最低成本。区分实测、运行与曲线快照日期,修复客户端缓存过期和刷新竞态。 校正摄取流程中的物理芯片数,并提供默认只读、保守判定的历史数据修复工具; 同步更新使用相同计算逻辑的运行详情页和排行榜。 --- docs/architecture.md | 3 +- docs/data-pipeline.md | 3 + docs/index.md | 1 + docs/overview-data-integrity.md | 114 +++++++++++++++ packages/app/cypress/e2e/overview.cy.ts | 131 +++++++++-------- .../src/app/api/unofficial-run/route.test.ts | 14 ++ packages/app/src/app/rankings/[slug]/page.tsx | 14 +- packages/app/src/app/run/[slug]/page.tsx | 16 +- .../app/src/app/zh/rankings/[slug]/page.tsx | 14 +- packages/app/src/app/zh/run/[slug]/page.tsx | 16 +- .../overview/overview-navigation.test.tsx | 107 ++++++++++++++ .../overview/overview-navigation.tsx | 70 ++++++++- .../overview/overview-presentation.test.tsx | 4 +- .../overview/overview-scorecard.tsx | 23 ++- .../components/overview/overview-strings.ts | 26 +++- packages/app/src/lib/api-documentation.ts | 39 ++++- packages/app/src/lib/api-route-catalog.ts | 9 +- packages/app/src/lib/overview-data.server.ts | 7 +- packages/app/src/lib/overview-data.test.ts | 130 ++++++++++++----- packages/app/src/lib/overview-data.ts | 92 +++++++----- .../app/src/lib/run-rankings-data.server.ts | 4 +- packages/constants/src/metric-keys.ts | 3 + packages/db/package.json | 3 +- .../src/backfill-benchmark-topology.test.ts | 137 ++++++++++++++++++ .../db/src/backfill-benchmark-topology.ts | 89 ++++++++++++ .../backfill-full-response-interactivity.ts | 14 +- packages/db/src/etl/benchmark-ingest.ts | 11 +- packages/db/src/etl/benchmark-mapper.test.ts | 24 ++- packages/db/src/etl/benchmark-mapper.ts | 32 +++- .../etl/full-response-interactivity.test.ts | 41 ++++++ .../db/src/etl/full-response-interactivity.ts | 29 +++- .../db/src/etl/trace-replay-ingest.test.ts | 8 +- packages/db/src/etl/trace-replay-ingest.ts | 15 +- .../db/src/lib/benchmark-topology.test.ts | 124 ++++++++++++++++ packages/db/src/lib/benchmark-topology.ts | 94 ++++++++++++ 35 files changed, 1246 insertions(+), 215 deletions(-) create mode 100644 docs/overview-data-integrity.md create mode 100644 packages/db/src/backfill-benchmark-topology.test.ts create mode 100644 packages/db/src/backfill-benchmark-topology.ts create mode 100644 packages/db/src/lib/benchmark-topology.test.ts create mode 100644 packages/db/src/lib/benchmark-topology.ts diff --git a/docs/architecture.md b/docs/architecture.md index 0b34558f1..3db0c0303 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,10 +18,11 @@ Some routes serve a purpose that raw DB rows cannot satisfy: - `/api/v1/overview` is a page-owned backend-for-frontend (BFF). It returns the compact `OverviewPageData` shape used by the initial server render, allowing selector changes to update the matrix without downloading every model's raw benchmark history or triggering a React Server Component (RSC) round trip. It is not a reusable public data API. - `/api/v1/benchmarks?view=calculator&sequence=...` is a page-owned compact view. It returns only the selected calculator scenario and the metric keys needed for interpolation, while the default endpoint preserves the raw-row contract. This reduces transfer and browser parsing without narrowing the reusable inference response. -Two rules keep that BFF honest, both enforced in `overview-navigation.tsx`: +Three rules keep that BFF honest, all enforced in `overview-navigation.tsx`: - **One cache key per data state.** Requests and the in-memory cache are keyed on a canonical href rebuilt from the resolved params (`overviewDataKey`), not the address bar. Explicit defaults, reordered params and campaign tags collapse to one key, so the CDN's day-long `s-maxage` is not fragmented by link variants. - **`ref` never reaches that key.** The reference hardware only chooses which column the percentages are measured against, and every cost that needs is already in the payload, so the client derives it from the URL and recomputes the ratio per row. The server still resolves `ref` for SSR, shared links and no-JS. +- **Visited states expire after five minutes.** Visible matrices check freshness every 30 seconds and on focus/visibility changes. Background refreshes share in-flight requests and cannot overwrite a newer selector navigation or server payload. They do not change history or focus. The server caches remain invalidated by ingestion; the client TTL bounds how long an open tab can hide that invalidation. Because the selector commit uses `History.prototype.pushState` rather than Next's patched version, `useSearchParams()` and `usePathname()` stay at the load-time URL on `/overview`. Anything that needs the live URL there listens for `CLIENT_SEARCH_CHANGE_EVENT`; the provider emits its own `$pageview`. Do not add a `useSearchParams()` consumer to the overview tree. diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index 9068f9527..2f27738fc 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -1,5 +1,8 @@ # Data Pipeline Design +For the overview denominator, minimum-SLO selection, measurement dates, and GPU +count repair procedure, see [Overview data integrity](./overview-data-integrity.md). + ## DB Schema Decisions ### Why Metrics Stay in JSONB diff --git a/docs/index.md b/docs/index.md index 9eabf162e..a38fffe49 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,6 +13,7 @@ Design rationale and non-obvious conventions. See [CLAUDE.md](../CLAUDE.md) for - [Architecture](./architecture.md) — Why client-first, route navigation, URL state, provider nesting, server-side caching (unstable_cache + blob), in-memory client cache, color system, analytics enforcement - [D3 Charts](./d3-charts.md) — Why 4 effects, in-place mutation, refs for zoom, rAF throttling, HTML tooltips, Pareto directions, gradient labels +- [Overview Data Integrity](./overview-data-integrity.md) — Verified throughput contract, minimum-SLO selection, date provenance, refresh behavior, and safe repair procedure - [Data Pipeline](./data-pipeline.md) — DB schema reasoning, ETL design, transform pipeline, spline method choice, normalizer resolution order (model/GPU/framework) - [Pitfalls](./pitfalls.md) — Failure modes: token type consistency, schema evolution, empty objects, zoom loss, stale closures, disaggregated metrics, negative splines, date stamping, ref stability, cost inheritance - [GPU Specs](./gpu-specs.md) — Unit conventions, topology invariants, SVG layout rationale, hardware gotchas diff --git a/docs/overview-data-integrity.md b/docs/overview-data-integrity.md new file mode 100644 index 000000000..c002f7a01 --- /dev/null +++ b/docs/overview-data-integrity.md @@ -0,0 +1,114 @@ +# Overview data contract and repairs + +The September 10, 2026 audit traced `/overview` from the main +`SemiAnalysisAI/InferenceX` producer through ETL, stored rows, API reads and the +page's serving frontiers. The app baseline was `9752d602`; the producer baseline +was `21bec226`. The producer's total-throughput arithmetic was correct. The +overview applied a second disaggregation penalty and hid useful measurements. + +## Cost and coverage + +`cost_per_million_total_tokens = gpu_hour_cost * 1_000_000 / (3600 * tput_per_gpu)`. +The rate is the hardware registry's hyperscaler GPU-hour assumption, not a +measured invoice. Input tokens include cached input. These are workload-specific +total-token costs, not output-token prices or a workload-independent ranking. + +The producer already divides total throughput by all physical deployment chips. +Do not multiply it by `decode / (prefill + decode)` again. Fixed-sequence +disaggregated input/output fields have role-specific denominators; AgentX's +three throughput fields all use the entire deployment. See the producer's +[fixed-sequence aggregation](https://github.com/SemiAnalysisAI/InferenceX/blob/21bec22689a8f8274f50123ff11c72a6c5e36410/infx/results/fixed_sequence.py) +and [AgentX aggregation](https://github.com/SemiAnalysisAI/InferenceX/blob/21bec22689a8f8274f50123ff11c72a6c5e36410/utils/agentic/aggregation/process_agentic_result.py). + +An independent request-profile calculation counted 3,603,566,231 input tokens +plus 26,257,390 output tokens over 3,629.862174258 seconds on 12 GPUs: +83,332.44823870831 total tok/s/GPU. At $2.31/GPU-hour this is +$0.00770008178 per million total tokens at that measured knot. The erroneous +8/12 factor raised it to $0.01155012267. The knot is above 50 tok/s/user; +the interpolated tier value is a separate calculation. Regression tests cover +both a source-derived AgentX observation and changing fixed-sequence topologies. + +Each hardware column now selects the lowest cost across eligible FP4/FP8 and +speculative/standard decoding serving series. It retains engine, model, scenario, +offload, release and snapshot boundaries. The chosen tier is a **minimum SLO**: +an observed endpoint above it qualifies at its observed throughput, with a visible +actual-speed label. Interpolation remains within each measured Pareto frontier; +there is no extrapolation. A platform whose eligible measurements all fall below +the SLO remains missing. Current and historical snapshots use the same policy. +The overview's curated hardware/model/scenario scopes remain intentional; it has +no unofficial-run overlay. Shared ingestion fixes also reach unofficial imports. + +## Dates and refresh + +The cost evidence link labels measurement completion when a retained profile +provides it, otherwise the workflow-run attempt date with “measurement date +unavailable.” Its logical curve snapshot is labeled separately and still pins +the source-dashboard link. A reused profile in the audit ran August 12–13, +belonged to an August 14 attempt, and appeared in an August 25 curve snapshot. +Those dates are not interchangeable. A curve with incomplete measurement-date +coverage uses the consistently labeled run-date domain for its evidence. + +Profile ingest records optional `measurement_start_unix_seconds` and +`measurement_end_unix_seconds` from successful profiling requests, including +one-token requests and excluding warmup/errors. The existing full-response +backfill fills missing dates without replacing already populated timing metrics +unless `--force` is used. Aggregate re-ingestion preserves attached-profile dates. + +The browser's visited-state cache expires after five minutes. Visible tabs check +every 30 seconds and on focus; stale responses cannot replace a newer selection +or repopulate a cache invalidated by fresh server props. Background refreshes +preserve history and focus. The overview, /run and /rankings derived server cache keys are bumped; normal +ingestion still invalidates the server's DB caches. A client refresh does not +repair failed ingestion or an uninvalidated server cache. + +## GPU topology and rollout + +For identified single-node InferenceX producers, physical chips are +`TP * PP * PCP`; EP partitions TP devices and does not multiply them. Future +fixed-sequence ingest now follows the same contract as AgentX, preserving +explicit physical counts. One archived Dynamo artifact labeled a complete +8-GPU colocated result as `disagg=true, 8P+0D`; its input plus output throughput +matches total throughput. ETL recognizes this complete zero-decode shape and +mirrors its single pool into the schema's two role columns with `disagg=false`. +This does not change the stored throughput. + +The audit found 105 current AgentX rows whose stored counts disagreed with their +independent total/per-GPU throughput ratio. Updating code does not repair these +existing rows. Run the following **read-only plan** first: + +```bash +bun run --cwd packages/db db:backfill-benchmark-topology +``` + +The September 10 read-only dry run selected 225 rows including history: 223 AgentX count +repairs and two zero-decode classifications. It repairs only the exact legacy +TP×EP fallback where AgentX's aggregate ratio confirms TP×PP×PCP, plus the complete +zero-decode shape. Manual/proprietary deployments, TPUs, unknown shapes, explicit +nonlegacy counts, and fixed-sequence counts without independent evidence are +left alone. Old fixed-sequence counts require source-artifact re-ingestion; +the tool does not infer their original producer contract from dimensions alone. + +After an operator reviews the plan, append `--apply` (interactive confirmation) +or `--apply --yes` (unattended). Apply uses one transaction, checks that the +planned row's config ID and metrics have not changed, and aborts on a uniqueness +conflict. It changes benchmark config references, retaining result IDs, metrics, +logs and replay links; shared config records and evaluations are not rewritten. +It refreshes `latest_benchmarks` after commit. If that refresh fails, the repair +is already committed: rerun the materialized-view refresh before verification. + +Separately, the existing command +`bun run --cwd packages/db db:backfill-full-response-interactivity` fills retained +profile dates. It prompts before writing; `--limit N` permits a small first batch. +Do not use `--force` merely to add dates. After either repair, invalidate the +website DB cache using the existing admin cache command, then compare the affected +raw rows and overview values. The audit and PR preparation do not apply these +production database writes. + +## Validation + +An independent Python implementation of Pareto filtering, Steffen slopes, cubic +Hermite interpolation, configuration selection and cost arithmetic matched all +4,200 checked cells across the captured current snapshot and four historical +snapshots, six SLO tiers, both engine scopes, and the expanded model set. Unit +regressions additionally cover UTC measurement boundaries, refresh races, +unofficial artifact normalization and dry-run/transaction-conflict behavior. diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts index 2bdd670d9..5543c35fb 100644 --- a/packages/app/cypress/e2e/overview.cy.ts +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -785,13 +785,13 @@ describe('Overview page', () => { .as('b200HistoryDelta') .should('have.attr', 'data-history-status', 'comparable') .and('have.attr', 'data-cost-polarity', 'cheaper') - .and('contain.text', '-17%'); + .and('contain.text', '-32%'); cy.get('@b200HistoryDelta') .should(($badge) => { expect($badge).not.to.have.attr('aria-label'); }) .find('.sr-only') - .should('have.text', '17% cheaper than this platform’s Jun 10 result'); + .should('have.text', '32% cheaper than this platform’s Jun 10 result'); platform('mi355x') .find('[data-testid="overview-cost-delta"]') .should('have.attr', 'data-history-status', 'comparable') @@ -1062,7 +1062,7 @@ describe('Overview page', () => { }); }); - it('prefers speculative decode and falls back to labelled standard-decode reads', () => { + it('chooses the cheapest eligible stack and labels measured SLO endpoints', () => { cy.viewport(1280, 900); cy.visit('/overview'); @@ -1092,24 +1092,25 @@ describe('Overview page', () => { .should( 'have.attr', 'title', - 'Estimated from validated benchmark runs. Open raw source dashboard for Jul 18: DeepSeek V4 Pro 0813 1.6T · B200 · SGLang · FP4 · MTP', + 'Estimated from validated benchmark runs. Open raw source dashboard. Run-attempt date: Jul 18; measurement date unavailable. Curve snapshot: Jul 18: DeepSeek V4 Pro 0813 1.6T · B200 · SGLang · FP4 · MTP', ) .and( 'have.attr', 'aria-label', - 'Approximately $0.059. Estimated from validated benchmark runs. Open raw source dashboard for Jul 18: DeepSeek V4 Pro 0813 1.6T · B200 · SGLang · FP4 · MTP', + 'Approximately $0.059. Estimated from validated benchmark runs. Open raw source dashboard. Run-attempt date: Jul 18; measurement date unavailable. Curve snapshot: Jul 18: DeepSeek V4 Pro 0813 1.6T · B200 · SGLang · FP4 · MTP', ); cy.get('[data-testid="overview-pair-missing"]').should('not.exist'); }); // GB300's two points are a single-node and a multi-node aggregate // deployment. They are separate serving series, so neither interpolates - // to the tier and the cell stays empty instead of blending them. + // to the tier; use the qualifying endpoint without blending them. platform('gb300').within(() => { - cy.get('[data-testid="overview-pair-value"]').should('not.exist'); - cy.get('[data-testid="overview-pair-missing"][data-hardware="gb300"]').should( - 'contain.text', - 'no exact @50 result', - ); + cy.get('[data-testid="overview-cost-evidence-link"]').should('have.text', '$0.095'); + cy.get('[data-testid="overview-measured-speed"]') + .should('have.text', 'Measured @60 tok/s/user') + .and('have.attr', 'title') + .and('include', 'without extrapolation'); + cy.get('[data-testid="overview-pair-missing"]').should('not.exist'); }); }); desktopModel('MiniMax-M3', AGENTX).within(() => { @@ -1119,9 +1120,9 @@ describe('Overview page', () => { .and('not.contain.text', 'STP'); }); cy.contains( - 'If a chip does not have FP4 spec decoding available, the next best available configuration is used.', + 'Choose the lowest-cost eligible configuration. AgentX uses P90 full-response token latency; 8K/1K uses median interactivity. Costs include input and output tokens, including cached input.', ).should('exist'); - cy.get('body').should('not.contain.text', 'P90'); + cy.get('body').should('contain.text', 'P90 full-response token latency'); }); it('color-grades the whole cell against B200 and badges a missing baseline with a neutral ∞', () => { @@ -1132,14 +1133,14 @@ describe('Overview page', () => { platform('b200').find('[data-testid="overview-cost-delta"]').should('not.exist'); platform('mi355x') .find('[data-testid="overview-cost-delta"]') - .should('contain.text', '-14%') - .and('have.attr', 'data-cost-polarity', 'cheaper') + .should('contain.text', '+5%') + .and('have.attr', 'data-cost-polarity', 'pricier') .then(($badge) => { // The shade lives on the cell now, never on the badge itself. expect($badge.attr('style') ?? '').not.to.contain('background'); }); - // Cheaper than B200: the whole cell carries the green wash. - expectCellTint('mi355x', 'rgba(16, 185, 129,'); + // The cheaper B200 FP8 baseline changes this cell to a red wash. + expectCellTint('mi355x', 'rgba(239, 68, 68,'); }); desktopModel('DeepSeek-V4-Pro', SINGLE_TURN).within(() => { @@ -1150,13 +1151,9 @@ describe('Overview page', () => { // +71% saturates the alpha ramp; read the computed value so the // assertion survives the browser normalizing `0.40` to `0.4`. expectCellTint('gb200', 'rgba(239, 68, 68, 0.4)'); - // No read at the tier means nothing to grade — the cell stays untinted. - platform('gb300').find('[data-testid="overview-cost-delta"]').should('not.exist'); - platform('gb300').then(([cell]) => { - expect(getComputedStyle(cell.closest('td')!).backgroundColor).to.match( - /rgba\(0, 0, 0, 0\)|transparent/, - ); - }); + // A measured endpoint satisfying the SLO also participates in cost comparison. + platform('gb300').find('[data-testid="overview-cost-delta"]').should('contain.text', '+60%'); + expectCellTint('gb300', 'rgba(239, 68, 68, 0.4)'); }); // The B200 reference column is never washed: its null delta means "no @@ -1200,7 +1197,7 @@ describe('Overview page', () => { cy.contains('h1', PAGE_TITLE).should('exist'); cy.contains( - 'Cost per million total tokens from each platform’s best observed serving envelope', + 'Lowest cost per million total tokens across eligible FP4/FP8 configurations meeting the selected minimum SLO for each scenario.', ).should('exist'); cy.get('[data-testid="overview-scope"]') .should('have.text', SCOPE_LINE) @@ -1370,12 +1367,12 @@ describe('Overview page', () => { .and( 'have.attr', 'title', - 'Open raw source dashboard for Jul 18: Qwen3.5 397B · MI355X · SGLang · FP8 · MTP', + 'Open raw source dashboard. Run-attempt date: Jul 18; measurement date unavailable. Curve snapshot: Jul 18: Qwen3.5 397B · MI355X · SGLang · FP8 · MTP', ) .and( 'have.attr', 'aria-label', - '$0.062. Open raw source dashboard for Jul 18: Qwen3.5 397B · MI355X · SGLang · FP8 · MTP', + '$0.062. Open raw source dashboard. Run-attempt date: Jul 18; measurement date unavailable. Curve snapshot: Jul 18: Qwen3.5 397B · MI355X · SGLang · FP8 · MTP', ) .should('have.attr', 'href') .and('include', '/inference?') @@ -1388,17 +1385,17 @@ describe('Overview page', () => { cy.get( '[data-testid="overview-pair-value"][data-hardware="b200"] [data-testid="overview-cost-evidence-link"]', ) - .should('contain.text', '$0.073') + .should('contain.text', '$0.059') .should('have.attr', 'href') - .and('include', 'i_prec=fp4') + .and('include', 'i_prec=fp8') .and('include', 'i_gpus=b200_sglang_mtp'); }); }); desktopModel('DeepSeek-V4-Pro', SINGLE_TURN).within(() => { - // A cell without a read at the tier carries no evidence link either. + // A measured endpoint has the same source evidence link as an interpolated read. platform('gb300').within(() => { - cy.get('[data-testid="overview-cost-evidence-link"]').should('not.exist'); + cy.get('[data-testid="overview-cost-evidence-link"]').should('have.text', '$0.095'); }); }); expectNoVisibleDatesOrSnapshot(); @@ -1409,17 +1406,15 @@ describe('Overview page', () => { cy.visit('/overview'); desktopModel('DeepSeek-V4-Pro', SINGLE_TURN).within(() => { - platform('mi355x').within(() => { - cy.get('[data-testid="overview-pair-missing"][data-hardware="mi355x"]') - .should('contain.text', '—') - .and('not.contain.text', '∞') - .and('contain.text', 'no exact @50 result'); - }); - platform('gb300').within(() => { - cy.get('[data-testid="overview-pair-missing"][data-hardware="gb300"]') - .should('contain.text', '—') - .and('contain.text', 'no exact @50 result'); - }); + for (const hardware of ['mi355x', 'gb300']) { + platform(hardware).within(() => { + cy.get('[data-testid="overview-pair-missing"]').should('not.exist'); + cy.get('[data-testid="overview-measured-speed"]').should( + 'have.text', + 'Measured @60 tok/s/user', + ); + }); + } platform('b200') .find('[data-testid="overview-cost-evidence-link"]') .should('have.attr', 'title') @@ -1522,17 +1517,14 @@ describe('Overview page', () => { cy.visit('/overview?tier=30'); desktopModel('DeepSeek-V4-Pro', SINGLE_TURN).within(() => { - platform('b300').within(() => { - cy.get('[data-testid="overview-pair-missing"][data-hardware="b300"]') - .should('contain.text', '—') - .and('contain.text', 'no exact @30 result'); - }); + platform('b300') + .find('[data-testid="overview-measured-speed"]') + .should('have.text', 'Measured @70 tok/s/user'); platform('b200') - .find('[data-testid="overview-pair-missing"]') - .should('contain.text', '—') - .and('contain.text', 'no exact @30 result'); + .find('[data-testid="overview-measured-speed"]') + .should('have.text', 'Measured @40 tok/s/user'); }); - // Exact @30 read priced without a B200 baseline: cost plus the ∞ badge. + // B200 now has a measured point above the minimum SLO, so comparison is available. desktopModel('Qwen-3.5-397B-A17B', SINGLE_TURN).within(() => { platform('b300').within(() => { cy.get('[data-testid="overview-pair-value"][data-hardware="b300"]').should( @@ -1540,8 +1532,8 @@ describe('Overview page', () => { '$0.049', ); cy.get('[data-testid="overview-cost-delta"][data-hardware="b300"]') - .should('contain.text', '∞') - .and('have.attr', 'data-cost-polarity', 'no-baseline'); + .should('contain.text', '-18%') + .and('have.attr', 'data-cost-polarity', 'cheaper'); }); }); cy.get('body') @@ -1585,10 +1577,9 @@ describe('Overview page', () => { cy.get( '[data-testid="overview-pair-value"][data-hardware="b200"] [data-testid="overview-cost-evidence-link"]', ).should('have.text', '$0.059'); - cy.get('[data-testid="overview-pair-missing"][data-hardware="gb300"]').should( - 'contain.text', - 'no exact @50 result', - ); + platform('gb300') + .find('[data-testid="overview-measured-speed"]') + .should('have.text', 'Measured @60 tok/s/user'); }); expectNoVisibleDatesOrSnapshot(); expectNoHorizontalOverflow(); @@ -1611,7 +1602,13 @@ describe('Overview page', () => { for (let index = 1; index < rects.length; index += 1) { expect(rects[index - 1].bottom).to.be.at.most(rects[index].top + 1); } - expect(rows.every((row) => row.getBoundingClientRect().height <= 88)).to.equal(true); + for (const row of rows) { + // A qualifying measured endpoint adds one metadata line. + const maxHeight = row.querySelector('[data-testid="overview-measured-speed"]') + ? 104 + : 88; + expect(row.getBoundingClientRect().height).to.be.at.most(maxHeight); + } }); cy.get('[data-testid="overview-mobile-hardware"]').then(($labels) => { @@ -1776,7 +1773,7 @@ describe('Overview page', () => { .and('contain.text', '总览'); cy.contains('h1', PAGE_TITLE_ZH).should('exist'); cy.contains( - '根据各模型标注的测试场景,采用各平台实测的最优服务包络线,计算每百万总 token 成本', + '针对各测试场景,从满足所选最低 SLO 的有效 FP4/FP8 配置中,选取每百万总 token 成本最低的结果。', ).should('exist'); cy.get('[data-testid="overview-scope"]').should('have.text', SCOPE_LINE_ZH); cy.get('[data-testid="overview-source-link"]') @@ -1800,7 +1797,10 @@ describe('Overview page', () => { .as('estimatedB200') .invoke('attr', 'title') .should('include', '根据已验证的基准测试结果估算。') - .and('include', '原始数据仪表板:DeepSeek V4 Pro 0813 1.6T · B200 · SGLang · FP4 · MTP'); + .and( + 'include', + '曲线快照日期:7月18日:DeepSeek V4 Pro 0813 1.6T · B200 · SGLang · FP4 · MTP', + ); cy.get('@estimatedB200') .invoke('attr', 'aria-label') .should('include', '约 $0.059。根据已验证的基准测试结果估算。'); @@ -1810,9 +1810,10 @@ describe('Overview page', () => { .and('include', '/zh/inference?') .and('include', 'g_model=DeepSeek-V4-Pro'); desktopModel('DeepSeek-V4-Pro', SINGLE_TURN) - .find('[data-testid="overview-pair-missing"][data-hardware="gb300"]') - .should('contain.text', '—') - .and('contain.text', '无精确 @50 结果'); + .find( + '[data-testid="overview-platform"][data-hardware="gb300"] [data-testid="overview-measured-speed"]', + ) + .should('have.text', '实测 @60 tok/s/user'); cy.get('body') .invoke('text') .should('not.match', /回退/); @@ -1848,7 +1849,9 @@ describe('Overview page', () => { '[data-testid="overview-pair-value"][data-hardware="b200"] [data-testid="overview-cost-evidence-link"]', ).should('have.text', '$0.064'); }); - cy.contains('如果某款芯片没有可用的 FP4 投机解码配置,则改用次优配置。').should('exist'); + cy.contains( + '选取成本最低的有效配置。AgentX 采用完整响应的 P90 token 延迟,8K/1K 采用中位数交互速度。成本按输入与输出 token 总数计算,包含缓存命中的输入 token。', + ).should('exist'); cy.visit('/zh/overview?tier=100'); cy.get('[data-testid="overview-scope"]').should('have.text', SCOPE_LINE_ZH); diff --git a/packages/app/src/app/api/unofficial-run/route.test.ts b/packages/app/src/app/api/unofficial-run/route.test.ts index d4a8e5f4d..4c6dade7b 100644 --- a/packages/app/src/app/api/unofficial-run/route.test.ts +++ b/packages/app/src/app/api/unofficial-run/route.test.ts @@ -104,6 +104,20 @@ function rawEvalRow(overrides: Record = {}): Record { + it('applies the same physical-chip contract to unofficial fixed-sequence overlays', () => { + const [ep, pp, explicit] = normalizeArtifactRows( + [ + rawRow({ tp: 8, ep: 8, pp: 1, pcp_size: 1 }), + rawRow({ tp: 8, ep: 1, pp: 2, pcp_size: 1 }), + rawRow({ tp: 8, ep: 8, num_gpus: 16 }), + ], + '2026-09-10', + ); + expect([ep.num_decode_gpu, pp.num_decode_gpu, explicit.num_decode_gpu]).toEqual([8, 16, 16]); + expect(ep.num_prefill_gpu).toBe(8); + expect(ep.metrics.tput_per_gpu).toBe(100.5); + }); + it('converts raw artifact row to BenchmarkRow shape', () => { const rows = normalizeArtifactRows([rawRow()], '2026-03-01'); expect(rows).toHaveLength(1); diff --git a/packages/app/src/app/rankings/[slug]/page.tsx b/packages/app/src/app/rankings/[slug]/page.tsx index 57e9587f0..4109348ad 100644 --- a/packages/app/src/app/rankings/[slug]/page.tsx +++ b/packages/app/src/app/rankings/[slug]/page.tsx @@ -84,12 +84,12 @@ function buildFaq( question: `What is the fastest GPU for ${model} inference?`, answer: first && typeof first.throughputPerGpu === 'number' - ? `As of the latest benchmark runs, ${first.hardwareLabel} leads at ${fmtThroughput(first.throughputPerGpu)} tokens/s per GPU on ${workload}, measured at a matched interactivity target of ${data.tier} tokens/s per user.` + ? `As of the latest benchmark runs, ${first.hardwareLabel} leads at ${fmtThroughput(first.throughputPerGpu)} tokens/s per GPU on ${workload}, measured at a minimum interactivity target of ${data.tier} tokens/s per user.` : fallbackLeader, }, { question: `How is "fastest" measured?`, - answer: `Every platform is read at the same interactivity tier (${data.tier} tokens/s per user) on ${workload}, then ranked by measured throughput per GPU. This is the same derivation the InferenceX overview leaderboard renders, so the ranking can never disagree with the dashboard.`, + answer: `Every platform must meet the same minimum interactivity target (${data.tier} tokens/s per user) on ${workload}, then ranked by frontier-derived throughput per GPU. This is the same derivation the InferenceX overview leaderboard renders, so the ranking can never disagree with the dashboard.`, }, ] : [ @@ -97,12 +97,12 @@ function buildFaq( question: `What is the cheapest GPU to run ${model}?`, answer: first && typeof first.costPerMtok === 'number' - ? `As of the latest benchmark runs, ${first.hardwareLabel} is cheapest at ${fmtCostPerMtok(first.costPerMtok)} per million total tokens on ${workload}, at hyperscaler $/GPU/hr pricing and a matched interactivity target of ${data.tier} tokens/s per user.` + ? `As of the latest benchmark runs, ${first.hardwareLabel} is cheapest at ${fmtCostPerMtok(first.costPerMtok)} per million total tokens on ${workload}, at hyperscaler $/GPU/hr pricing and a minimum interactivity target of ${data.tier} tokens/s per user.` : fallbackLeader, }, { question: `How is cost per million tokens calculated?`, - answer: `Measured throughput per GPU at the ${data.tier} tokens/s per user tier is converted to $ per million total (input plus output) tokens using hyperscaler $/GPU/hr rates from the SemiAnalysis AI Cloud TCO model. Slower interactivity targets or cheaper rental tiers change the absolute numbers but rarely the order.`, + answer: `Frontier-derived throughput per GPU meeting the minimum ${data.tier} tokens/s per user tier is converted to $ per million total (input plus output) tokens using hyperscaler $/GPU/hr rates from the SemiAnalysis AI Cloud TCO model. Slower interactivity targets or cheaper rental tiers change the absolute numbers but rarely the order.`, }, ]; return [ @@ -163,7 +163,7 @@ export default async function RankingPage({ params }: Props) { const t: RankingsStrings = { backLabel: 'GPU rankings', heading, - scenarioNote: `Measured on ${scenarioLabel(data.scenario, 'en')} at a matched interactivity target of ${data.tier} tokens/s per user. Hardware without a measurement at this operating point is not ranked.`, + scenarioNote: `Measured on ${scenarioLabel(data.scenario, 'en')} at a minimum interactivity target of ${data.tier} tokens/s per user. Faster measured endpoints qualify without extrapolation; platforms that cannot meet the target are excluded.`, tableCaption: `${heading}: live benchmark ranking`, colRank: 'Rank', colGpu: 'GPU', @@ -182,8 +182,8 @@ export default async function RankingPage({ params }: Props) { : `See the fastest GPU for ${entry.model.seoName}`, methodologyHeading: 'Methodology', methodologyBody: [ - `Every number on this page is a measurement, not a spec-sheet estimate. The InferenceX fleet serves ${entry.model.seoName} on real hardware with community serving engines, sweeping concurrency to trace each platform's throughput-versus-interactivity frontier.`, - `Platforms are then read at the same operating point (${data.tier} tokens/s per user) so the comparison is iso-interactivity: a GPU cannot win by quoting throughput at an unusably slow per-user speed. Cost converts measured throughput to $ per million total tokens using $/GPU/hr rates from the SemiAnalysis AI Cloud TCO model.`, + `These values are derived from benchmark measurements and the stated pricing assumptions. The InferenceX fleet serves ${entry.model.seoName} on real hardware with community serving engines, sweeping concurrency to trace each platform's throughput-versus-interactivity frontier.`, + `Each platform must meet the minimum of ${data.tier} tokens/s per user. We interpolate within measured frontiers or use a faster measured endpoint at its observed throughput, without extrapolation. Cost converts measured throughput to $ per million total tokens using $/GPU/hr rates from the SemiAnalysis AI Cloud TCO model.`, `The derivation is shared with the InferenceX overview leaderboard, and results re-run continuously, so this ranking updates as new engine releases and configs land.`, ], faqHeading: 'Frequently asked questions', diff --git a/packages/app/src/app/run/[slug]/page.tsx b/packages/app/src/app/run/[slug]/page.tsx index a06352919..e3764397f 100644 --- a/packages/app/src/app/run/[slug]/page.tsx +++ b/packages/app/src/app/run/[slug]/page.tsx @@ -46,7 +46,7 @@ function statLedDescription(entry: RunPageEntry, data: RunPageData): string { read.costPerMtok === null ? '' : `, ${fmtCostPerMtok(read.costPerMtok)} per million tokens at hyperscaler pricing`; - return `Measured: ${entry.model.seoName} sustains ${fmtThroughput(read.throughputPerGpu)} tokens/s per GPU on ${entry.chip.label} at ${data.primaryTier} tokens/s per user${cost}. Live data from ${data.configCount} benchmarked configs.`; + return `Measured: ${entry.model.seoName} sustains ${fmtThroughput(read.throughputPerGpu)} tokens/s per GPU on ${entry.chip.label} at a minimum of ${data.primaryTier} tokens/s per user${cost}. Live data from ${data.configCount} benchmarked configs.`; } export async function generateMetadata({ params }: Props): Promise { @@ -87,17 +87,17 @@ function buildFaq(entry: RunPageEntry, data: RunPageData): { question: string; a data.bestThroughputPerGpu === null ? 'n/a' : fmtThroughput(data.bestThroughputPerGpu); const throughputAnswer = typeof read?.throughputPerGpu === 'number' - ? `At an interactivity target of ${data.primaryTier} tokens/s per user on ${workload}, ${chip} sustains ${fmtThroughput(read.throughputPerGpu)} tokens/s per GPU serving ${model}${read.framework ? ` with ${read.framework}` : ''}${read.precision ? ` in ${read.precision.toUpperCase()}` : ''}. Peak measured throughput across all configs is ${peak} tokens/s per GPU.` + ? `With a minimum interactivity target of ${data.primaryTier} tokens/s per user on ${workload}, ${chip} sustains ${fmtThroughput(read.throughputPerGpu)} tokens/s per GPU serving ${model}${read.framework ? ` with ${read.framework}` : ''}${read.precision ? ` in ${read.precision.toUpperCase()}` : ''}. Peak measured throughput across all configs is ${peak} tokens/s per GPU.` : `The InferenceX fleet has ${data.configCount} benchmarked configs for this pairing; see the interactivity ladder above for the operating points reached so far.`; const costAnswer = typeof read?.costPerMtok === 'number' - ? `${fmtCostPerMtok(read.costPerMtok)} per million total tokens at large-hyperscaler-volume ownership $/GPU/hr pricing, at ${data.primaryTier} tokens/s per user. The retail rental tier is tabulated above; slower interactivity targets lower the cost further.` + ? `${fmtCostPerMtok(read.costPerMtok)} per million total tokens at large-hyperscaler-volume ownership $/GPU/hr pricing, at a minimum of ${data.primaryTier} tokens/s per user. The retail rental tier is tabulated above; a lower minimum target may allow a lower cost.` : `Cost per million tokens is derived from measured throughput and $/GPU/hr rates from the SemiAnalysis AI Cloud TCO model; it appears once this pairing reaches the primary interactivity tier.`; const servingAnswer = `The runs behind this page used ${data.frameworks.join(', ')} in ${data.precisions.map((p) => p.toUpperCase()).join(', ')}${data.hasDisagg ? ', including disaggregated prefill' : ''}${data.hasMultinode ? ' and multi-node serving' : ''}. Engines are rebuilt and re-benchmarked continuously, so the best config can change between visits.`; - const methodologyAnswer = `Every number is measured on real ${chip} hardware by the InferenceX fleet, sweeping concurrency on ${workload} to trace the throughput-versus-interactivity frontier${data.newest ? `; the newest run landed on ${data.newest}` : ''}. The same derivation powers the InferenceX overview leaderboard.`; + const methodologyAnswer = `Results are derived from measurements on real ${chip} hardware by the InferenceX fleet, sweeping concurrency on ${workload} to trace the throughput-versus-interactivity frontier${data.newest ? `; the newest run landed on ${data.newest}` : ''}. The same derivation powers the InferenceX overview leaderboard.`; return [ { question: questions.throughput, answer: throughputAnswer }, @@ -161,7 +161,7 @@ export default async function RunPage({ params }: Props) { : ''; const quickAnswer = typeof read?.throughputPerGpu === 'number' - ? `${entry.model.seoName} sustains ${fmtThroughput(read.throughputPerGpu)} tokens/s per GPU on ${entry.chip.label} at ${data.primaryTier} tokens/s per user${quickAnswerCost}${read.framework ? `, served by ${read.framework}` : ''}.${quickAnswerLatency}` + ? `${entry.model.seoName} sustains ${fmtThroughput(read.throughputPerGpu)} tokens/s per GPU on ${entry.chip.label} at a minimum of ${data.primaryTier} tokens/s per user${quickAnswerCost}${read.framework ? `, served by ${read.framework}` : ''}.${quickAnswerLatency}` : `${entry.model.seoName} runs on ${entry.chip.label}: ${data.configCount} benchmarked configs so far. See the interactivity ladder below for measured operating points.`; const t: RunStrings = { @@ -175,14 +175,14 @@ export default async function RunPage({ params }: Props) { statPrecisions: 'Precisions', statFreshness: 'Run dates', ladderHeading: 'Throughput at every interactivity target', - ladderIntro: `Serving is a trade-off: push more concurrent users through a GPU and each user's tokens arrive slower. The ladder below reads the measured frontier at each per-user speed target on ${scenarioLabel(data.scenario, 'en')}, using the best engine and precision at that point.`, - colTier: 'Per-user target', + ladderIntro: `Serving is a trade-off: push more concurrent users through a GPU and each user's tokens arrive slower. The ladder below reads the measured frontier for each minimum per-user speed target on ${scenarioLabel(data.scenario, 'en')}, using the best eligible engine and precision. Faster measured endpoints qualify at their observed throughput; interpolation stays inside the measured frontier.`, + colTier: 'Minimum per-user target', colThroughput: 'Tokens/s per GPU', colCost: '$ / 1M tokens', colEngine: 'Engine', colPrecision: 'Precision', costHeading: 'What serving actually costs', - costIntro: `Converting the ${data.primaryTier} tokens/s per user operating point to $ per million total tokens across rental pricing tiers from the SemiAnalysis AI Cloud TCO model.`, + costIntro: `Converting frontier throughput meeting the minimum of ${data.primaryTier} tokens/s per user to $ per million total tokens across pricing tiers from the SemiAnalysis AI Cloud TCO model.`, colPriceTier: 'Pricing tier', colGpuHour: '$ / GPU / hr', colCostPerMtok: '$ / 1M tokens', diff --git a/packages/app/src/app/zh/rankings/[slug]/page.tsx b/packages/app/src/app/zh/rankings/[slug]/page.tsx index d651e59ed..a9b523171 100644 --- a/packages/app/src/app/zh/rankings/[slug]/page.tsx +++ b/packages/app/src/app/zh/rankings/[slug]/page.tsx @@ -82,12 +82,12 @@ function buildFaqZh( question: `${model} 推理最快的 GPU 是哪款?`, answer: first && typeof first.throughputPerGpu === 'number' - ? `按最新基准测试结果,${first.hardwareLabel} 在${workload}下以单 GPU 每秒 ${fmtThroughput(first.throughputPerGpu)} token 领先,交互速度统一设定为每用户每秒 ${data.tier} token。` + ? `按最新基准测试结果,${first.hardwareLabel} 在${workload}下以单 GPU 每秒 ${fmtThroughput(first.throughputPerGpu)} token 领先,最低交互速度要求为每用户每秒 ${data.tier} token。` : fallbackLeader, }, { question: `“最快”是如何衡量的?`, - answer: `所有平台都在相同的交互速度档位(每用户每秒 ${data.tier} token)和${workload}下读取,再按实测单 GPU 吞吐排名。推导方式与 InferenceX 总览排行榜完全一致,因此本页排名不会与仪表盘出现分歧。`, + answer: `所有平台均需在${workload}下达到相同的最低交互速度要求(每用户每秒 ${data.tier} token),再按实测前沿推导的单 GPU 吞吐量排名。推导方式与 InferenceX 总览排行榜完全一致,因此本页排名不会与仪表盘出现分歧。`, }, ] : [ @@ -95,12 +95,12 @@ function buildFaqZh( question: `运行 ${model} 最省钱的 GPU 是哪款?`, answer: first && typeof first.costPerMtok === 'number' - ? `按最新基准测试结果,${first.hardwareLabel} 在${workload}下成本最低,每百万 token(输入加输出)仅 ${fmtCostPerMtok(first.costPerMtok)},按超大规模云 $/GPU/小时 价格计算,交互速度统一设定为每用户每秒 ${data.tier} token。` + ? `按最新基准测试结果,${first.hardwareLabel} 在${workload}下成本最低,每百万 token(输入加输出)仅 ${fmtCostPerMtok(first.costPerMtok)},按超大规模云 $/GPU/小时 价格计算,最低交互速度要求为每用户每秒 ${data.tier} token。` : fallbackLeader, }, { question: `每百万 token 成本是如何计算的?`, - answer: `将每用户每秒 ${data.tier} token 档位下的实测单 GPU 吞吐,按 SemiAnalysis AI Cloud TCO 模型中的超大规模云 $/GPU/小时 价格换算为每百万 token(输入加输出)成本。更慢的交互档位或更便宜的租用价格会改变绝对数值,但很少改变排名顺序。`, + answer: `将满足每用户每秒至少 ${data.tier} token 要求的实测前沿吞吐量,按 SemiAnalysis AI Cloud TCO 模型中的超大规模云 $/GPU/小时 价格换算为每百万 token(输入加输出)成本。改变最低交互速度要求或 GPU 小时费率假设,可能同时改变成本和排名。`, }, ]; return [ @@ -168,7 +168,7 @@ export default async function ZhRankingPage({ params }: Props) { const t: RankingsStrings = { backLabel: 'GPU 排行榜', heading, - scenarioNote: `测试基于${scenarioLabel(data.scenario, 'zh')},交互速度统一设定为每用户每秒 ${data.tier} token。在该工作点没有实测数据的硬件不参与排名。`, + scenarioNote: `测试基于${scenarioLabel(data.scenario, 'zh')},最低交互速度要求为每用户每秒 ${data.tier} token。超过要求的实测点可直接计入,无需外推;无法达到要求的硬件不参与排名。`, tableCaption: `${heading}:实时基准排行`, colRank: '排名', colGpu: 'GPU', @@ -184,8 +184,8 @@ export default async function ZhRankingPage({ params }: Props) { : `查看 ${entry.model.seoName} 推理最快的 GPU`, methodologyHeading: '测试方法', methodologyBody: [ - `本页所有数字都来自实测,而非纸面规格估算。InferenceX 集群使用社区推理引擎在真实硬件上部署 ${entry.model.seoName},通过扫描并发数绘制每个平台的吞吐与交互速度前沿曲线。`, - `各平台均在同一工作点(每用户每秒 ${data.tier} token)读取数据,确保在相同交互速度下进行比较。单用户速度低到无法正常使用时,即使吞吐量很高,也不能据此获得更高排名。每百万 token(输入加输出)的成本由实测吞吐量和 SemiAnalysis AI Cloud TCO 模型提供的 $/GPU/hr 费率换算得出。`, + `本页数值基于基准测试实测数据和注明的价格假设推导。InferenceX 集群使用社区推理引擎在真实硬件上部署 ${entry.model.seoName},通过扫描并发数绘制每个平台的吞吐与交互速度前沿曲线。`, + `各平台均需达到每用户每秒至少 ${data.tier} token 的要求。我们在实测前沿内插值,或直接采用超过要求的实测点及其实际吞吐量,不进行外推。每百万 token(输入加输出)的成本由实测吞吐量和 SemiAnalysis AI Cloud TCO 模型提供的 $/GPU/hr 费率换算得出。`, `推导逻辑与 InferenceX 总览排行榜共用,基准测试持续重跑,新引擎版本和新配置落地后排行会自动更新。`, ], faqHeading: '常见问题', diff --git a/packages/app/src/app/zh/run/[slug]/page.tsx b/packages/app/src/app/zh/run/[slug]/page.tsx index b74635c08..c5e294b36 100644 --- a/packages/app/src/app/zh/run/[slug]/page.tsx +++ b/packages/app/src/app/zh/run/[slug]/page.tsx @@ -45,7 +45,7 @@ function statLedDescriptionZh(entry: RunPageEntry, data: RunPageData): string { ? '' : `,按超大规模云价格每百万 token 成本 ${fmtCostPerMtok(read.costPerMtok)}`; const chipLabel = entry.chip.label; - return `实测数据:运行 ${entry.model.seoName} 时,${chipLabel} 在每用户每秒 ${data.primaryTier} token 的交互速度下,单 GPU 总吞吐量(输入加输出)可持续达到 ${fmtThroughput(read.throughputPerGpu)} token/s${cost}。共 ${data.configCount} 组实测配置,数据持续更新。`; + return `实测数据:运行 ${entry.model.seoName} 时,${chipLabel} 在每用户每秒至少 ${data.primaryTier} token 的要求下,单 GPU 总吞吐量(输入加输出)可持续达到 ${fmtThroughput(read.throughputPerGpu)} token/s${cost}。共 ${data.configCount} 组实测配置,数据持续更新。`; } export async function generateMetadata({ params }: Props): Promise { @@ -90,17 +90,17 @@ function buildFaqZh( data.bestThroughputPerGpu === null ? '暂无' : fmtThroughput(data.bestThroughputPerGpu); const throughputAnswer = typeof read?.throughputPerGpu === 'number' - ? `在${workload}下运行 ${model},目标交互速度为每用户每秒 ${data.primaryTier} token 时,${chipLabel} 的单 GPU 总吞吐量(输入加输出)可持续达到 ${fmtThroughput(read.throughputPerGpu)} token/s${read.framework ? `,推理引擎为 ${read.framework}` : ''}${read.precision ? `,精度为 ${read.precision.toUpperCase()}` : ''}。全部配置中的实测峰值总吞吐量为单 GPU ${peak} token/s。` + ? `在${workload}下运行 ${model},交互速度至少达到每用户每秒 ${data.primaryTier} token 时,${chipLabel} 的单 GPU 总吞吐量(输入加输出)可持续达到 ${fmtThroughput(read.throughputPerGpu)} token/s${read.framework ? `,推理引擎为 ${read.framework}` : ''}${read.precision ? `,精度为 ${read.precision.toUpperCase()}` : ''}。全部配置中的实测峰值总吞吐量为单 GPU ${peak} token/s。` : `InferenceX 集群已为该组合完成 ${data.configCount} 组实测配置,各交互档位的实测结果见上方阶梯表。`; const costAnswer = typeof read?.costPerMtok === 'number' - ? `按超大规模云大批量自有 $/GPU/小时 价格、每用户每秒 ${data.primaryTier} token 档位计算,每百万 token(输入加输出)成本为 ${fmtCostPerMtok(read.costPerMtok)}。零售租用价格见上方表格;更慢的交互档位成本更低。` + ? `按超大规模云大批量自有 $/GPU/小时 价格、每用户每秒至少 ${data.primaryTier} token 的要求计算,每百万 token(输入加输出)成本为 ${fmtCostPerMtok(read.costPerMtok)}。零售租用价格见上方表格;降低最低交互速度要求可能进一步降低成本。` : `每百万 token 成本由实测吞吐和 SemiAnalysis AI Cloud TCO 模型的 $/GPU/小时 价格换算得出,待该组合达到主交互档位后即会显示。`; const servingAnswer = `本页数据来自 ${data.frameworks.join('、')},精度覆盖 ${data.precisions.map((p) => p.toUpperCase()).join('、')}${data.hasDisagg ? ',包含 prefill 分离部署' : ''}${data.hasMultinode ? ',并包含多节点部署' : ''}。推理引擎持续重新构建并重跑基准,最优配置可能随时变化。`; - const methodologyAnswer = `所有数字均由 InferenceX 集群在真实 ${chipLabel} 硬件上实测,通过在${workload}下扫描并发数绘制吞吐与交互速度前沿曲线${data.newest ? `;最新一次运行落在 ${data.newest}` : ''}。推导方式与 InferenceX 总览排行榜完全一致。`; + const methodologyAnswer = `结果基于 InferenceX 集群在真实 ${chipLabel} 硬件上的实测数据推导,通过在${workload}下扫描并发数绘制吞吐与交互速度前沿曲线${data.newest ? `;最新一次运行落在 ${data.newest}` : ''}。推导方式与 InferenceX 总览排行榜完全一致。`; return [ { question: questions.throughput, answer: throughputAnswer }, @@ -169,7 +169,7 @@ export default async function ZhRunPage({ params }: Props) { : ''; const quickAnswer = typeof read?.throughputPerGpu === 'number' - ? `运行 ${model} 时,${chipLabel} 在每用户每秒 ${data.primaryTier} token 的交互速度下,单 GPU 总吞吐量(输入加输出)可持续达到 ${fmtThroughput(read.throughputPerGpu)} token/s${quickAnswerCost}${read.framework ? `,推理引擎为 ${read.framework}` : ''}。${quickAnswerLatency}` + ? `运行 ${model} 时,${chipLabel} 在每用户每秒至少 ${data.primaryTier} token 的要求下,单 GPU 总吞吐量(输入加输出)可持续达到 ${fmtThroughput(read.throughputPerGpu)} token/s${quickAnswerCost}${read.framework ? `,推理引擎为 ${read.framework}` : ''}。${quickAnswerLatency}` : `${model} 可在 ${chipLabel} 上运行:目前已有 ${data.configCount} 组实测配置,各档位实测结果见下方阶梯表。`; const t: RunStrings = { @@ -183,14 +183,14 @@ export default async function ZhRunPage({ params }: Props) { statPrecisions: '精度', statFreshness: '运行日期', ladderHeading: '各交互档位下的吞吐表现', - ladderIntro: `推理部署是一种权衡:单 GPU 上并发用户越多,每个用户收到 token 的速度就越慢。下表在${scenarioLabel(data.scenario, 'zh')}下按每个单用户速度目标读取实测前沿,取该工作点上最优的引擎与精度。`, - colTier: '单用户速度目标', + ladderIntro: `推理部署是一种权衡:单 GPU 上并发用户越多,每个用户收到 token 的速度就越慢。下表在${scenarioLabel(data.scenario, 'zh')}下,按最低单用户速度要求选取有效引擎与精度中的最优结果。超过要求的实测点按其实际吞吐量计入;插值仅在实测前沿范围内进行。`, + colTier: '最低单用户速度', colThroughput: '单 GPU 每秒 token 数', colCost: '每百万 token 成本', colEngine: '推理引擎', colPrecision: '精度', costHeading: '实际部署成本', - costIntro: `将每用户每秒 ${data.primaryTier} token 工作点的实测吞吐,按 SemiAnalysis AI Cloud TCO 模型的各档租用价格换算为每百万 token(输入加输出)成本。`, + costIntro: `将满足每用户每秒至少 ${data.primaryTier} token 要求的实测前沿吞吐量,按 SemiAnalysis AI Cloud TCO 模型的各档租用价格换算为每百万 token(输入加输出)成本。`, colPriceTier: '价格档位', colGpuHour: '$/GPU/小时', colCostPerMtok: '每百万 token 成本', diff --git a/packages/app/src/components/overview/overview-navigation.test.tsx b/packages/app/src/components/overview/overview-navigation.test.tsx index 746057542..cabeb0a4a 100644 --- a/packages/app/src/components/overview/overview-navigation.test.tsx +++ b/packages/app/src/components/overview/overview-navigation.test.tsx @@ -17,6 +17,7 @@ vi.mock('next/navigation', () => ({ import { OverviewNavigationProvider, + OVERVIEW_CLIENT_CACHE_TTL_MS, useOverviewData, useOverviewNavigationError, useOverviewNavigation, @@ -68,6 +69,7 @@ function Probe() { return ( <> {data.tier} + {data.unchangedRowCount} {reference} {navigation.isPending ? 'pending' : 'settled'} {navigationError ? 'error' : 'ok'} @@ -120,9 +122,96 @@ afterEach(() => { selectHardwareRowScope = undefined; prefetchTier = undefined; vi.unstubAllGlobals(); + vi.useRealTimers(); + vi.restoreAllMocks(); }); describe('OverviewNavigationProvider', () => { + it('refreshes an expired visible matrix without changing history or focus', async () => { + vi.useFakeTimers(); + const { stub, settlers } = deferredFetch(); + vi.spyOn(document, 'visibilityState', 'get').mockReturnValue('visible'); + renderProvider(pageData(50), '/overview'); + const push = vi.spyOn(History.prototype, 'pushState'); + const focused = document.activeElement; + await act(async () => { + await vi.advanceTimersByTimeAsync(OVERVIEW_CLIENT_CACHE_TTL_MS - 1); + }); + expect(stub).not.toHaveBeenCalled(); + await act(async () => { + await vi.advanceTimersByTimeAsync(1); + }); + expect(stub).toHaveBeenCalledTimes(1); + await act(async () => { + settlers[0].resolve(Response.json({ ...pageData(50), unchangedRowCount: 9 })); + await Promise.resolve(); + }); + expect(readProbe('revision')).toBe('9'); + expect(readProbe('pending')).toBe('settled'); + expect(push).not.toHaveBeenCalled(); + expect(document.activeElement).toBe(focused); + }); + + it('waits while hidden, refreshes on focus, and ignores refreshes overtaken by a selector', async () => { + vi.useFakeTimers(); + const { stub, settlers } = deferredFetch(); + const visibility = vi.spyOn(document, 'visibilityState', 'get').mockReturnValue('hidden'); + renderProvider(pageData(50), '/overview'); + await act(async () => { + await vi.advanceTimersByTimeAsync(OVERVIEW_CLIENT_CACHE_TTL_MS); + }); + expect(stub).not.toHaveBeenCalled(); + visibility.mockReturnValue('visible'); + act(() => { + window.dispatchEvent(new Event('focus')); + document.dispatchEvent(new Event('visibilitychange')); + }); + expect(stub).toHaveBeenCalledTimes(1); + act(() => selectTier?.()); + await act(async () => { + settlers[1].resolve(Response.json(pageData(75))); + await Promise.resolve(); + }); + await act(async () => { + settlers[0].resolve(Response.json({ ...pageData(50), unchangedRowCount: 9 })); + await Promise.resolve(); + }); + expect(readProbe('tier')).toBe('75'); + expect(readProbe('revision')).toBe('0'); + }); + + it('refetches an expired visited selection and recovers from a failed background refresh', async () => { + vi.useFakeTimers(); + const { stub, settlers } = deferredFetch(); + vi.spyOn(document, 'visibilityState', 'get').mockReturnValue('visible'); + renderProvider(pageData(50), '/overview'); + act(() => selectTier?.()); + await act(async () => { + settlers[0].resolve(Response.json(pageData(75))); + await Promise.resolve(); + }); + await act(async () => { + await vi.advanceTimersByTimeAsync(OVERVIEW_CLIENT_CACHE_TTL_MS); + }); + await act(async () => { + settlers[1].reject(new Error('offline')); + await Promise.resolve(); + }); + expect(readProbe('error')).toBe('error'); + expect(readProbe('tier')).toBe('75'); + act(() => { + window.history.replaceState({}, '', '/overview'); + window.dispatchEvent(new PopStateEvent('popstate')); + }); + expect(stub).toHaveBeenCalledTimes(3); + await act(async () => { + settlers[2].resolve(Response.json({ ...pageData(50), unchangedRowCount: 3 })); + await Promise.resolve(); + }); + expect(readProbe('revision')).toBe('3'); + expect(readProbe('error')).toBe('ok'); + }); + it('ignores an older selector response after fresh server props arrive', async () => { const { settlers } = deferredFetch(); @@ -144,6 +233,24 @@ describe('OverviewNavigationProvider', () => { expect(readProbe('tier')).toBe('100'); }); + it('does not recache a stale response after fresh server props invalidate it', async () => { + const { stub, settlers } = deferredFetch(); + renderProvider(pageData(50), '/overview'); + act(() => selectTier?.()); + renderProvider(pageData(100), '/overview?tier=100'); + await act(async () => { + settlers[0].resolve(Response.json({ ...pageData(75), unchangedRowCount: 1 })); + await Promise.resolve(); + }); + act(() => selectTier?.()); + expect(stub).toHaveBeenCalledTimes(2); + await act(async () => { + settlers[1].resolve(Response.json({ ...pageData(75), unchangedRowCount: 2 })); + await Promise.resolve(); + }); + expect(readProbe('revision')).toBe('2'); + }); + it('does not write history or route after the provider unmounts', async () => { const { settlers } = deferredFetch(); const replaceState = vi.spyOn(History.prototype, 'replaceState'); diff --git a/packages/app/src/components/overview/overview-navigation.tsx b/packages/app/src/components/overview/overview-navigation.tsx index d3dc933e8..4d12130f2 100644 --- a/packages/app/src/components/overview/overview-navigation.tsx +++ b/packages/app/src/components/overview/overview-navigation.tsx @@ -93,6 +93,12 @@ const OverviewComparisonContext = createContext(n const OverviewNavigationErrorContext = createContext(false); const OverviewNavigationContext = createContext(null); +export const OVERVIEW_CLIENT_CACHE_TTL_MS = 5 * 60 * 1000; +interface CachedOverviewData { + data: OverviewPageData; + fetchedAt: number; +} + export function OverviewNavigationProvider({ initialData, initialHref, @@ -109,31 +115,47 @@ export function OverviewNavigationProvider({ const pendingHrefRef = useRef(initialHref); const committedHrefRef = useRef(initialHref); const navigationIdRef = useRef(0); + const cacheEpochRef = useRef(0); const focusIntentRef = useRef(null); const dataCacheRef = useRef( - new Map([[overviewDataKey(initialHref), initialData]]), + new Map([ + [ + overviewDataKey(initialHref), + { + data: initialData, + fetchedAt: Date.now(), + }, + ], + ]), ); const requestCacheRef = useRef(new Map>()); const load = useCallback((href: string): Promise => { const key = overviewDataKey(href); const cached = dataCacheRef.current.get(key); - if (cached !== undefined) return Promise.resolve(cached); + if (cached !== undefined && Date.now() - cached.fetchedAt < OVERVIEW_CLIENT_CACHE_TTL_MS) { + return Promise.resolve(cached.data); + } const pending = requestCacheRef.current.get(key); if (pending !== undefined) return pending; const url = new URL(key, window.location.origin); + const cacheEpoch = cacheEpochRef.current; const request = fetch(`/api/v1/overview${url.search}`, { headers: { Accept: 'application/json' }, }) .then(async (response) => { if (!response.ok) throw new Error(`Overview request failed (${response.status})`); const nextData = (await response.json()) as OverviewPageData; - dataCacheRef.current.set(key, nextData); + if (cacheEpoch === cacheEpochRef.current) { + dataCacheRef.current.set(key, { data: nextData, fetchedAt: Date.now() }); + } return nextData; }) - .finally(() => requestCacheRef.current.delete(key)); + .finally(() => { + if (requestCacheRef.current.get(key) === request) requestCacheRef.current.delete(key); + }); requestCacheRef.current.set(key, request); return request; @@ -219,12 +241,15 @@ export function OverviewNavigationProvider({ useEffect(() => { ++navigationIdRef.current; + ++cacheEpochRef.current; + dataCacheRef.current.clear(); + requestCacheRef.current.clear(); // Known params always come from the server-resolved props; only the extras // (utm_*, gclid, a fragment) are adopted from the address bar, so a stale // location can never key the cache to the wrong payload. const actual = `${window.location.pathname}${window.location.search}${window.location.hash}`; const href = mergeOverviewControlHref(actual, initialHref, OVERVIEW_SERVER_SEARCH_KEYS); - dataCacheRef.current.set(overviewDataKey(href), initialData); + dataCacheRef.current.set(overviewDataKey(href), { data: initialData, fetchedAt: Date.now() }); committedHrefRef.current = href; setCommittedHref(href); pendingHrefRef.current = href; @@ -233,6 +258,41 @@ export function OverviewNavigationProvider({ setNavigationError(false); }, [initialData, initialHref]); + useEffect(() => { + const refreshVisibleData = () => { + if (document.visibilityState === 'hidden') return; + const href = committedHrefRef.current; + const key = overviewDataKey(href); + if (key !== overviewDataKey(pendingHrefRef.current)) return; + const cached = dataCacheRef.current.get(key); + if (cached && Date.now() - cached.fetchedAt < OVERVIEW_CLIENT_CACHE_TTL_MS) return; + const generation = navigationIdRef.current; + void load(href) + .then((nextData) => { + // A refresh must not replace a later selection, Back navigation, or a + // newly mounted server payload. It never writes history or steals focus. + if ( + generation !== navigationIdRef.current || + key !== overviewDataKey(committedHrefRef.current) + ) + return; + setData(nextData); + setNavigationError(false); + }) + .catch(() => { + if (generation === navigationIdRef.current) setNavigationError(true); + }); + }; + const interval = window.setInterval(refreshVisibleData, 30_000); + window.addEventListener('focus', refreshVisibleData); + document.addEventListener('visibilitychange', refreshVisibleData); + return () => { + window.clearInterval(interval); + window.removeEventListener('focus', refreshVisibleData); + document.removeEventListener('visibilitychange', refreshVisibleData); + }; + }, [load]); + useEffect(() => { const handlePopState = () => { if (window.location.pathname !== '/overview' && window.location.pathname !== '/zh/overview') { diff --git a/packages/app/src/components/overview/overview-presentation.test.tsx b/packages/app/src/components/overview/overview-presentation.test.tsx index 0bc104600..e78a91efe 100644 --- a/packages/app/src/components/overview/overview-presentation.test.tsx +++ b/packages/app/src/components/overview/overview-presentation.test.tsx @@ -44,7 +44,9 @@ describe('OVERVIEW_STRINGS.zh', () => { expect(zh.scopeAria).toContain('hyperscaler'); expect(zh.caption).toContain('token'); expect(zh.scenarioLabels.agentx).toContain('AgentX'); - expect(zh.methodologyNote).toContain('FP4'); + expect(zh.caption).toContain('FP4/FP8'); + expect(zh.methodologyNote).toContain('P90'); + expect(zh.methodologyNote).toContain('缓存命中'); expect(zh.historyCaption(7)).toContain('7–14'); expect(zh.costDeltaAria('20%', true, 'B200')).toEqual( expect.stringMatching(/B200.*20%|20%.*B200/), diff --git a/packages/app/src/components/overview/overview-scorecard.tsx b/packages/app/src/components/overview/overview-scorecard.tsx index 9f3331d91..0fe1a6b9c 100644 --- a/packages/app/src/components/overview/overview-scorecard.tsx +++ b/packages/app/src/components/overview/overview-scorecard.tsx @@ -370,6 +370,14 @@ function CellValue({ ].join(' · '); const evidenceDateLabel = evidenceDate === null ? '' : formatEvidenceDate(formatters, evidenceDate); + const evidenceDescription = [ + member.read.evidenceDateBasis === 'measurement' + ? strings.evidenceMeasurementDate(evidenceDateLabel) + : strings.evidenceRunDate(evidenceDateLabel), + config === null ? '' : strings.evidenceSnapshotDate(formatters.shortDate(config.latestDate)), + ] + .filter(Boolean) + .join('. '); const formattedValue = formatters.cost.format(member.costPerMtok); const estimateExplanation = member.read.estimated ? strings.estimatedTooltip(evidenceTopologies) @@ -379,7 +387,7 @@ function CellValue({ const evidenceAria = config === null || stack === null ? null - : strings.rawDashboardAria(evidenceDateLabel, model.modelLabel, stack); + : strings.rawDashboardAria(evidenceDescription, model.modelLabel, stack); const costText = formattedValue; const comparison = displayedComparison(member, comparisonMode, referenceHardware, referenceCost); const historicalConfig = @@ -445,6 +453,19 @@ function CellValue({ /> )} + {member.read.observedInteractivity === undefined ? null : ( +
+ {strings.measuredSpeed( + member.read.observedInteractivity.toLocaleString(locale, { + maximumFractionDigits: 2, + }), + )} +
+ )} {member.precision === null ? null : (
{config === null ? ( diff --git a/packages/app/src/components/overview/overview-strings.ts b/packages/app/src/components/overview/overview-strings.ts index 478ff8be9..d14afc3b7 100644 --- a/packages/app/src/components/overview/overview-strings.ts +++ b/packages/app/src/components/overview/overview-strings.ts @@ -34,7 +34,7 @@ export const OVERVIEW_STRINGS = { hardwareComparisonLabel: (reference: string) => `vs ${reference}`, referenceSelectorAria: 'Reference hardware', caption: - 'Cost per million total tokens from each platform’s best observed serving envelope for the scenario shown with each model.', + 'Lowest cost per million total tokens across eligible FP4/FP8 configurations meeting the selected minimum SLO for each scenario.', historyCaption: (days: number) => `Current cost and change versus the latest validated platform result ${days}–${days * 2} days earlier.`, modelHeader: 'Model · Scenario', @@ -56,7 +56,13 @@ export const OVERVIEW_STRINGS = { compareCurvesAria: (modelLabel: string, hardwareLabel: string) => `Compare current and historical ${hardwareLabel} cost curves for ${modelLabel}`, rawDashboardAria: (evidenceDate: string, modelLabel: string, stack: string) => - `Open raw source dashboard for ${evidenceDate}: ${modelLabel} · ${stack}`, + `Open raw source dashboard. ${evidenceDate}: ${modelLabel} · ${stack}`, + evidenceRunDate: (date: string) => `Run-attempt date: ${date}; measurement date unavailable`, + evidenceMeasurementDate: (date: string) => `Measurement completed: ${date}`, + evidenceSnapshotDate: (date: string) => `Curve snapshot: ${date}`, + measuredSpeed: (speed: string) => `Measured @${speed} tok/s/user`, + measuredSloTooltip: (tier: number) => + `This measured point exceeds the minimum SLO of ${tier} tok/s/user. Its throughput is used without extrapolation.`, estimatedTooltip: (topologies: readonly string[]) => topologies.length === 0 ? 'Estimated from validated benchmark runs.' @@ -71,7 +77,7 @@ export const OVERVIEW_STRINGS = { }), standardDecodeLabel: 'STP', methodologyNote: - 'If a chip does not have FP4 spec decoding available, the next best available configuration is used.', + 'Choose the lowest-cost eligible configuration. AgentX uses P90 full-response token latency; 8K/1K uses median interactivity. Costs include input and output tokens, including cached input.', costDeltaAria: (pct: string, cheaper: boolean, reference: string) => `${pct} ${cheaper ? 'cheaper' : 'more expensive'} than ${reference}`, costDeltaEvenAria: (reference: string) => `About the same cost as ${reference}`, @@ -147,7 +153,8 @@ export const OVERVIEW_STRINGS = { historyWindowSelectAria: '对比时间窗口', hardwareComparisonLabel: (reference: string) => `对比 ${reference}`, referenceSelectorAria: '基准硬件', - caption: '根据各模型标注的测试场景,采用各平台实测的最优服务包络线,计算每百万总 token 成本。', + caption: + '针对各测试场景,从满足所选最低 SLO 的有效 FP4/FP8 配置中,选取每百万总 token 成本最低的结果。', historyCaption: (days: number) => `当前成本,以及相较于 ${days}–${days * 2} 天前该平台最近一次有效结果的变化。`, modelHeader: '模型 · 场景', @@ -166,7 +173,13 @@ export const OVERVIEW_STRINGS = { compareCurvesAria: (modelLabel: string, hardwareLabel: string) => `对比 ${modelLabel} 在 ${hardwareLabel} 上当前与历史成本曲线`, rawDashboardAria: (evidenceDate: string, modelLabel: string, stack: string) => - `打开 ${evidenceDate} 原始数据仪表板:${modelLabel} · ${stack}`, + `打开原始数据仪表板。${evidenceDate}:${modelLabel} · ${stack}`, + evidenceRunDate: (date: string) => `运行尝试日期:${date};缺少实测日期`, + evidenceMeasurementDate: (date: string) => `实测完成日期:${date}`, + evidenceSnapshotDate: (date: string) => `曲线快照日期:${date}`, + measuredSpeed: (speed: string) => `实测 @${speed} tok/s/user`, + measuredSloTooltip: (tier: number) => + `该实测点超过 ${tier} tok/s/user 的最低 SLO,直接采用其实测吞吐量,不进行外推。`, estimatedTooltip: (topologies: readonly string[]) => topologies.length === 0 ? '根据已验证的基准测试结果估算。' @@ -180,7 +193,8 @@ export const OVERVIEW_STRINGS = { no_exact_at_tier: `无精确 @${tier} 结果`, }), standardDecodeLabel: 'STP', - methodologyNote: '如果某款芯片没有可用的 FP4 投机解码配置,则改用次优配置。', + methodologyNote: + '选取成本最低的有效配置。AgentX 采用完整响应的 P90 token 延迟,8K/1K 采用中位数交互速度。成本按输入与输出 token 总数计算,包含缓存命中的输入 token。', costDeltaAria: (pct: string, cheaper: boolean, reference: string) => `成本比 ${reference} ${cheaper ? '低' : '高'} ${pct}`, costDeltaEvenAria: (reference: string) => `与 ${reference} 成本基本持平`, diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts index 0da921eb3..9980e657f 100644 --- a/packages/app/src/lib/api-documentation.ts +++ b/packages/app/src/lib/api-documentation.ts @@ -226,13 +226,25 @@ const benchmarkMetricsSchema: ApiSchema = { type: 'object', additionalProperties: numberSchema, description: - 'Scalar metric map. Time metrics, including p99_itl and p99_tpot, are in seconds. p99_itl measures inter-token latency; p99_tpot measures per-request time per output token. Use the actual p99_itl field for an inter-token latency requirement, not the reciprocal of p99_intvty. Throughput metrics use tokens per second per GPU unless their name states otherwise; output_tput_per_gpu counts output tokens. Keys evolve independently; measured power / energy / GPU-telemetry keys are typed below.', - properties: Object.fromEntries( - POWER_METRIC_KEYS.map((key): [string, ApiSchema] => [ - key, - { type: 'number', description: powerMetricDescriptions[key] }, - ]), - ), + 'Scalar metric map. Latency metrics, including p99_itl and p99_tpot, are in seconds. p99_itl measures inter-token latency; p99_tpot measures per-request time per output token. Use the actual p99_itl field for an inter-token latency requirement, not the reciprocal of p99_intvty. tput_per_gpu is total input plus output throughput divided by all physical deployment chips; do not apply an additional disaggregation factor. Fixed-sequence disaggregated input/output throughput uses the respective role chip count (zero decode workers uses the total count); AgentX throughput uses all chips for all token types. Optional measurement_*_unix_seconds fields date successful profiling requests, independently of workflow or curve snapshot dates. Keys evolve independently.', + properties: { + ...Object.fromEntries( + POWER_METRIC_KEYS.map((key): [string, ApiSchema] => [ + key, + { type: 'number', description: powerMetricDescriptions[key] }, + ]), + ), + measurement_start_unix_seconds: { + type: 'number', + description: + 'Earliest successful profiling request start, Unix seconds UTC; excludes warmup. Absent when unknown.', + }, + measurement_end_unix_seconds: { + type: 'number', + description: + 'Latest successful profiling request completion, Unix seconds UTC; excludes warmup. Absent when unknown.', + }, + }, }; const powerAuditSchema: ApiSchema = { type: 'object', @@ -2696,6 +2708,19 @@ const overview = { }, ], schemaNotes: [ + { + id: 'throughput-and-measurement-dates', + title: text('Throughput denominators and measurement dates', '吞吐量分母与测量日期'), + description: text( + 'tput_per_gpu already divides input plus output throughput by every physical chip in the deployment, including prefill and decode chips. Do not normalize it again. Fixed-sequence disaggregated input_tput_per_gpu and output_tput_per_gpu use the prefill and decode chip counts respectively; with no separate decode workers, output uses the total count. AgentX uses the total physical chip count for all three throughput fields. Optional measurement_start_unix_seconds and measurement_end_unix_seconds bound successful profiling requests in UTC, excluding warmup and errors. The completion date can precede the workflow-run date or logical curve snapshot because artifacts can be reused. Missing timestamps mean the measurement date is unknown.', + 'tput_per_gpu 已将输入与输出吞吐量之和除以整个部署的物理芯片总数,其中包括 prefill 和 decode 芯片,不应再次归一化。固定序列长度的分离式基准测试中,input_tput_per_gpu 和 output_tput_per_gpu 分别使用 prefill 和 decode 芯片数;没有独立 decode worker 时,输出吞吐量使用总芯片数。AgentX 的这三个吞吐量字段均使用物理芯片总数。可选字段 measurement_start_unix_seconds 和 measurement_end_unix_seconds 记录成功 profiling 请求的起止时间,以 UTC Unix 秒表示,不含 warmup 和失败请求。由于产物可能被复用,测量完成日期可能早于工作流运行日期或曲线快照日期。时间戳缺失表示测量日期未知。', + ), + shape: 'BenchmarkRows', + example: { + measurement_start_unix_seconds: 1786577819.732095, + measurement_end_unix_seconds: 1786581449.59427, + }, + }, { id: 'benchmark-row', title: text('BenchmarkRow', 'BenchmarkRow'), diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index 0c024758e..63a04d354 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -679,10 +679,11 @@ export const apiContractSourceDigests = [ }, { source: 'src/lib/overview-data.ts', - // Reviewed for the Neocloud cost-tier removal: only a comment describing - // the hyperscaler-volume cost basis changed — no parameter or - // OverviewPageData shape change, so the docs stand. - sourceSha256: '16c1cf933ebb09c4421a30417de8d7107ebc5e71dab750f6d849dc11127f989d', + // Reviewed: total-throughput denominator, minimum-SLO/lowest-cost selection, + // optional observedInteractivity/evidenceDateBasis and evidence-date semantics. + // The page-owned contract is documented in docs/overview-data-integrity.md; + // public throughput and profile timestamps are in api-documentation.ts. + sourceSha256: '30102cfc99ff66f914a1f32da20130acf588e88002b4b4d261ff9a4a53f81cf2', reviewArea: { en: 'Overview BFF tier, engine, comparison-window, reference, and model-scope parameters plus the OverviewPageData response shape.', zh: '概览 BFF 的档位、引擎、对比时间窗口、参考硬件和模型范围参数,以及 OverviewPageData 响应结构。', diff --git a/packages/app/src/lib/overview-data.server.ts b/packages/app/src/lib/overview-data.server.ts index 5dd8787de..0f03870d2 100644 --- a/packages/app/src/lib/overview-data.server.ts +++ b/packages/app/src/lib/overview-data.server.ts @@ -134,10 +134,9 @@ async function buildOverviewPageData( ); } -// v2: AgentX rows group above 8K/1K rows (#918). The assembled models array is -// stored in the derived cache, so the display-order change needs a new key — -// v1 entries keep serving the old order until they expire otherwise. -const getCachedOverviewPageData = cachedDerivedData(buildOverviewPageData, 'overview-page-v2'); +// v3 changes total-throughput normalization, SLO coverage, selection and evidence dates. +// Expire assembled values from older code independently of the raw benchmark cache. +const getCachedOverviewPageData = cachedDerivedData(buildOverviewPageData, 'overview-page-v3'); export function getOverviewPageData( tier: OverviewTier = OVERVIEW_PRIMARY_TIER, diff --git a/packages/app/src/lib/overview-data.test.ts b/packages/app/src/lib/overview-data.test.ts index 038848de1..8dee7de9e 100644 --- a/packages/app/src/lib/overview-data.test.ts +++ b/packages/app/src/lib/overview-data.test.ts @@ -154,6 +154,57 @@ function platformFor( } describe('overview engine scope and scenario selection', () => { + it('prices an AgentX disaggregated observation using all-GPU throughput exactly once', () => { + const throughput = (3603566231 + 26257390) / 3629.862174258 / 12; + const summary = buildOverviewModelSummary( + Model.DeepSeek_V4_Pro, + [ + agenticRow(50, 100, throughput, 1, { + model: 'dsv4', + hardware: 'gb300', + disagg: true, + is_multinode: true, + num_prefill_gpu: 4, + num_decode_gpu: 8, + }), + ], + 50, + 'community', + 'agentx', + ); + const platform = summary.platforms.find((p) => p.hardware === 'gb300')!; + expect(platform.read.value).toBeCloseTo(throughput, 3); + expect(platform.costPerMtok).toBeCloseTo((2.31 * 1e6) / (3600 * throughput), 9); + }); + + it('labels profile completion separately from the run date and logical curve snapshot', () => { + const metrics = { + median_intvty: 50, + tput_per_gpu: 8700, + measurement_start_unix_seconds: Date.parse('2026-08-12T23:36:59Z') / 1000, + measurement_end_unix_seconds: Date.parse('2026-08-13T00:37:29Z') / 1000, + }; + const make = (values: Record) => + buildOverviewModelSummary(Model.Qwen3_5, [ + row({ date: '2026-08-14', curve_date: '2026-08-25', metrics: values }), + ]).platforms.find((p) => p.hardware === 'b200')!.read; + expect(make(metrics)).toMatchObject({ + evidenceDate: { from: '2026-08-13', to: '2026-08-13' }, + evidenceDateBasis: 'measurement', + config: { latestDate: '2026-08-25' }, + }); + for (const end of [undefined, Infinity, 1e100, 1]) { + const values = { ...metrics }; + if (end === undefined) delete (values as Record).measurement_end_unix_seconds; + else values.measurement_end_unix_seconds = end; + expect(make(values)).toMatchObject({ + evidenceDate: { from: '2026-08-14', to: '2026-08-14' }, + evidenceDateBasis: 'run', + config: { latestDate: '2026-08-25' }, + }); + } + }); + it('accepts only supported hardware references and defaults invalid input to B200', () => { expect(resolveOverviewReferenceHardware('b300')).toBe('b300'); expect(resolveOverviewReferenceHardware(['mi355x', 'b200'])).toBe('mi355x'); @@ -501,7 +552,7 @@ describe('overview engine scope and scenario selection', () => { }); }); - it('normalizes disaggregated total throughput by all deployed GPUs before interpolation', () => { + it('uses producer-normalized disaggregated total throughput without a second GPU penalty', () => { const summary = buildOverviewModelSummary(Model.DeepSeek_V4_Pro, [ row({ hardware: 'gb300', @@ -541,7 +592,7 @@ describe('overview engine scope and scenario selection', () => { ]); expect(headlinePairOf(summary, 'gb300-vs-b200')?.candidate.read).toMatchObject({ - value: 1224.393, + value: 4546.165, boundary: 'interpolated', evidenceTopologies: ['24P+8D', '16P+8D'], }); @@ -565,7 +616,7 @@ describe('overview engine scope and scenario selection', () => { }); }); - it('normalizes disaggregated rows even when they run on one node', () => { + it('keeps the producer denominator on single-node disaggregated rows', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ ...frontier([16200, 13500, 10800, 8100], { hardware: 'gb300', @@ -578,7 +629,7 @@ describe('overview engine scope and scenario selection', () => { ]); expect(headlinePairOf(summary, 'gb300-vs-b200')?.candidate.read).toMatchObject({ - value: 9000, + value: 13500, evidenceTopologies: ['4P+8D'], }); }); @@ -613,7 +664,9 @@ describe('overview engine scope and scenario selection', () => { ]); expect(headlinePairOf(summary, 'gb300-vs-b200')?.candidate.read).toMatchObject({ - value: null, + value: 7200, + observedInteractivity: 60, + estimated: false, boundary: 'clamped_low', config: { latestDate: '2026-07-20' }, }); @@ -646,10 +699,15 @@ describe('overview engine scope and scenario selection', () => { }), ]); - expect(headlinePairOf(summary, 'gb300-vs-b200')?.candidate.read.value).toBeNull(); + expect(headlinePairOf(summary, 'gb300-vs-b200')?.candidate.read).toMatchObject({ + value: 7200, + boundary: 'clamped_low', + observedInteractivity: 60, + estimated: false, + }); }); - it('uses speculative FP4, speculative FP8, standard FP4, then standard FP8', () => { + it('minimizes cost across eligible precision and speculation buckets', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ ...frontier([8100, 6300, 4500, 2700], { hardware: 'b200', precision: Precision.FP4 }), ...frontier([12600, 10800, 9000, 7200], { @@ -692,9 +750,9 @@ describe('overview engine scope and scenario selection', () => { value: read.value, })), ).toEqual([ - { hardware: 'b200', precision: Precision.FP4, value: 6300 }, - { hardware: 'mi355x', precision: Precision.FP8, value: 8100 }, - { hardware: 'b300', precision: Precision.FP4, value: 9900 }, + { hardware: 'b200', precision: Precision.FP4, value: 10800 }, + { hardware: 'mi355x', precision: Precision.FP4, value: 11700 }, + { hardware: 'b300', precision: Precision.FP8, value: 11700 }, { hardware: 'gb200', precision: Precision.FP8, value: 9000 }, { hardware: 'gb300', precision: Precision.FP4, value: 8100 }, ]); @@ -925,7 +983,7 @@ describe('assembleOverviewHistoricalPageData', () => { }); describe('overview platform selection', () => { - it('keeps FP4 platform boundaries when neither side has an exact read', () => { + it('uses a faster measured endpoint but keeps an unreachable platform missing', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ ...frontierAt( [ @@ -951,8 +1009,10 @@ describe('overview platform selection', () => { expect(pair?.candidate.precision).toBe(Precision.FP4); expect(pair?.baseline.precision).toBe(Precision.FP4); expect(pair?.candidate.read).toMatchObject({ - value: null, + value: 8100, boundary: 'clamped_low', + observedInteractivity: 60, + estimated: false, config: { hardware: 'mi355x' }, }); expect(pair?.baseline.read).toMatchObject({ @@ -960,7 +1020,7 @@ describe('overview platform selection', () => { boundary: 'unreachable', config: { hardware: 'b200' }, }); - expect(pair?.candidate.missingReason).toBe('no_exact_at_tier'); + expect(pair?.candidate.missingReason).toBeNull(); expect(pair?.baseline.missingReason).toBe('cannot_reach_at_tier'); }); @@ -1061,9 +1121,10 @@ describe('overview platform selection', () => { ...frontierAt(unreachable, { hardware: 'mi355x', precision: Precision.FP4 }), ...frontierAt(underSwept, { hardware: 'mi355x', precision: Precision.FP8 }), ]); - expect(headlinePairOf(mixed, 'mi355x-vs-b200')?.candidate.missingReason).toBe( - 'no_exact_at_tier', - ); + expect(headlinePairOf(mixed, 'mi355x-vs-b200')?.candidate).toMatchObject({ + missingReason: null, + read: { value: 8100, observedInteractivity: 60 }, + }); const allUnreachable = buildOverviewModelSummary(Model.Qwen3_5, [ ...baseline, @@ -1321,8 +1382,8 @@ describe('tier-parameterized overview', () => { expect(at50?.baseline.read.value).toBe(9000); const at100 = headlinePairOf(page(100), 'mi355x-vs-b200'); - expect(at100?.candidate.precision).toBe(Precision.FP4); - expect(at100?.candidate.read.value).toBe(3600); + expect(at100?.candidate.precision).toBe(Precision.FP8); + expect(at100?.candidate.read.value).toBe(6300); expect(at100?.baseline.precision).toBe(Precision.FP8); expect(at100?.baseline.read.value).toBe(5400); }); @@ -1361,7 +1422,7 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { expect(page.tier).toBe(50); // DeepSeek: only each series' latest-date sweep survives. B300 and MI355X - // therefore have no exact @50 read; GB200's independent FP8 remains visible. + // use measured endpoints above the minimum SLO; GB200's FP8 remains visible. // GB300's points are single-node and multi-node aggregate deployments, so // they must not be interpolated into one synthetic serving curve. const deepseek = page.models.find( @@ -1373,9 +1434,13 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { (JULY_2026_HYPERSCALER_TCO.b200 * 1e6) / (8101.968 * 3600), 6, ); - expect(dsB300.candidate.read.value).toBeNull(); - expect(dsB300.candidate.costPerMtok).toBeNull(); - expect(dsB300.candidate.missingReason).toBe('no_exact_at_tier'); + expect(dsB300.candidate.read).toMatchObject({ + value: 9000, + boundary: 'clamped_low', + estimated: false, + }); + expect(dsB300.candidate.costPerMtok).toBeGreaterThan(0); + expect(dsB300.candidate.missingReason).toBeNull(); const dsGb200 = headlinePairOf(deepseek, 'gb200-vs-b200')!; expect(dsGb200.candidate.precision).toBe(Precision.FP8); expect(dsGb200.candidate.read.value).toBe(5100); @@ -1383,13 +1448,12 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { (JULY_2026_HYPERSCALER_TCO.gb200 * 1e6) / (5100 * 3600), 6, ); - expect(headlinePairOf(deepseek, 'mi355x-vs-b200')?.candidate.missingReason).toBe( - 'no_exact_at_tier', - ); + expect(headlinePairOf(deepseek, 'mi355x-vs-b200')?.candidate.missingReason).toBeNull(); const dsGb300 = headlinePairOf(deepseek, 'gb300-vs-b200')!; - expect(dsGb300.candidate.read.value).toBeNull(); + expect(dsGb300.candidate.read).toMatchObject({ boundary: 'clamped_low', estimated: false }); + expect(dsGb300.candidate.read.value).toBeGreaterThan(0); expect(dsGb300.candidate.read.evidenceTopologies).toEqual([]); - expect(dsGb300.candidate.missingReason).toBe('no_exact_at_tier'); + expect(dsGb300.candidate.missingReason).toBeNull(); // DeepSeek's AgentX row is priced from its agentic-trace rows alone — the // single-turn sweeps never leak into it, so only the two benchmarked @@ -1435,7 +1499,7 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { ); expect(mmGb300.candidate.costVsReferencePct).toBeNull(); - // Qwen: MI355X independently falls back to FP8 while B200 and B300 use FP4. + // Qwen: MI355X and B200 choose cheaper FP8; B300 chooses FP4. const qwen = page.models.find( (m) => m.model === Model.Qwen3_5 && m.scenario === 'single_turn_8k1k', )!; @@ -1446,20 +1510,20 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { (JULY_2026_HYPERSCALER_TCO.mi355x * 1e6) / (6688 * 3600), 6, ); - expect(qwenMi.baseline.precision).toBe(Precision.FP4); - expect(qwenMi.baseline.read.value).toBeCloseTo(6602.344); + expect(qwenMi.baseline.precision).toBe(Precision.FP8); + expect(qwenMi.baseline.read.value).toBeCloseTo(8100); expect(qwenMi.candidate.costVsReferencePct).toBeCloseTo( (JULY_2026_HYPERSCALER_TCO.mi355x * 1e6) / (6688 * 3600) / - ((JULY_2026_HYPERSCALER_TCO.b200 * 1e6) / (6602.344 * 3600)) - + ((JULY_2026_HYPERSCALER_TCO.b200 * 1e6) / (8100 * 3600)) - 1, 6, ); const qwenB300 = headlinePairOf(qwen, 'b300-vs-b200')!; expect(qwenB300.candidate.precision).toBe(Precision.FP4); expect(qwenB300.candidate.read.value).toBeCloseTo(10585.75); - expect(qwenB300.baseline.precision).toBe(Precision.FP4); - expect(qwenB300.baseline.read.value).toBeCloseTo(6602.344); + expect(qwenB300.baseline.precision).toBe(Precision.FP8); + expect(qwenB300.baseline.read.value).toBeCloseTo(8100); // Standard-decode-only rows remain visible as explicitly labelled // fallbacks: Qwen's GB300 slice has no speculative read, so the platform diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index fb80f36f3..8c9ec4fbf 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -152,6 +152,10 @@ export interface OverviewTierValue { evidenceDate: { from: string; to: string } | null; /** P/D topology labels from the frontier knot(s) backing this tier read. */ evidenceTopologies: string[]; + /** Observed speed when a measured endpoint exceeds the requested minimum SLO. */ + observedInteractivity?: number; + /** Measurement dates require retained request timestamps for the whole series. */ + evidenceDateBasis?: 'measurement' | 'run'; } /** One chart-equivalent serving series. Topology and GPU-count variants may @@ -186,11 +190,12 @@ export interface OverviewTierRead { evidenceDate: { from: string; to: string } | null; evidenceTopologies: string[]; config: OverviewConfigView | null; + observedInteractivity?: number; + evidenceDateBasis?: 'measurement' | 'run'; } -/** Why a platform shows `∞`. `cannot_reach_at_tier` = every - * eligible serving series tops out below the tier; `no_exact_at_tier` = merely - * under-swept. */ +/** Why a platform shows `∞`. Faster measured endpoints satisfy the minimum SLO; + * `cannot_reach_at_tier` means every eligible series tops out below it. */ export type OverviewMissingReason = | 'int4_bf16_only' | 'no_scenario_data' @@ -502,6 +507,8 @@ function readConfigAtTier(config: OverviewConfigResult, tier: number): OverviewT estimated: tierValue?.estimated ?? false, evidenceDate: tierValue?.evidenceDate ?? null, evidenceTopologies: tierValue?.evidenceTopologies ?? [], + observedInteractivity: tierValue?.observedInteractivity, + evidenceDateBasis: tierValue?.evidenceDateBasis, config, }; } @@ -510,9 +517,9 @@ interface ConfigTierRead extends OverviewTierRead { config: OverviewConfigView; } -/** In-range reads only: a clamped or unreachable read remains a coverage gap. */ -const isInRangeTierRead = (read: T): read is T & { value: number } => - read.value !== null && read.boundary === 'interpolated'; +/** A measured endpoint above the minimum SLO is usable without extrapolation. */ +const isUsableTierRead = (read: T): read is T & { value: number } => + read.value !== null && (read.boundary === 'interpolated' || read.boundary === 'clamped_low'); export function overviewTierEvidenceDate(read: OverviewTierRead): string | null { return read.evidenceDate?.to ?? read.config?.latestDate ?? null; @@ -524,7 +531,7 @@ function readFreshness(read: ConfigTierRead): string { function compareTierReads(a: ConfigTierRead, b: ConfigTierRead): number { return ( - Number(isInRangeTierRead(b)) - Number(isInRangeTierRead(a)) || + Number(isUsableTierRead(b)) - Number(isUsableTierRead(a)) || (b.value ?? -1) - (a.value ?? -1) || readFreshness(b).localeCompare(readFreshness(a)) || b.config.latestDate.localeCompare(a.config.latestDate) || @@ -549,7 +556,7 @@ function nonComparableAsMissing( tier: number, ): OverviewTierRead { if (read === undefined) return nullTierRead(tier); - return isInRangeTierRead(read) + return isUsableTierRead(read) ? read : { ...read, @@ -579,17 +586,10 @@ function selectPlatformRead( return { ...readConfigAtTier(config, tier), config: view }; }); - for (const priority of OVERVIEW_SLICE_PRIORITY) { - const exact = reads - .filter( - (read) => - read.config.precision === priority.precision && - isSpeculativeDecode(read.config.specMethod) === priority.speculative && - isInRangeTierRead(read), - ) - .toSorted(compareTierReads)[0]; - if (exact) return exact; - } + // Every candidate on a hardware column has the same GPU-hour rate. Maximize + // total throughput across eligible precision/speculation buckets to minimize cost. + const best = reads.filter(isUsableTierRead).toSorted(compareTierReads)[0]; + if (best) return best; const bestMissingRead = reads.toSorted( (a, b) => @@ -604,15 +604,14 @@ function missingReasonForPlatform( read: OverviewTierRead, bucketReads: readonly OverviewTierRead[], ): OverviewMissingReason | null { - if (isInRangeTierRead(read)) return null; + if (isUsableTierRead(read)) return null; const hardwareRows = workloadRows.filter((row) => row.hardware === hardware); if (hardwareRows.length === 0) return 'no_scenario_data'; const supportedRows = hardwareRows.filter((row) => OVERVIEW_PRECISIONS.includes(row.precision)); if (supportedRows.length === 0) return 'int4_bf16_only'; if (bucketReads.length === 0) return 'no_scenario_data'; // `cannot reach` is a claim about the whole platform, so it holds only when - // EVERY qualified serving series tops out below the tier — one merely - // under-swept stack downgrades the gap to a missing exact read. + // EVERY qualified serving series tops out below the tier. return bucketReads.every((r) => r.boundary === 'unreachable') ? 'cannot_reach_at_tier' : 'no_exact_at_tier'; @@ -682,19 +681,36 @@ function buildPlatformResults( })); } -function deployedGpuFactor(row: BenchmarkRow): number { - const totalGpus = row.num_prefill_gpu + row.num_decode_gpu; - return row.disagg && row.num_prefill_gpu > 0 && row.num_decode_gpu > 0 && totalGpus > 0 - ? row.num_decode_gpu / totalGpus - : 1; -} - function topologyEvidence(row: BenchmarkRow): string | undefined { return row.disagg && row.num_prefill_gpu > 0 && row.num_decode_gpu > 0 ? `${row.num_prefill_gpu}P+${row.num_decode_gpu}D` : undefined; } +function measurementDate(row: BenchmarkRow): string | null { + const start = row.metrics.measurement_start_unix_seconds; + const end = row.metrics.measurement_end_unix_seconds; + return Number.isFinite(start) && + Number.isFinite(end) && + start > 0 && + end >= start && + Number.isFinite(new Date(end * 1000).getTime()) + ? new Date(end * 1000).toISOString().slice(0, 10) + : null; +} + +/** Use a consistently labeled date domain; unknown measurement dates remain unknown. */ +function evidenceDates(rows: readonly BenchmarkRow[]): { + basis: 'measurement' | 'run'; + date: (row: BenchmarkRow) => string; +} { + const measured = rows.every((row) => measurementDate(row) !== null); + return { + basis: measured ? 'measurement' : 'run', + date: (row) => (measured ? measurementDate(row)! : row.date), + }; +} + interface OverviewAgenticTierPoint extends TcoTierPoint { e2eLatency: number; } @@ -702,12 +718,13 @@ interface OverviewAgenticTierPoint extends TcoTierPoint { /** AgentX: the tier axis stays P90 interactivity (the chart's SLA contract); * the throughput read at the tier is total tok/s per deployed GPU. */ function buildAgenticTierReads(rows: readonly BenchmarkRow[]): TcoTierRead[] { + const evidence = evidenceDates(rows); const points = rows.flatMap((row): OverviewAgenticTierPoint[] => { const entry = rowToAggDataEntry(row); - const factor = deployedGpuFactor(row); const interactivity = entry.p90_intvty; const e2eLatency = entry.p90_e2el; - const totalThroughput = entry.tput_per_gpu * factor; + // Both producer schemas already divide total throughput by ALL deployed GPUs. + const totalThroughput = entry.tput_per_gpu; if ( !Number.isFinite(interactivity) || interactivity <= 0 || @@ -723,7 +740,7 @@ function buildAgenticTierReads(rows: readonly BenchmarkRow[]): TcoTierRead[] { interactivity, e2eLatency, throughput: totalThroughput, - date: benchmarkCurveDate(row), + date: evidence.date(row), evidenceLabel: topologyEvidence(row), }, ]; @@ -736,6 +753,7 @@ function buildAgenticTierReads(rows: readonly BenchmarkRow[]): TcoTierRead[] { * usable total-throughput metric are dropped — the overview cannot price * them, so they must not shape the frontier either. */ function buildSingleTurnTierReads(rows: readonly BenchmarkRow[]): TcoTierRead[] { + const evidence = evidenceDates(rows); const points = rows.flatMap((row): TcoTierPoint[] => { const interactivity = singleTurnInteractivity(row.metrics); const totalTput = row.metrics.tput_per_gpu; @@ -743,8 +761,8 @@ function buildSingleTurnTierReads(rows: readonly BenchmarkRow[]): TcoTierRead[] return [ { interactivity, - throughput: totalTput * deployedGpuFactor(row), - date: benchmarkCurveDate(row), + throughput: totalTput, + date: evidence.date(row), evidenceLabel: topologyEvidence(row), }, ]; @@ -808,9 +826,13 @@ function buildConfigResult( estimated: value !== null && row.is_interpolated, evidenceDate: value === null ? null : row.evidence_date, evidenceTopologies: value === null ? [] : (row.evidence_labels ?? []), + observedInteractivity: + row.boundary === 'clamped_low' ? row.frontier_min_interactivity : undefined, + evidenceDateBasis: evidenceDates(rows).basis, }; }), - latestDate: feed[0].latest_date, + // Logical snapshots select curves and pin drilldowns; measurement dates label evidence. + latestDate: rows.map(benchmarkCurveDate).toSorted().at(-1)!, }; } diff --git a/packages/app/src/lib/run-rankings-data.server.ts b/packages/app/src/lib/run-rankings-data.server.ts index dcb3655fa..6d01d68c1 100644 --- a/packages/app/src/lib/run-rankings-data.server.ts +++ b/packages/app/src/lib/run-rankings-data.server.ts @@ -212,7 +212,7 @@ async function loadRunPageData(slug: string): Promise { return buildRunPageData(entry, rows); } -const getCachedRunPageData = cachedDerivedData(loadRunPageData, 'run-page-data-v2'); +const getCachedRunPageData = cachedDerivedData(loadRunPageData, 'run-page-data-v3'); export function getRunPageData(slug: string): Promise { return getCachedRunPageData(slug); @@ -260,7 +260,7 @@ async function loadRankingPageData(slug: string): Promise { return getCachedRankingPageData(slug); diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index e2a33e843..df3f3f9c5 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -145,6 +145,9 @@ export const METRIC_KEYS = new Set([ 'p99_full_response_intvty', 'p99.9_full_response_intvty', 'std_full_response_intvty', + // UTC wall-clock bounds of successful profiling requests, excluding warmup. + 'measurement_start_unix_seconds', + 'measurement_end_unix_seconds', // QPS — queries per second (agentic aiperf) 'median_qps', 'mean_qps', diff --git a/packages/db/package.json b/packages/db/package.json index b354efc8a..083ad09fb 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -33,7 +33,8 @@ "db:reset": "bun --env-file=../../.env src/reset-db.ts", "db:verify": "bun --env-file=../../.env src/verify-db.ts", "test:unit": "vitest run", - "test:unit:coverage": "vitest run --coverage" + "test:unit:coverage": "vitest run --coverage", + "db:backfill-benchmark-topology": "bun --env-file=../../.env src/backfill-benchmark-topology.ts" }, "dependencies": { "@neondatabase/serverless": "^1.1.0", diff --git a/packages/db/src/backfill-benchmark-topology.test.ts b/packages/db/src/backfill-benchmark-topology.test.ts new file mode 100644 index 000000000..7859250e7 --- /dev/null +++ b/packages/db/src/backfill-benchmark-topology.test.ts @@ -0,0 +1,137 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const state = vi.hoisted(() => ({ + rows: [] as unknown[], + conflict: false, + calls: [] as { text: string; values: unknown[] }[], + committed: false, + rolledBack: false, + readerEnded: false, + writerEnded: false, + connections: [] as boolean[], + refresh: vi.fn(), +})); + +vi.mock('./etl/db-utils.js', () => ({ + refreshLatestBenchmarks: state.refresh, + createAdminSql: ({ readonly = false }: { readonly?: boolean }) => { + state.connections.push(readonly); + const sql = Object.assign( + (strings: TemplateStringsArray, ...values: unknown[]) => { + const text = strings.join('?'); + state.calls.push({ text, values }); + if (readonly) return state.rows; + if (text.includes('insert into configs')) return [{ id: 8 }]; + if (state.conflict) return []; + return [{ id: '42' }]; + }, + { + json: (value: unknown) => value, + begin: async (callback: (tx: unknown) => Promise) => { + try { + await callback(sql); + state.committed = true; + } catch (error) { + state.rolledBack = true; + throw error; + } + }, + end: () => { + if (readonly) state.readerEnded = true; + else state.writerEnded = true; + }, + }, + ); + return sql; + }, +})); + +const originalArgv = process.argv; +const originalExitCode = process.exitCode; +beforeEach(() => { + vi.resetModules(); + state.calls = []; + state.connections = []; + state.conflict = false; + state.committed = false; + state.rolledBack = false; + state.readerEnded = false; + state.writerEnded = false; + state.refresh.mockClear(); + state.rows = [ + { + id: '42', + config_id: 7, + benchmark_type: 'agentic_traces', + metrics: { total_tput_tps: 800, tput_per_gpu: 100 }, + config: { + hardware: 'h200', + framework: 'vllm', + model: 'qwen3.5', + precision: 'fp8', + specMethod: 'none', + disagg: false, + isMultinode: false, + prefillTp: 8, + prefillEp: 8, + prefillDpAttn: false, + prefillNumWorkers: 0, + decodeTp: 8, + decodeEp: 8, + decodeDpAttn: false, + decodeNumWorkers: 0, + numPrefillGpu: 64, + numDecodeGpu: 64, + }, + }, + ]; + process.argv = ['bun', 'backfill-benchmark-topology.ts']; + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); +}); +afterEach(() => { + process.argv = originalArgv; + process.exitCode = originalExitCode; + vi.restoreAllMocks(); +}); + +async function run() { + await import('./backfill-benchmark-topology'); + await vi.waitFor(() => expect(state.readerEnded).toBe(true)); +} + +describe('topology repair command', () => { + it('defaults to a read-only plan even with --yes', async () => { + process.argv.push('--yes'); + await run(); + expect(state.connections).toEqual([true]); + expect(state.committed).toBe(false); + expect(state.refresh).not.toHaveBeenCalled(); + expect(console.log).toHaveBeenCalledWith(expect.stringContaining('Dry run: no writes.')); + }); + + it('applies only config references with an optimistic guard, then refreshes', async () => { + process.argv.push('--apply', '--yes'); + await run(); + expect(state.connections).toEqual([true, false]); + const update = state.calls.find((call) => call.text.includes('update benchmark_results'))!; + expect(update.text).toMatch(/set config_id = \?/); + expect(update.text).toMatch(/where id = \? and config_id = \?\s+and metrics = \?/); + expect(update.values).toEqual([8, '42', 7, { total_tput_tps: 800, tput_per_gpu: 100 }]); + expect(state.committed).toBe(true); + expect(state.refresh).toHaveBeenCalledOnce(); + expect(state.writerEnded).toBe(true); + }); + + it('rolls back the plan if a benchmark changed and never refreshes uncommitted data', async () => { + process.argv.push('--apply', '--yes'); + state.conflict = true; + await run(); + expect(state.rolledBack).toBe(true); + expect(state.committed).toBe(false); + expect(state.refresh).not.toHaveBeenCalled(); + expect(state.writerEnded).toBe(true); + expect(process.exitCode).toBe(1); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining('changed since planning')); + }); +}); diff --git a/packages/db/src/backfill-benchmark-topology.ts b/packages/db/src/backfill-benchmark-topology.ts new file mode 100644 index 000000000..3514e2acf --- /dev/null +++ b/packages/db/src/backfill-benchmark-topology.ts @@ -0,0 +1,89 @@ +/** + * Repair benchmark config references without modifying shared config rows or metrics. + * Dry run (default): bun --env-file=../../.env src/backfill-benchmark-topology.ts + * Apply the printed plan: append --apply [--yes]. Conflicts roll back the entire plan. + */ +import { confirm, hasNoSslFlag, hasYesFlag } from './cli-utils.js'; +import { createConfigCache, type ConfigParams } from './etl/config-cache.js'; +import { createAdminSql, refreshLatestBenchmarks, type Sql } from './etl/db-utils.js'; +import { correctedBenchmarkConfig } from './lib/benchmark-topology.js'; + +const apply = process.argv.includes('--apply'); +const reader = createAdminSql({ readonly: true, noSsl: hasNoSslFlag(), max: 1 }); + +async function main(): Promise { + const rows = await reader< + { + id: string; + config_id: number; + benchmark_type: string; + metrics: Record; + config: ConfigParams; + }[] + >` + select br.id, br.config_id, br.benchmark_type, br.metrics, + jsonb_build_object( + 'hardware', c.hardware, 'framework', c.framework, 'model', c.model, + 'precision', c.precision, 'specMethod', c.spec_method, + 'disagg', c.disagg, 'isMultinode', c.is_multinode, + 'prefillTp', c.prefill_tp, 'prefillEp', c.prefill_ep, + 'prefillDpAttn', c.prefill_dp_attention, 'prefillNumWorkers', c.prefill_num_workers, + 'decodeTp', c.decode_tp, 'decodeEp', c.decode_ep, + 'decodeDpAttn', c.decode_dp_attention, 'decodeNumWorkers', c.decode_num_workers, + 'numPrefillGpu', c.num_prefill_gpu, 'numDecodeGpu', c.num_decode_gpu + ) as config + from benchmark_results br join configs c on c.id = br.config_id + where br.benchmark_type in ('single_turn', 'agentic_traces') + and (not c.is_multinode or (c.disagg and c.num_decode_gpu = 0)) + order by br.id + `; + const plan = rows.flatMap((row) => { + const corrected = correctedBenchmarkConfig(row.config, row.benchmark_type, row.metrics); + return corrected === row.config ? [] : [{ ...row, corrected }]; + }); + for (const row of plan) { + console.log( + JSON.stringify({ + id: row.id, + configId: row.config_id, + benchmarkType: row.benchmark_type, + before: row.config, + after: row.corrected, + }), + ); + } + console.log( + `${plan.length} benchmark rows require repair. ${apply ? 'Apply requested.' : 'Dry run: no writes.'}`, + ); + if (!apply || plan.length === 0) return; + if (!hasYesFlag() && !(await confirm('Apply this plan in one transaction? (y/N) '))) return; + + const writer = createAdminSql({ noSsl: hasNoSslFlag(), max: 1 }); + try { + await writer.begin(async (tx) => { + const configs = createConfigCache(tx as unknown as Sql); + for (const row of plan) { + const configId = await configs.getOrCreateConfig(row.corrected); + const changed = await tx` + update benchmark_results set config_id = ${configId} + where id = ${row.id} and config_id = ${row.config_id} + and metrics = ${tx.json(row.metrics)} + returning id + `; + if (changed.length !== 1) + throw new Error(`Benchmark ${row.id} changed since planning; retry the dry run.`); + } + }); + await refreshLatestBenchmarks(writer); + console.log('Repair committed. Invalidate the website DB cache before verifying public reads.'); + } finally { + await writer.end(); + } +} + +main() + .catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }) + .finally(() => reader.end()); diff --git a/packages/db/src/backfill-full-response-interactivity.ts b/packages/db/src/backfill-full-response-interactivity.ts index 6a59dfcfa..9af298f69 100644 --- a/packages/db/src/backfill-full-response-interactivity.ts +++ b/packages/db/src/backfill-full-response-interactivity.ts @@ -48,7 +48,9 @@ async function main(): Promise { join agentic_trace_replay atr on atr.id = br.trace_replay_id where br.benchmark_type = 'agentic_traces' and atr.profile_export_jsonl_gz is not null - and not (br.metrics ? 'median_full_response_itl') + and (not (br.metrics ? 'median_full_response_itl') + or not (br.metrics ? 'measurement_start_unix_seconds') + or not (br.metrics ? 'measurement_end_unix_seconds')) order by br.id ${flags.limit ? sql`limit ${flags.limit}` : sql``} `; @@ -71,10 +73,18 @@ async function main(): Promise { console.warn(` id=${id}: profile has no usable request samples, skipping`); return 'skipped'; } + const dates = Object.fromEntries( + Object.entries(patch).filter( + ([key]) => + key === 'measurement_start_unix_seconds' || key === 'measurement_end_unix_seconds', + ), + ); await sql` update benchmark_results - set metrics = metrics || ${jsonbParam(sql, patch)} + set metrics = case when not ${flags.force} and metrics ? 'median_full_response_itl' + then ${jsonbParam(sql, dates)} || metrics + else metrics || ${jsonbParam(sql, patch)} end where id = ${id} `; return 'ok'; diff --git a/packages/db/src/etl/benchmark-ingest.ts b/packages/db/src/etl/benchmark-ingest.ts index 617c51efe..b907262a1 100644 --- a/packages/db/src/etl/benchmark-ingest.ts +++ b/packages/db/src/etl/benchmark-ingest.ts @@ -113,11 +113,14 @@ export async function bulkIngestBenchmarkRows( ) do update set -- Replace metrics with the fresh artifact values, but carry over - -- kv_cache_pool_tokens: it is derived from the server log at - -- insertServerLog time (not present in any artifact JSON), so a later - -- upsert from the aggregated results_bmk artifact would silently wipe it. + -- fields derived from the attached server log / trace profile. A later + -- upsert from an aggregate artifact must not wipe that evidence. metrics = excluded.metrics || jsonb_strip_nulls( - jsonb_build_object('kv_cache_pool_tokens', benchmark_results.metrics->'kv_cache_pool_tokens') + jsonb_build_object( + 'kv_cache_pool_tokens', benchmark_results.metrics->'kv_cache_pool_tokens', + 'measurement_start_unix_seconds', benchmark_results.metrics->'measurement_start_unix_seconds', + 'measurement_end_unix_seconds', benchmark_results.metrics->'measurement_end_unix_seconds' + ) ), image = excluded.image, workers = excluded.workers diff --git a/packages/db/src/etl/benchmark-mapper.test.ts b/packages/db/src/etl/benchmark-mapper.test.ts index df9de7f47..bc5b5ce9c 100644 --- a/packages/db/src/etl/benchmark-mapper.test.ts +++ b/packages/db/src/etl/benchmark-mapper.test.ts @@ -91,6 +91,28 @@ function dirtyPowerPayload(): Record { describe('mapBenchmarkRow', () => { describe('v1 schema', () => { + it.each([ + { tp: 8, ep: 8, pp: 1, pcp_size: 1, expected: 8 }, + { tp: 8, ep: 1, pp: 2, pcp_size: 1, expected: 16 }, + { tp: 8, ep: 1, pp: 1, pcp_size: 2, expected: 16 }, + ])('matches the fixed-sequence producer physical GPU count: %j', ({ expected, ...shape }) => { + const mapped = mapBenchmarkRow( + makeV1Row({ ...shape, is_multinode: false, disagg: false }), + createSkipTracker(), + )!; + expect(mapped.config.numPrefillGpu).toBe(expected); + expect(mapped.config.numDecodeGpu).toBe(expected); + expect(mapped.metrics.tput_per_gpu).toBe(1234.5); + }); + + it('retains an explicit physical GPU count', () => { + const mapped = mapBenchmarkRow( + makeV1Row({ tp: 8, ep: 8, num_gpus: 16, is_multinode: false, disagg: false }), + createSkipTracker(), + )!; + expect(mapped.config.numDecodeGpu).toBe(16); + }); + it('maps a valid v1 row to BenchmarkParams', () => { const tracker = createSkipTracker(); const result = mapBenchmarkRow(makeV1Row(), tracker); @@ -1184,7 +1206,7 @@ describe('mapBenchmarkRow — v3 agentic nested agg schema', () => { [{ framework: 'mori-sglang' }, 16], [{ pp: true }, 16], [{ pp: null }, 16], - [{ request_metrics: undefined }, 16], + [{ request_metrics: undefined }, 4], ])('counts physical GPUs for the AgentX producer shape %j', (overrides, expected) => { // Qwen3.8 H200 run 33038487711 uses TP4/EP4 on four GPUs, not sixteen. const result = mapBenchmarkRow( diff --git a/packages/db/src/etl/benchmark-mapper.ts b/packages/db/src/etl/benchmark-mapper.ts index 2dccff9bb..d7b2aed8d 100644 --- a/packages/db/src/etl/benchmark-mapper.ts +++ b/packages/db/src/etl/benchmark-mapper.ts @@ -26,6 +26,7 @@ import { } from './normalizers'; import { normalizeLegacyTpuRow, physicalChipCount, roleChipCount } from './tpu-normalization'; import { extractRuntimeMetadata } from './runtime-metadata'; +import { isColocatedResult, PHYSICAL_TP_FRAMEWORKS } from '../lib/benchmark-topology'; export { flattenAgenticAggRow }; @@ -253,10 +254,26 @@ export function mapBenchmarkRow( // deployments such as one distributed vLLM server. A non-zero decode worker // pool, however, is structural proof of disaggregation and preserves older // Dynamo artifacts that incorrectly emitted disagg=false. - const disagg = frameworkDisagg || parallelism.decodeNumWorkers > 0; + let disagg = frameworkDisagg || parallelism.decodeNumWorkers > 0; let metrics = captureNumericMetrics(row); normalizePowerContractMetrics(row, metrics); if (isAgentic) metrics = preferFullResponseMetrics(metrics); + if ( + isColocatedResult( + { + hardware: gpuKey, + framework, + model: modelKey, + precision, + specMethod, + disagg, + isMultinode, + ...parallelism, + }, + metrics, + ) + ) + disagg = false; if (!disagg) { const usePrefill = parallelism.decodeTp <= 0 || @@ -443,15 +460,16 @@ function resolveParallelism(row: Record, frameworkDisagg: boolean): if ( row.num_gpus === undefined && !frameworkDisagg && - String(row.scenario_type ?? '').startsWith('agentic') && - row.request_metrics && - typeof row.request_metrics === 'object' && - !Array.isArray(row.request_metrics) && + hwToGpuKey(String(row.hw ?? '')) !== 'tpuv7' && + PHYSICAL_TP_FRAMEWORKS.has( + normalizeFramework(String(row.framework ?? ''), row.disagg).framework, + ) && parseOptionalBool(row.is_multinode) === false && parseOptionalBool(row.disagg) === false ) { - // Match the v3 AgentX producer's physical-device count. EP and DCP share - // TP devices; PP and PCP add devices. Legacy flat rows keep their fallback. + // Both fixed-sequence and AgentX producers use this physical-device count. + // EP and DCP share TP devices; PP and PCP add devices. Unidentified legacy + // rows keep their fallback and explicit num_gpus remains authoritative. const physicalTp = physicalChipCount(row.tp); const pp = physicalChipCount(row.pp === undefined ? 1 : row.pp); const pcp = physicalChipCount(row.pcp_size === undefined ? 1 : row.pcp_size); diff --git a/packages/db/src/etl/full-response-interactivity.test.ts b/packages/db/src/etl/full-response-interactivity.test.ts index f0c29ca9b..0f2d6aab0 100644 --- a/packages/db/src/etl/full-response-interactivity.test.ts +++ b/packages/db/src/etl/full-response-interactivity.test.ts @@ -54,6 +54,47 @@ describe('fullResponseItlSample', () => { }); describe('fullResponseMetricsFromProfile', () => { + it('dates the successful profiling window, including one-token requests, across UTC midnight', () => { + const start = Date.parse('2026-08-12T23:59:00Z') / 1000; + const record = (offset: number, phase = 'profiling', error?: string) => + profileRecord({ + metadata: { + benchmark_phase: phase, + request_start_ns: (start + offset) * 1e9, + request_end_ns: (start + offset + 120) * 1e9, + }, + metrics: { output_sequence_length: { value: 1 } }, + error, + }); + const result = fullResponseMetricsFromProfile( + [ + record(-1000, 'warmup'), + record(0), + record(30), + record(1000, 'profiling', 'failed'), + 'null', + '{bad', + ].join('\n'), + ); + expect(result).toEqual({ + measurement_start_unix_seconds: start, + measurement_end_unix_seconds: start + 150, + }); + expect(new Date(result.measurement_end_unix_seconds * 1000).toISOString()).toBe( + '2026-08-13T00:01:30.000Z', + ); + }); + + it('does not invent measurement dates for missing or reversed timestamps', () => { + const result = fullResponseMetricsFromProfile( + profileRecord({ + metadata: { benchmark_phase: 'profiling', request_start_ns: 2e9, request_end_ns: 1e9 }, + }), + ); + expect(result).not.toHaveProperty('measurement_end_unix_seconds'); + expect(fullResponseMetricsFromProfile('null\n{}\n{bad')).toEqual({}); + }); + it('skips warmup, failed, malformed, and one-token records', () => { const valid = profileRecord({ metrics: { diff --git a/packages/db/src/etl/full-response-interactivity.ts b/packages/db/src/etl/full-response-interactivity.ts index eeb95c7cb..c471bb9ac 100644 --- a/packages/db/src/etl/full-response-interactivity.ts +++ b/packages/db/src/etl/full-response-interactivity.ts @@ -121,19 +121,42 @@ export function fullResponseItlSample(record: ProfileRecord): number | undefined */ export function fullResponseMetricsFromProfile(jsonl: string): Record { const samples: number[] = []; + let startSeconds = Infinity; + let endSeconds = -Infinity; for (const line of jsonl.split('\n')) { if (!line.trim()) continue; try { - const sample = fullResponseItlSample(JSON.parse(line) as ProfileRecord); + const record = JSON.parse(line) as ProfileRecord; + const sample = fullResponseItlSample(record); if (sample !== undefined && Number.isFinite(sample) && sample > 0) samples.push(sample); + // Match the producer's successful profiling window, including one-token + // requests that cannot contribute an ITL sample. Never date from warmup. + const start = Number(record.metadata?.request_start_ns) / 1e9; + const end = Number(record.metadata?.request_end_ns) / 1e9; + if ( + !record.error && + record.metadata?.benchmark_phase === 'profiling' && + Number.isFinite(start) && + start > 0 && + Number.isFinite(end) && + end >= start && + Number.isFinite(new Date(end * 1000).getTime()) + ) { + startSeconds = Math.min(startSeconds, start); + endSeconds = Math.max(endSeconds, end); + } } catch { // A malformed record does not invalidate the remaining profile. } } - if (samples.length === 0) return {}; + const patch: Record = {}; + if (Number.isFinite(startSeconds) && Number.isFinite(endSeconds)) { + patch.measurement_start_unix_seconds = startSeconds; + patch.measurement_end_unix_seconds = endSeconds; + } + if (samples.length === 0) return patch; const stats = percentileStats(samples); - const patch: Record = {}; for (const stat of FULL_RESPONSE_STAT_KEYS) { const itl = stats[stat]; patch[`${stat}_full_response_itl`] = itl; diff --git a/packages/db/src/etl/trace-replay-ingest.test.ts b/packages/db/src/etl/trace-replay-ingest.test.ts index 1252cdbdc..9e46e08a8 100644 --- a/packages/db/src/etl/trace-replay-ingest.test.ts +++ b/packages/db/src/etl/trace-replay-ingest.test.ts @@ -45,6 +45,8 @@ function preparedFixture(): PreparedTraceReplay { computeMs: 20, cacheHitRates: null, fullResponseMetrics: { + measurement_start_unix_seconds: 1786577819, + measurement_end_unix_seconds: 1786581449, median_full_response_itl: 0.005, median_full_response_intvty: 200, median_itl: 0.005, @@ -177,9 +179,13 @@ describe('persistPreparedTraceReplay', () => { ).toHaveLength(6); expect(linkCall?.values.some((value) => Array.isArray(value) && value.includes(41))).toBe(true); const metricUpdate = calls.find((call) => - call.text.includes("not (metrics ? 'median_full_response_itl')"), + call.text.includes("case when metrics ? 'median_full_response_itl'"), ); expect(metricUpdate?.values).toContainEqual(preparedFixture().fullResponseMetrics); + expect(metricUpdate?.values).toContainEqual({ + measurement_start_unix_seconds: 1786577819, + measurement_end_unix_seconds: 1786581449, + }); expect(metricUpdate?.values).not.toContain( JSON.stringify(preparedFixture().fullResponseMetrics), ); diff --git a/packages/db/src/etl/trace-replay-ingest.ts b/packages/db/src/etl/trace-replay-ingest.ts index 77a5843ee..859ec84f6 100644 --- a/packages/db/src/etl/trace-replay-ingest.ts +++ b/packages/db/src/etl/trace-replay-ingest.ts @@ -373,13 +373,22 @@ export async function persistPreparedTraceReplay( log('updated cache-hit metrics from chart series'); } if (Object.keys(fullResponseMetrics).length > 0) { + // Existing producer timing remains authoritative; attach only missing + // profile dates, without mixing new percentiles into its metric family. + const dates = Object.fromEntries( + Object.entries(fullResponseMetrics).filter( + ([key]) => + key === 'measurement_start_unix_seconds' || key === 'measurement_end_unix_seconds', + ), + ); await tx` update benchmark_results - set metrics = metrics || ${tx.json(fullResponseMetrics)} + set metrics = case when metrics ? 'median_full_response_itl' + then ${tx.json(dates)} || metrics + else metrics || ${tx.json(fullResponseMetrics)} end where id = any(${tx.array(unlinked.map((row) => row.id))}::bigint[]) - and not (metrics ? 'median_full_response_itl') `; - log('filled full-response ITL and interactivity from the AIPerf profile'); + log('filled measurement dates and full-response timing from the AIPerf profile'); } linkedCount = unlinked.length; await updateAtomKvCachePoolTokens( diff --git a/packages/db/src/lib/benchmark-topology.test.ts b/packages/db/src/lib/benchmark-topology.test.ts new file mode 100644 index 000000000..a5418510a --- /dev/null +++ b/packages/db/src/lib/benchmark-topology.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from 'vitest'; +import { mapBenchmarkRow } from '../etl/benchmark-mapper'; +import type { ConfigParams } from '../etl/config-cache'; +import { createSkipTracker } from '../etl/skip-tracker'; +import { correctedBenchmarkConfig } from './benchmark-topology'; + +const config: ConfigParams = { + hardware: 'h200', + framework: 'vllm', + model: 'qwen3.5', + precision: 'fp8', + specMethod: 'none', + disagg: false, + isMultinode: false, + prefillTp: 8, + prefillEp: 8, + prefillDpAttn: false, + prefillNumWorkers: 0, + decodeTp: 8, + decodeEp: 8, + decodeDpAttn: false, + decodeNumWorkers: 0, + numPrefillGpu: 64, + numDecodeGpu: 64, +}; + +describe('persisted benchmark topology repair', () => { + it('repairs the independently verified AgentX 8-GPU shape without changing throughput', () => { + const metrics = { total_tput_tps: 76145.02392, tput_per_gpu: 9518.12799, pp: 1, pcp_size: 1 }; + const before = structuredClone(metrics); + const fixed = correctedBenchmarkConfig(config, 'agentic_traces', metrics); + expect(fixed).toEqual({ ...config, numPrefillGpu: 8, numDecodeGpu: 8 }); + expect(metrics).toEqual(before); + expect(correctedBenchmarkConfig(fixed, 'agentic_traces', metrics)).toBe(fixed); + }); + + it.each(['pp', 'pcp_size'])( + 'counts %s devices when the aggregate ratio confirms them', + (dimension) => { + const old = { ...config, prefillEp: 1, decodeEp: 1, numPrefillGpu: 8, numDecodeGpu: 8 }; + expect( + correctedBenchmarkConfig(old, 'agentic_traces', { + [dimension]: 2, + total_tput_tps: 1600, + tput_per_gpu: 100, + }), + ).toEqual({ + ...old, + numPrefillGpu: 16, + numDecodeGpu: 16, + }); + }, + ); + + it.each([ + [{}, 'agentic_traces', {}], + [{}, 'single_turn', {}], + [{}, 'agentic_traces', { total_tput_tps: 100, tput_per_gpu: 10 }], + [{}, 'single_turn', { pp: 0 }], + [{}, 'single_turn', { pcp_size: 1.5 }], + [{ hardware: 'tpuv7' }, 'single_turn', {}], + [{ framework: 'trtllm' }, 'single_turn', {}], + [{ isMultinode: true }, 'single_turn', {}], + [{ numPrefillGpu: 16, numDecodeGpu: 16 }, 'single_turn', {}], + [{ disagg: true }, 'single_turn', {}], + ])('retains ambiguous or nonlegacy topology %j', (overrides, type, metrics) => { + const candidate = { ...config, ...overrides } as ConfigParams; + expect( + correctedBenchmarkConfig(candidate, type as string, metrics as Record), + ).toBe(candidate); + }); + + it('recognizes the complete zero-decode fixed-sequence artifact in ingest and backfill', () => { + // Original 25948327237 / attempt 5: one 8-GPU pool completed both phases. + const raw = { + infmax_model_prefix: 'dsv4', + hw: 'gb200-nvl', + framework: 'dynamo-sglang', + precision: 'fp4', + disagg: true, + is_multinode: true, + prefill_tp: 4, + prefill_ep: 1, + prefill_num_workers: 2, + num_prefill_gpu: 8, + decode_tp: 0, + decode_ep: 0, + decode_num_workers: 0, + num_decode_gpu: 0, + isl: 8192, + osl: 1024, + conc: 1, + tput_per_gpu: 149.7145445014, + input_tput_per_gpu: 132.9915129295, + output_tput_per_gpu: 16.7230315719, + }; + const mapped = mapBenchmarkRow(raw, createSkipTracker())!; + expect(mapped.config).toMatchObject({ + disagg: false, + isMultinode: true, + numPrefillGpu: 8, + numDecodeGpu: 8, + prefillTp: 4, + decodeTp: 4, + decodeNumWorkers: 2, + }); + const old = { + ...mapped.config, + disagg: true, + decodeTp: 0, + decodeEp: 0, + decodeNumWorkers: 0, + numDecodeGpu: 0, + }; + expect(correctedBenchmarkConfig(old, 'single_turn', mapped.metrics)).toEqual(mapped.config); + // An incomplete or role-normalized result is not proof of colocation. + expect( + correctedBenchmarkConfig(old, 'single_turn', { ...mapped.metrics, output_tput_per_gpu: 0 }), + ).toBe(old); + expect( + correctedBenchmarkConfig(old, 'single_turn', { ...mapped.metrics, tput_per_gpu: 500 }), + ).toBe(old); + }); +}); diff --git a/packages/db/src/lib/benchmark-topology.ts b/packages/db/src/lib/benchmark-topology.ts new file mode 100644 index 000000000..8d8f7044f --- /dev/null +++ b/packages/db/src/lib/benchmark-topology.ts @@ -0,0 +1,94 @@ +import type { ConfigParams } from '../etl/config-cache.js'; + +/** These InferenceX producers count TP devices; EP partitions those devices. */ +export const PHYSICAL_TP_FRAMEWORKS = new Set([ + 'vllm', + 'sglang', + 'trt', + 'atom', + 'dynamo-vllm', + 'dynamo-sglang', + 'dynamo-trt', +]); + +const positive = (n: unknown): n is number => typeof n === 'number' && Number.isFinite(n) && n > 0; +const integer = (n: unknown): n is number => positive(n) && Number.isSafeInteger(n); +const close = (a: number, b: number) => Math.abs(a - b) <= Math.max(1e-4, Math.abs(a) * 1e-6); + +/** A complete colocated result can legitimately have no separate decode worker. */ +export function isColocatedResult(config: ConfigParams, metrics: Record): boolean { + if ( + !config.disagg || + !config.isMultinode || + !PHYSICAL_TP_FRAMEWORKS.has(config.framework) || + config.numDecodeGpu !== 0 || + config.decodeNumWorkers !== 0 || + !integer(config.numPrefillGpu) || + config.prefillNumWorkers <= 0 + ) + return false; + const total = metrics.tput_per_gpu; + const input = metrics.input_tput_per_gpu; + const output = metrics.output_tput_per_gpu; + return positive(total) && positive(input) && positive(output) && close(total, input + output); +} + +/** + * Repair only established producer contracts. Metrics are never rescaled. + * Explicit non-legacy counts, manual/proprietary deployments and TPUs are retained. + * Stored aggregate rows require AgentX's independent total/per-GPU ratio. + * Fixed-sequence rows lack that independent evidence: repair their count only + * by re-ingesting the source artifact, not by guessing from TP/EP dimensions. + */ +export function correctedBenchmarkConfig( + config: ConfigParams, + benchmarkType: string, + metrics: Record, +): ConfigParams { + if (isColocatedResult(config, metrics)) { + return { + ...config, + disagg: false, + decodeTp: config.prefillTp, + decodeEp: config.prefillEp, + decodeDpAttn: config.prefillDpAttn, + decodeNumWorkers: config.prefillNumWorkers, + numDecodeGpu: config.numPrefillGpu, + }; + } + if ( + config.disagg || + config.isMultinode || + config.hardware.startsWith('tpu') || + !PHYSICAL_TP_FRAMEWORKS.has(config.framework) + ) + return config; + const widths = [ + metrics.pp, + metrics.prefill_pp, + metrics.decode_pp, + metrics.pcp_size, + metrics.prefill_pcp_size, + metrics.decode_pcp_size, + ]; + if (widths.some((width) => width !== undefined && !integer(width))) return config; + const pp = Math.max(metrics.pp ?? 1, metrics.prefill_pp ?? 1, metrics.decode_pp ?? 1); + const pcp = Math.max( + metrics.pcp_size ?? 1, + metrics.prefill_pcp_size ?? 1, + metrics.decode_pcp_size ?? 1, + ); + const expected = config.decodeTp * pp * pcp; + if (!integer(expected)) return config; + if (benchmarkType === 'agentic_traces') { + const total = metrics.total_tput_tps; + const perGpu = metrics.tput_per_gpu; + if (!positive(total) || !positive(perGpu) || !close(total / perGpu, expected)) return config; + } else return config; + // The old mapper's precise fallback is the only persisted count we replace. + const legacy = config.decodeTp * config.decodeEp; + if (config.numPrefillGpu !== legacy || config.numDecodeGpu !== legacy || expected === legacy) { + return config; + } + return { ...config, numPrefillGpu: expected, numDecodeGpu: expected }; +} From 93785bdf688cabddddca242d6e30ce6d61e5c781 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:03:42 -0500 Subject: [PATCH 2/2] fix: stabilize chart assertions and prevent backfill batch starvation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read coach-mark and marker coordinates together in retryable assertions. Count successful updates toward the full-response backfill limit and scan past dateless profiles and no-op updates, with regression coverage. 中文:修复图表断言的时序问题,避免历史数据回填批次被无效记录占满。 在同一次可重试断言中读取提示箭头和数据点坐标;回填的数量限制只统计 实际更新的记录,跳过无法恢复时间戳或无需更新的记录,并补充回归测试。 --- docs/overview-data-integrity.md | 5 +- .../e2e/agentic-point-coach-mark.cy.ts | 29 ++-- ...ckfill-full-response-interactivity.test.ts | 157 ++++++++++++++++++ .../backfill-full-response-interactivity.ts | 33 +++- packages/db/src/lib/backfill-runner.test.ts | 25 +++ packages/db/src/lib/backfill-runner.ts | 14 +- 6 files changed, 239 insertions(+), 24 deletions(-) create mode 100644 packages/db/src/backfill-full-response-interactivity.test.ts diff --git a/docs/overview-data-integrity.md b/docs/overview-data-integrity.md index c002f7a01..30969e8f9 100644 --- a/docs/overview-data-integrity.md +++ b/docs/overview-data-integrity.md @@ -98,7 +98,10 @@ is already committed: rerun the materialized-view refresh before verification. Separately, the existing command `bun run --cwd packages/db db:backfill-full-response-interactivity` fills retained -profile dates. It prompts before writing; `--limit N` permits a small first batch. +profile dates. It prompts before writing; `--limit N` caps rows actually updated, +scanning past profiles with no recoverable metrics or timestamps. Such profiles +remain eligible for a future retry if their retained content changes, but cannot +crowd recoverable rows out of a limited batch. No placeholder dates are written. Do not use `--force` merely to add dates. After either repair, invalidate the website DB cache using the existing admin cache command, then compare the affected raw rows and overview values. The audit and PR preparation do not apply these diff --git a/packages/app/cypress/e2e/agentic-point-coach-mark.cy.ts b/packages/app/cypress/e2e/agentic-point-coach-mark.cy.ts index 57c1b1dd0..99bc4251a 100644 --- a/packages/app/cypress/e2e/agentic-point-coach-mark.cy.ts +++ b/packages/app/cypress/e2e/agentic-point-coach-mark.cy.ts @@ -93,17 +93,19 @@ describe('Agentic point coach mark', () => { // highlight ring sits exactly on the anchor (the pointer line deliberately // stops short so its arrowhead doesn't cover the dot). cy.get('[data-testid="agentic-point-coach-mark-pointer"]').should('exist'); - cy.get('[data-testid="agentic-point-coach-mark-target"]').then(($ring) => { + // D3 transitions and the coach mark's animation-frame callback can still + // be moving. Retry a single, same-frame measurement of both coordinates; + // separate `.then()` commands compare positions from different frames. + cy.get('[data-testid="agentic-point-coach-mark-target"]').should(($ring) => { const tipX = Number($ring.attr('cx')); const tipY = Number($ring.attr('cy')); - cy.get(AGENTIC_MARKERS).then(($points) => { - const hit = [...$points].some((point) => { - const { x, y } = centreOf(point); - return Math.abs(x - tipX) < 1 && Math.abs(y - tipY) < 1; - }); - expect(hit, 'pointer ends on an agentic point').to.eq(true); + const points = $ring[0].ownerDocument.querySelectorAll(AGENTIC_MARKERS); + const hit = [...points].some((point) => { + const { x, y } = centreOf(point); + return Math.abs(x - tipX) < 1 && Math.abs(y - tipY) < 1; }); + expect(hit, 'pointer ends on an agentic point').to.eq(true); }); }); @@ -247,7 +249,7 @@ describe('Agentic point coach mark', () => { ); cy.get(COACH_MARK).should('be.visible'); - cy.get('[data-testid="agentic-point-coach-mark-target"]').then(($ring) => { + cy.get('[data-testid="agentic-point-coach-mark-target"]').should(($ring) => { const tipX = Number($ring.attr('cx')); const tipY = Number($ring.attr('cy')); const onPoint = (element: Element) => { @@ -255,12 +257,11 @@ describe('Agentic point coach mark', () => { return Math.abs(x - tipX) < 1 && Math.abs(y - tipY) < 1; }; - cy.get('[data-testid="scatter-graph"] .unofficial-overlay-pt').then(($overlay) => { - expect([...$overlay].some(onPoint), 'pointer avoids overlay markers').to.eq(false); - }); - cy.get(AGENTIC_MARKERS).then(($official) => { - expect([...$official].some(onPoint), 'pointer lands on an official point').to.eq(true); - }); + const doc = $ring[0].ownerDocument; + const overlay = doc.querySelectorAll('[data-testid="scatter-graph"] .unofficial-overlay-pt'); + const official = doc.querySelectorAll(AGENTIC_MARKERS); + expect([...overlay].some(onPoint), 'pointer avoids overlay markers').to.eq(false); + expect([...official].some(onPoint), 'pointer lands on an official point').to.eq(true); }); }); diff --git a/packages/db/src/backfill-full-response-interactivity.test.ts b/packages/db/src/backfill-full-response-interactivity.test.ts new file mode 100644 index 000000000..86e7c2ce5 --- /dev/null +++ b/packages/db/src/backfill-full-response-interactivity.test.ts @@ -0,0 +1,157 @@ +import { gzipSync } from 'node:zlib'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const state = vi.hoisted(() => ({ + rows: [] as { id: number; profile_export_jsonl_gz: Buffer; has_full_response: boolean }[], + calls: [] as { text: string; values: unknown[] }[], + updated: [] as number[], + noOpIds: [] as number[], + ended: false, + refresh: vi.fn(), +})); + +vi.mock('./etl/db-utils.js', () => ({ + refreshLatestBenchmarks: state.refresh, + createAdminSql: () => + Object.assign( + (strings: TemplateStringsArray, ...values: unknown[]) => { + const text = strings.join('?'); + state.calls.push({ text, values }); + if (text.includes('select br.id')) { + return state.rows.filter((row) => !state.updated.includes(row.id)); + } + if (text.includes('select atr.profile_export_jsonl_gz')) { + return state.rows.filter((row) => row.id === values[0]); + } + if (text.includes('update benchmark_results')) { + const id = values[3] as number; + if (state.noOpIds.includes(id)) return []; + state.updated.push(id); + return [{ id }]; + } + throw new Error(`Unexpected SQL: ${text}`); + }, + { + json: (value: unknown) => value, + end: () => { + state.ended = true; + }, + }, + ), +})); + +const originalArgv = process.argv; +const originalExitCode = process.exitCode; +const startNs = 1786577819e9; + +function profile(withDates: boolean): Buffer { + return gzipSync( + JSON.stringify({ + metadata: { + benchmark_phase: 'profiling', + ...(withDates ? { request_start_ns: startNs, request_end_ns: startNs + 10e9 } : {}), + }, + metrics: { full_response_inter_token_latency: { value: 10, unit: 'ms' } }, + }), + ); +} + +beforeEach(() => { + vi.resetModules(); + state.rows = [1, 2, 3].map((id) => ({ + id, + profile_export_jsonl_gz: profile(id !== 1), + has_full_response: true, + })); + state.calls = []; + state.updated = []; + state.noOpIds = []; + state.ended = false; + state.refresh.mockClear(); + process.argv = ['bun', 'backfill-full-response-interactivity.ts', '--yes', '--limit', '1']; + process.exitCode = undefined; + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); +}); + +afterEach(() => { + process.argv = originalArgv; + process.exitCode = originalExitCode; + vi.restoreAllMocks(); +}); + +async function run() { + await import('./backfill-full-response-interactivity'); + await vi.waitFor(() => expect(state.ended).toBe(true)); +} + +describe('full-response backfill batches', () => { + it('scans past dateless profiles on successive limited runs without writing placeholder dates', async () => { + await run(); + expect(state.updated).toEqual([2]); + expect(console.warn).toHaveBeenCalledWith( + ' id=1: profile has no measurement timestamps, skipping', + ); + const update = state.calls.find((call) => call.text.includes('update benchmark_results'))!; + expect(update.values[1]).toEqual({ + measurement_start_unix_seconds: 1786577819, + measurement_end_unix_seconds: 1786577829, + }); + expect( + state.calls.filter((call) => call.text.includes('select atr.')).map((call) => call.values[0]), + ).toEqual([1, 2]); + + vi.resetModules(); + state.ended = false; + await run(); + expect(state.updated).toEqual([2, 3]); + expect(process.exitCode).toBeUndefined(); + }); + + it('does not let a concurrent no-op update consume the successful-row limit', async () => { + state.rows[0]!.profile_export_jsonl_gz = profile(true); + state.noOpIds = [1]; + await run(); + expect(state.updated).toEqual([2]); + const update = state.calls.find((call) => call.text.includes('update benchmark_results'))!; + expect(update.text).toContain('metrics is distinct from'); + expect(update.text).toContain('returning id'); + }); + + it.each([false, true])( + 'still backfills dateless ITL when recomputation is needed (force=%s)', + async (force) => { + state.rows[0]!.has_full_response = force; + if (force) process.argv.push('--force'); + await run(); + expect(state.updated).toEqual([1]); + const update = state.calls.find((call) => call.text.includes('update benchmark_results'))!; + expect(update.values[2]).toMatchObject({ + median_full_response_itl: 0.01, + median_intvty: 100, + }); + expect(update.values[2]).not.toHaveProperty('measurement_start_unix_seconds'); + }, + ); + + it('scans past unreadable profiles without erasing stored metrics', async () => { + state.rows[0]!.profile_export_jsonl_gz = Buffer.from('invalid gzip'); + await run(); + expect(state.updated).toEqual([2]); + expect(console.warn).toHaveBeenCalledWith( + ' id=1: profile has no usable request samples, skipping', + ); + }); + + it.each(['0', '-1', '1.5', 'Infinity'])( + 'rejects --limit %s before reading or updating rows', + async (limit) => { + process.argv[process.argv.indexOf('--limit') + 1] = limit; + await run(); + expect(state.calls).toHaveLength(0); + expect(state.refresh).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + }, + ); +}); diff --git a/packages/db/src/backfill-full-response-interactivity.ts b/packages/db/src/backfill-full-response-interactivity.ts index 9af298f69..c407c3edc 100644 --- a/packages/db/src/backfill-full-response-interactivity.ts +++ b/packages/db/src/backfill-full-response-interactivity.ts @@ -7,7 +7,7 @@ * * Usage: * bun run --cwd packages/db db:backfill-full-response-interactivity - * [--limit N] only process the first N candidate benchmark rows + * [--limit N] update at most N benchmark rows; skipped profiles do not count * [--force] recompute rows that already have the namespaced metric * [--yes] skip the confirmation prompt */ @@ -26,6 +26,9 @@ const flags = parseLimitForceFlags(); const sql = createAdminSql({ noSsl: hasNoSslFlag(), max: 1, onnotice: () => {} }); async function main(): Promise { + if (flags.limit !== null && (!Number.isSafeInteger(flags.limit) || flags.limit < 1)) { + throw new Error('--limit requires a positive integer'); + } console.log('=== backfill-full-response-interactivity ==='); console.log(` force = ${flags.force}`); console.log(` limit = ${flags.limit ?? 'none'}`); @@ -40,7 +43,6 @@ async function main(): Promise { where br.benchmark_type = 'agentic_traces' and atr.profile_export_jsonl_gz is not null order by br.id - ${flags.limit ? sql`limit ${flags.limit}` : sql``} ` : await sql<{ id: number }[]>` select br.id @@ -52,13 +54,15 @@ async function main(): Promise { or not (br.metrics ? 'measurement_start_unix_seconds') or not (br.metrics ? 'measurement_end_unix_seconds')) order by br.id - ${flags.limit ? sql`limit ${flags.limit}` : sql``} `; return candidates.map((candidate) => candidate.id); }, async (id) => { - const [row] = await sql<{ profile_export_jsonl_gz: Buffer | null }[]>` - select atr.profile_export_jsonl_gz + const [row] = await sql< + { profile_export_jsonl_gz: Buffer | null; has_full_response: boolean }[] + >` + select atr.profile_export_jsonl_gz, + br.metrics ? 'median_full_response_itl' as has_full_response from benchmark_results br join agentic_trace_replay atr on atr.id = br.trace_replay_id where br.id = ${id} @@ -79,17 +83,32 @@ async function main(): Promise { key === 'measurement_start_unix_seconds' || key === 'measurement_end_unix_seconds', ), ); + if (!flags.force && row.has_full_response && Object.keys(dates).length === 0) { + console.warn(` id=${id}: profile has no measurement timestamps, skipping`); + return 'skipped'; + } - await sql` + const changed = await sql` update benchmark_results set metrics = case when not ${flags.force} and metrics ? 'median_full_response_itl' then ${jsonbParam(sql, dates)} || metrics else metrics || ${jsonbParam(sql, patch)} end where id = ${id} + and metrics is distinct from + case when not ${flags.force} and metrics ? 'median_full_response_itl' + then ${jsonbParam(sql, dates)} || metrics + else metrics || ${jsonbParam(sql, patch)} end + returning id `; + if (changed.length === 0) { + console.warn(` id=${id}: no new metrics to store, skipping`); + return 'skipped'; + } return 'ok'; }, - (count) => `${count} candidate benchmark row(s).`, + (count) => + `${count} candidate benchmark row(s).${flags.limit === null ? '' : ` Scan until ${flags.limit} row(s) are updated; skipped profiles do not count.`}`, + flags.limit ?? undefined, ); if (processedCandidates && process.exitCode !== 1) await refreshLatestBenchmarks(sql); diff --git a/packages/db/src/lib/backfill-runner.test.ts b/packages/db/src/lib/backfill-runner.test.ts index 3beb2053a..a53cd69ee 100644 --- a/packages/db/src/lib/backfill-runner.test.ts +++ b/packages/db/src/lib/backfill-runner.test.ts @@ -151,6 +151,31 @@ describe('runPerIdBackfill', () => { expect(logged.at(-1)).toContain('=== backfill complete: 1 ok, 1 failed'); expect(vi.mocked(console.error).mock.calls[0]?.[0]).toContain('✗ id=1: boom'); }); + + it('limits successful updates without letting skipped or failed rows consume the batch', async () => { + const seen: number[] = []; + await runPerIdBackfill( + [1, 2, 3, 4, 5], + (id) => { + seen.push(id); + if (id === 2) return Promise.reject(new Error('unreadable profile')); + return Promise.resolve(id === 1 ? 'skipped' : 'ok'); + }, + 2, + ); + expect(seen).toEqual([1, 2, 3, 4]); + expect(process.exitCode).toBe(1); + expect(console.log).toHaveBeenLastCalledWith(expect.stringContaining('2 ok, 1 failed')); + }); + + it.each([0, -1, 1.5, Infinity, NaN])( + 'rejects an invalid successful-row limit %s before processing', + async (limit) => { + const processRow = vi.fn(() => Promise.resolve('ok' as const)); + await expect(runPerIdBackfill([1], processRow, limit)).rejects.toThrow('positive integer'); + expect(processRow).not.toHaveBeenCalled(); + }, + ); }); describe('runCandidateIdBackfill', () => { diff --git a/packages/db/src/lib/backfill-runner.ts b/packages/db/src/lib/backfill-runner.ts index 876e559b2..639d59be0 100644 --- a/packages/db/src/lib/backfill-runner.ts +++ b/packages/db/src/lib/backfill-runner.ts @@ -111,13 +111,21 @@ export async function confirmProceed(candidatesLabel: string): Promise * hundreds of MB decompressed — serial processing keeps memory bounded), * logging per-row progress and a final summary. `processRow` returns 'ok' * (counts toward the ✓ log) or 'skipped' (e.g. row vanished — the callback - * logs its own warning); throwing marks the row failed. Sets + * logs its own warning); throwing marks the row failed. An optional successful-row + * limit scans past skipped/failed ids so they cannot starve later candidates. Sets * `process.exitCode = 1` when any row failed. */ export async function runPerIdBackfill( ids: readonly number[], processRow: (id: number) => Promise<'ok' | 'skipped'>, + maxSuccessfulRows?: number, ): Promise { + if ( + maxSuccessfulRows !== undefined && + (!Number.isSafeInteger(maxSuccessfulRows) || maxSuccessfulRows < 1) + ) { + throw new Error('The successful-row limit must be a positive integer'); + } let ok = 0; let failed = 0; const t0 = Date.now(); @@ -129,6 +137,7 @@ export async function runPerIdBackfill( const elapsed = Math.round((Date.now() - start) / 1000); const elapsedTotal = Math.round((Date.now() - t0) / 1000); console.log(` ✓ id=${id} (${elapsed}s, ${ok}/${ids.length} done, ${elapsedTotal}s total)`); + if (maxSuccessfulRows !== undefined && ok >= maxSuccessfulRows) break; } catch (error) { failed++; console.error(` ✗ id=${id}: ${error instanceof Error ? error.message : String(error)}`); @@ -149,6 +158,7 @@ export async function runCandidateIdBackfill( loadCandidateIds: () => Promise, processRow: (id: number) => Promise<'ok' | 'skipped'>, formatCandidates: (count: number) => string = (count) => `${count} candidate row(s).`, + maxSuccessfulRows?: number, ): Promise { const ids = await loadCandidateIds(); if (ids.length === 0) { @@ -156,7 +166,7 @@ export async function runCandidateIdBackfill( return false; } if (!(await confirmProceed(formatCandidates(ids.length)))) return false; - await runPerIdBackfill(ids, processRow); + await runPerIdBackfill(ids, processRow, maxSuccessfulRows); return true; }