perf(router): fuse AST cloning and provenance construction

.changeset/spicy-keys-throw.md

@@ -0,0 +1,6 @@
+---
+'@tanstack/router-plugin': patch
+'@tanstack/router-utils': patch
+---
+
+Fuse AST cloning with provenance construction to reduce compiler copying and traversal work while preserving independent output trees.

benchmarks/compiler/README.md

@@ -468,3 +468,31 @@ wins justify both changes. Results retain the same source hashes, iteration
 counts, output counts and generated-byte totals; see `*-yuku-after.json`.
 The destructuring table deliberately omits the earlier cold Babel number to
 avoid comparing different warmup/iteration configurations.
+
+## Fused AST cloning
+
+See the [fused cloning evaluation](fused-cloning.md) for final measurements,
+validation, attribution, and the peak-memory limitation.
+
+`clone.mjs` measures cloning independently of analysis and printing. Build the
+selected checkout's router-utils through Nx first, then run each case in a fresh
+process with the same Node version and parameters in both checkouts:
+
+```sh
+node --expose-gc benchmarks/compiler/clone.mjs --workspace /path/to/checkout --case route
+```
+
+Cases are `route`, `wide` (1,000 declarations), `literals` (comments, regexes,
+BigInts, and array elisions), `deep` (200 unary expressions), `identifier`,
+`generated` (a CSS-loading fragment), and `shared` (aliased child references).
+Each process checks copied fields, node isolation, module provenance, and module
+code/maps before timing. Generated-reference semantics are covered by unit tests.
+The default is 100 samples of 20 clones after 200 warmup clones. Results include
+raw batch samples, mean, standard deviation, median, p99, peak process RSS,
+post-GC heap delta, output digest, and source/built compiler fingerprints.
+
+For this native-to-native optimization use `measure.mjs --modes yuku-splitter`
+in **both** workspaces, with identical `--fixtures-root`, warmups and iterations.
+The older `compare.mjs` intentionally selects Babel for its baseline and must
+not be used unchanged. Use separate `--digest-output` runs to verify emitted
+code and source maps; alternate baseline/candidate ordering for timing runs.

benchmarks/compiler/clone.mjs

@@ -0,0 +1,148 @@
+import assert from 'node:assert/strict'
+import { createHash } from 'node:crypto'
+import { createRequire } from 'node:module'
+import path from 'node:path'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+import { compilerFingerprint } from './provenance.mjs'
+
+// Run in a fresh process after building the selected workspace through Nx.
+// Analysis and correctness checks are outside the timed repeated-copy workload.
+const args = process.argv.slice(2)
+const option = (name, fallback) => {
+  const index = args.indexOf(name)
+  return index < 0 ? fallback : args[index + 1]
+}
+const workspace = path.resolve(
+  option('--workspace', fileURLToPath(new URL('../../', import.meta.url))),
+)
+const selected = option('--case', 'route')
+const samples = Number(option('--samples', '100'))
+const batch = Number(option('--batch', '20'))
+assert.ok(Number.isSafeInteger(samples) && samples > 0)
+assert.ok(Number.isSafeInteger(batch) && batch > 0)
+const utils = await import(
+  pathToFileURL(path.join(workspace, 'packages/router-utils/dist/esm/index.js'))
+)
+const require = createRequire(
+  path.join(workspace, 'packages/router-utils/package.json'),
+)
+const { b, walk } = await import(pathToFileURL(require.resolve('yuku-ast')))
+const sources = {
+  route: `import { createFileRoute } from '@tanstack/react-router';
+    const state = { count: 0 }; const read = () => ++state.count;
+    export const Route = createFileRoute('/')({ loader: read,
+      component: () => <button onClick={read}>{state.count}</button> });`,
+  wide: Array.from(
+    { length: 1000 },
+    (_, i) => `export const value${i} = { index: ${i}, values: [1, 2, 3] };`,
+  ).join('\n'),
+  literals: Array.from(
+    { length: 200 },
+    (_, i) =>
+      `/* comment ${i} */ export const value${i} = [/ab+/gi, 123n, null, 'é😀', , true];`,
+  ).join('\n'),
+  deep: `export const value = ${'!'.repeat(200)}external;`,
+}
+let operation
+const digest = createHash('sha256')
+if (selected in sources) {
+  const source = sources[selected]
+  const module = utils.analyzeModule({ code: source, filename: 'clone.tsx' })
+  const output = utils.cloneModuleAst(module)
+  assert.deepEqual(output.program, module.ast)
+  const originals = []
+  walk(module.ast, {
+    enter(node) {
+      originals.push(node)
+    },
+  })
+  let index = 0
+  walk(output.program, {
+    enter(node) {
+      const original = originals[index++]
+      assert.notEqual(node, original)
+      assert.equal(output.originalNodes.get(node), original)
+      if (output.copiedNodes) {
+        assert.equal(output.copiedNodes.get(original), node)
+      }
+    },
+  })
+  const options = { source, filename: 'clone.tsx' }
+  assert.deepEqual(
+    utils.generateModule(output.program, options),
+    utils.generateModule(module.ast, options),
+  )
+  digest.update(JSON.stringify(utils.generateModule(output.program, options)))
+  operation = () => utils.cloneModuleAst(module).program.body.length
+} else {
+  const reference = utils.linkGeneratedReference(
+    b.Identifier({ name: 'loadCss' }),
+    'loadCss',
+  )
+  const fragments = {
+    identifier: reference,
+    generated: utils.parseExpression(
+      'Promise.all([loadCss("/a.css"), loadCss("/b.css")]).then(() => render(value))',
+    ),
+    shared: b.BinaryExpression({
+      operator: '+',
+      left: reference,
+      right: reference,
+    }),
+  }
+  assert.ok(selected in fragments, `Unknown case: ${selected}`)
+  const fragment = fragments[selected]
+  const output = utils.cloneGeneratedNode(fragment)
+  assert.deepEqual(output, fragment)
+  assert.notEqual(output, fragment)
+  if (selected === 'shared') {
+    assert.equal(output.left, output.right)
+  }
+  digest.update(JSON.stringify(output))
+  operation = () => utils.cloneGeneratedNode(fragment).start
+}
+let consumed = 0
+for (let i = 0; i < batch * 10; i++) {
+  consumed += operation()
+}
+globalThis.gc?.()
+const before = process.memoryUsage()
+const milliseconds = []
+for (let i = 0; i < samples; i++) {
+  const start = performance.now()
+  for (let j = 0; j < batch; j++) {
+    consumed += operation()
+  }
+  milliseconds.push((performance.now() - start) / batch)
+}
+globalThis.gc?.()
+const after = process.memoryUsage()
+const sorted = [...milliseconds].sort((a, b) => a - b)
+const mean = milliseconds.reduce((sum, value) => sum + value, 0) / samples
+const standardDeviation = Math.sqrt(
+  milliseconds.reduce((sum, value) => sum + (value - mean) ** 2, 0) /
+    Math.max(1, samples - 1),
+)
+console.log(
+  JSON.stringify(
+    {
+      case: selected,
+      node: process.version,
+      workspace,
+      compiler: compilerFingerprint(workspace),
+      samples,
+      batch,
+      meanMs: mean,
+      standardDeviationMs: standardDeviation,
+      medianMs: sorted[Math.floor(samples / 2)],
+      p99Ms: sorted[Math.min(samples - 1, Math.floor(samples * 0.99))],
+      peakRssKiB: process.resourceUsage().maxRSS,
+      retainedHeapDelta: after.heapUsed - before.heapUsed,
+      outputSha256: digest.digest('hex'),
+      consumed,
+      milliseconds,
+    },
+    null,
+    2,
+  ),
+)

benchmarks/compiler/fused-cloning.md

@@ -0,0 +1,155 @@
+# Fused AST cloning and provenance
+
+This change is stacked on PR #8504 at
+`9594d49eaa19911d02fe25088897cffe4ccc02a9`. The baseline PR is unchanged.
+
+## Implementation
+
+`router-utils` copies AST fields and constructs provenance in one iterative
+traversal. Each output remains independently mutable. The copying lookup
+preserves shared objects within an output, including node aliases, arrays,
+and comments; native RegExp values are copied explicitly. Primitive values,
+BigInt, array holes, comments, source spans, and generated symbol/name references
+are preserved. The work stack avoids a recursive clone depth limit.
+
+The splitter consumes the source-to-copy lookup directly instead of walking the
+copied program to reconstruct it. The lookup is local to one output and exposes
+only `get`, because its private bookkeeping includes non-node AST records.
+Copied-node-to-source provenance remains a WeakMap. There is no global cache,
+shared mutable output, parser change, or selective-copy path.
+
+## Correctness and workflow
+
+The original implementation failed the public-builder 12,000-level generated
+fragment test in `structuredClone`; the fused copier passes it. This establishes
+clone depth handling, not unlimited parser or printer depth. Native-field,
+source-map, generated-reference, alias, nested-mutation, sparse-array, numeric
+value, and shadowing coverage also passes. React and Solid splitter fixtures
+exercise shared RegExp/BigInt values in every grouping; 26 snapshots were
+reviewed, including shared initializer ownership and empty shared outputs when
+all consumers belong to one chunk.
+
+The repository's bundle-size skill supplied five independent read-only reviews.
+Their additional test/performance cases were incorporated. Tests and implementation
+remained uncommitted during measurement. A separate baseline worktree kept the
+before implementation intact while measurement variants were evaluated.
+
+## Measurements
+
+Final CPU measurements are below. Exploratory
+WeakMap and property-enumeration variants are retained separately in
+`results/fused-clone-weakmap-*.json` and
+`fused-clone-enumeration-*.json`; they are not final acceptance results.
+
+The property-enumeration variant removed temporary key arrays but slowed the
+splitter, so it was rejected. An output-local Map performed better than a
+WeakMap for the forward copying lookup and reduced observed peak memory in the
+follow-up samples. Neither variation changes node ownership or emitted code.
+
+Measurements use Node 24.8.0, pnpm 11.21.0, and Yuku 0.12.0 on macOS arm64.
+Timing runs are serialized, with no concurrent tests/builds from this task.
+Five fresh process pairs alternate baseline/candidate order. Compiler speedups
+are workload-specific; percentages from separate workloads must not be added.
+Peak RSS is the process high-water mark, not isolated allocation accounting or
+retained memory. GC/allocator effects and run variation limit small memory claims.
+
+## Reproduction
+
+Install each checkout at its root with `CI=1 pnpm install --frozen-lockfile`.
+Build workspace packages through Nx before using the standalone compiler harnesses.
+Use the candidate fixture root for both checkouts so they read identical inputs.
+
+```sh
+CI=1 NX_DAEMON=false pnpm nx run @tanstack/router-plugin:build --outputStyle=stream --skipRemoteCache
+CI=1 NX_DAEMON=false pnpm nx run @tanstack/start-plugin-core:build --outputStyle=stream --skipRemoteCache
+node --expose-gc benchmarks/compiler/clone.mjs --workspace /path/to/checkout --case route
+node benchmarks/compiler/measure.mjs --workspace /path/to/checkout --fixtures-root /path/to/candidate --modes yuku-splitter --repetitions 1 --iterations 10 --warmups 2
+node benchmarks/compiler/measure.mjs --workspace /path/to/checkout --modes yuku-splitter --bindings 1000 --repetitions 1 --iterations 5 --warmups 2
+node benchmarks/compiler/route-heavy.mjs --workspace /path/to/checkout --routes 250 --repetitions 1
+node benchmarks/compiler/hydrate-scaling.mjs --workspace /path/to/checkout --sizes 100,500 --env client --repetitions 1 --iterations 5
+```
+
+Repeat the commands in alternating AB/BA order five times. Direct clone cases
+are `route`, `wide`, `literals`, `deep`, `identifier`, `generated`, and `shared`.
+Code/map digest checks use a separate splitter run with `--digest-output
+--repetitions 1 --iterations 1 --warmups 0`; diagnostic timing is excluded from
+speed claims. The older Babel comparison driver is not used for this
+native-to-native comparison.
+
+## Final BEFORE / AFTER CPU and memory results
+
+| Workload                                | Median ms, before → after | Time reduction | Timing CV, before / after | Median peak RSS MiB, before → after |
+| --------------------------------------- | ------------------------: | -------------: | ------------------------: | ----------------------------------: |
+| 56-fixture / 567-output splitter corpus |            101.91 → 64.27 |          36.9% |               5.2% / 4.5% |                       211.4 → 271.2 |
+| 1,000 shared bindings                   |           185.95 → 123.61 |          33.5% |               2.6% / 1.9% |                       260.8 → 267.8 |
+| 250-route production Vite build         |          1094.26 → 947.55 |          13.4% |               3.1% / 5.1% |                       647.8 → 649.2 |
+| 100-boundary Start Hydrate compilation  |             13.94 → 12.36 |          11.3% |               7.2% / 5.7% |                         98.8 → 97.6 |
+| 500-boundary Start Hydrate compilation  |             51.80 → 45.68 |          11.8% |               9.0% / 5.5% |                       211.2 → 211.8 |
+
+[Raw alternating compiler samples](results/fused-clone-compiler.json) include every process, input hash, source/built fingerprint, timing, and memory sample. Splitter runs use two warmups and ten corpus passes; dense runs use two warmups and five passes. Hydrate runs use the harness's one warmup and five compiles. All source and executable fingerprints remain stable within each variant.
+
+Peak RSS does **not** consistently improve. The final splitter workload has higher peak RSS despite essentially unchanged post-GC retained heap (about 8.2–8.3 MiB). Dense compilation also rises slightly; the production build and Hydrate peaks are nearly unchanged. Earlier exploratory samples varied substantially. This PR is justified by lower CPU cost; it does not establish lower compiler memory usage.
+
+| Direct cloning case | Median process mean, µs before → after | Time reduction |
+| ------------------- | -------------------------------------: | -------------: |
+| route               |                          43.90 → 13.60 |          69.0% |
+| wide                |                     10903.20 → 4401.94 |          59.6% |
+| literals            |                       1623.32 → 626.58 |          61.4% |
+| deep                |                         128.26 → 40.22 |          68.6% |
+| identifier          |                            1.81 → 0.38 |          78.9% |
+| generated           |                           15.10 → 5.67 |          62.5% |
+| shared              |                            3.39 → 0.59 |          82.5% |
+
+[Raw focused samples](results/fused-clone-focused.json) retain all 100 batch timings per process, standard deviation, and p99. These repeated-clone timings exclude parsing/printing. Tiny fragment results are batched to reduce timer overhead and are not whole-application speed claims.
+
+## Attribution and bundle pass
+
+The production diff has two dependent groups: the shared fused copier (including
+generated fragments), and consuming its lookup in `createOutput`. The copier
+alone is valid with the old splitter indexing walk, so that combination was
+measured separately. With the final 56-fixture corpus, its five-process median
+was 72.06 ms (71.19–75.76 ms), versus 64.27 ms for the complete change in the
+alternating run. Both improve on the 101.91 ms baseline. These separate-run
+figures attribute removed work; their percentages must not be added together.
+See [copy-only raw samples](results/fused-clone-attribution-copy-only.json).
+
+All 18 scenarios were rebuilt with source attribution for the baseline,
+copy-only implementation, and final composed implementation. Every scenario
+has **0 bytes delta** in gzip, initial gzip, raw, initial raw, Brotli, and initial
+Brotli, with identical chunk lists and per-file byte metrics. All 38 final
+emitted client JavaScript files are byte-identical to baseline. The compiler
+implementation is build-time code; changing its copying mechanism changes no
+client output. No output-specific minification or annotation changes are needed.
+
+- [Baseline metrics and source attribution](results/fused-clone-bundle-before.json)
+- [Copy-only metrics and source attribution](results/fused-clone-bundle-copy-only.json)
+- [Final metrics and source attribution](results/fused-clone-bundle-after.json)
+- [Baseline client JS hashes](results/fused-clone-bundle-hashes-before.json)
+- [Final client JS hashes](results/fused-clone-bundle-hashes-after.json)
+
+All 567 splitter code/map outputs match across baseline, copy-only, and final,
+with combined digest
+`d2a65814695b37df7e700b343e1172d9523f30800334de79dfef6ac5653d6e95`.
+The final rebuilt source and executable compiler fingerprints match every final
+candidate timing sample. Digest records: [before](results/fused-clone-digest-before.json),
+[copy-only](results/fused-clone-digest-copy-only.json),
+and [after](results/fused-clone-digest-after.json).
+Exploratory variant measurements used the earlier 54-fixture corpus; compare
+variants only with their matching inputs, not absolute numbers across corpora.
+
+## Final validation
+
+- `router-utils`, `router-plugin`, `router-generator`, `start-plugin-core`, and
+  `react-start-rsc`: unit tests, TypeScript 5.6–7 matrix, ESLint, and package
+  build/export checks passed through Nx. `solid-start` unit and build checks
+  also passed (it has no corresponding type/lint scripts).
+- Final unit counts: router-utils 59, router-plugin 734, router-generator 261
+  runtime plus 12 fixture/type tests, start-plugin-core 584, react-start-rsc 61,
+  and solid-start 4. Router-utils and router-plugin were rerun during the final
+  bundle workflow after adding the fixtures.
+- Production browser checks: React code splitting 6, Solid code splitting 5,
+  Vue JSX routes 22, Start server functions with Vite 53 and Rsbuild 53: **139
+  passed**. The shared Start compiler unit suites also cover both adapters,
+  import protection, and deferred hydration.
+- `pnpm format` and `git diff --check` passed. No dependency or lockfile changes.
+- A patch changeset covers `router-utils` and `router-plugin`.