Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/android.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name: Android

on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths-ignore:
- 'docs/**'
- 'website/**'
Expand Down Expand Up @@ -30,6 +31,7 @@ concurrency:
jobs:
smoke-android:
name: Smoke Tests
if: ${{ !github.event.pull_request.draft }}
runs-on: ubuntu-latest
timeout-minutes: 80
steps:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/ios.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name: iOS

on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths-ignore:
- 'docs/**'
- 'website/**'
Expand Down Expand Up @@ -61,6 +62,7 @@ concurrency:
jobs:
smoke-ios:
name: Smoke Tests
if: ${{ !github.event.pull_request.draft }}
runs-on: macos-26
timeout-minutes: 80
env:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name: Linux

on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths-ignore:
- 'docs/**'
- 'website/**'
Expand Down Expand Up @@ -29,6 +30,7 @@ concurrency:
jobs:
smoke-linux:
name: Smoke Tests
if: ${{ !github.event.pull_request.draft }}
runs-on: ubuntu-latest
timeout-minutes: 30
env:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name: macOS

on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths-ignore:
- 'docs/**'
- 'website/**'
Expand Down Expand Up @@ -49,6 +50,7 @@ jobs:
# goes.
smoke-macos:
name: Smoke Tests
if: ${{ !github.event.pull_request.draft }}
runs-on: macos-26
timeout-minutes: 80
env:
Expand Down
28 changes: 20 additions & 8 deletions .github/workflows/mutation-affected.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,12 @@ name: Mutation Harness Canary
# Cost control: the kernel report that pays is the weekly sweep, so this lane
# spends mutants on one thing only — proving the harness still runs end to end
# when the harness itself changes. `select` therefore returns an empty matrix
# unless the diff touches the lane's own sources. Selecting on derived kernel
# ownership instead would run the full sweep on 24 of the last 40
# merged PRs.
# unless the diff touches the lane's own sources (LANE_SOURCES). Selecting on
# derived kernel ownership instead would run the full sweep on 24 of the last 40
# merged PRs. A diff confined to LANE_DEPENDENCIES runs the self-test only; most
# such diffs are unrelated workspace edits. Dispatch on the branch to spend the
# canary (pass -f base=<parent> for a stacked branch):
# gh workflow run mutation-affected.yml --ref <branch>
#
# On a harness diff the matrix is the canary (`LANE_CANARY`, selected by
# observed end-to-end runtime within the job budget — lane sources own no kernel,
Expand All @@ -22,8 +25,7 @@ name: Mutation Harness Canary
on:
pull_request:
paths:
# Exactly `LANE_TOOLING` in scripts/mutation/run.ts — the
# only diffs that can produce a non-empty matrix, asserted both ways by
# Exactly `LANE_TOOLING` in scripts/mutation/run.ts, asserted both ways by
# scripts/mutation/workflow.test.ts. A kernel or test path here would only
# start a job that selects nothing.
- 'scripts/mutation/**'
Expand All @@ -36,6 +38,11 @@ on:
- 'package.json'
- 'pnpm-lock.yaml'
- 'packages/*/package.json'
workflow_dispatch:
inputs:
base:
description: Branch the diff is measured against
default: main

permissions:
contents: read
Expand All @@ -44,6 +51,10 @@ concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
MUTATION_BASE: origin/${{ github.event.pull_request.base.ref || inputs.base }}
Comment thread
thymikee marked this conversation as resolved.
MUTATION_SCOPE: ${{ github.event_name == 'workflow_dispatch' && '--with-dependencies' || '' }}

jobs:
select:
name: Select harness validation kernels
Expand Down Expand Up @@ -72,7 +83,8 @@ jobs:
args: |
--list-affected
--base
origin/${{ github.event.pull_request.base.ref }}
${{ env.MUTATION_BASE }}
${{ env.MUTATION_SCOPE }}

# The self-test and the derivation run before any mutant, so a failure here
# would leave the lane with no envelope at all (#1430).
Expand Down Expand Up @@ -170,9 +182,9 @@ jobs:
expected=$(echo '${{ needs.select.outputs.modules }}' | jq length)
pnpm gate mutation-check --report-dir .tmp/mutation/shards --affected \
--expect-shards "$expected" \
--base "origin/${{ github.event.pull_request.base.ref }}"
--base "$MUTATION_BASE" $MUTATION_SCOPE
else
pnpm gate mutation-affected --base "origin/${{ github.event.pull_request.base.ref }}"
pnpm gate mutation-affected --base "$MUTATION_BASE" $MUTATION_SCOPE
fi

- name: Record a failed lane envelope
Expand Down
8 changes: 5 additions & 3 deletions docs/agents/pull-requests.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,23 @@ reported. If a gate fails, diagnose it and rerun after the cause is resolved; re
failures without claiming validation passed. Rerun a failed CI job once only when its log shows an
infrastructure fault (runner startup, network, device boot); the same failure twice is a finding.

Push once per review round, not per fix: each push restarts CI. Drafts skip device lanes.

- **Published**: branch pushed, PR opened, and validation recorded at a named commit.
- **Merge-ready**: required checks pass on the actual head, with live evidence for device-facing
changes. Docs-only and pure-tooling changes do not need device runs. Reporting publication does
not require waiting for CI.

A local unit run does not establish provider integration or coverage. When those checks are
selected, verify the **Integration Tests** and **Coverage** jobs on the PR head. Run
`pnpm check:fallow --base origin/main` when code-quality or dead-code risk warrants it.
selected, verify the **Integration Tests** and **Coverage** jobs on the PR head.

For device-facing changes, fixtures do not replace a live run whose output proves it reached the
route the change adds or modifies; a neighbouring route is not evidence. If blocked, record the
command and device needed and the remaining risk. Close manual sessions using
`docs/agents/device-verification.md` and report incomplete cleanup.

Gross diff budget: 1,000 lines by `git diff --stat origin/main...HEAD`. Rename-only move PRs titled
Gross diff budget: 1,000 lines by `git diff --stat origin/main...HEAD`; 2,500 for one coherent change
in self-contained commits. Stack only layers that land independently. Rename-only move PRs titled
`refactor(move)` are exempt when `git diff -M90% --stat origin/main...HEAD` proves no material
content change.

Expand Down
7 changes: 3 additions & 4 deletions docs/agents/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,8 @@ worker process, so a case that never returns or kills its process is reported as
against the exact input. Promote a discovered case with the command the harness prints — never
hand-copy an unshrunk input.

Mutation scope: `scripts/mutation/modules.ts#KERNEL_MODULES` (weekly), `scripts/mutation/run.ts#LANE_TOOLING`
(PR canary). Scores are advisory; incomplete sweeps fail. Pilot protocol: `pnpm mutation:run --help`.
Mutation scope: `scripts/mutation/modules.ts#KERNEL_MODULES` (weekly), `scripts/mutation/run.ts#LANE_SOURCES`
(PR canary; dispatch adds `LANE_DEPENDENCIES`). Scores are advisory; incomplete sweeps fail. Pilot protocol: `pnpm mutation:run --help`.

## Before editing a shared module

Expand All @@ -145,8 +145,7 @@ selects gates independently from the diff; it does not consume the depgraph repo
action with a literal gate id; `pnpm check:gate-manifest` verifies registration, workflow ownership,
path reachability, and routed device lanes. Raw shell text cannot declare ownership.

New check: update the catalog and its executable model. Changed path ownership: plant a path that
would previously be misrouted and watch the selector or manifest fail before fixing it.
New check: update the catalog and its executable model.

## Concurrency torture lane

Expand Down
35 changes: 35 additions & 0 deletions scripts/gate/workflows.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -291,3 +291,38 @@ on:
});
assert.equal(lanes.find((lane) => lane.workflow === 'planted.yml')?.unsupported.length, 1);
});

// Drafts are where agents iterate; a smoke lane that runs on them spends device
// runners on heads nobody is reviewing, and one that misses `ready_for_review`
// never runs on a draft marked ready until its next push. `converted_to_draft`
// starts a skipped run whose concurrency group cancels the one in flight.
test('device smoke lanes skip drafts and start when a draft is marked ready', () => {
const directory = path.join(repoRoot, '.github/workflows');
type Workflow = {
name: string;
on: { pull_request?: { types?: string[] } };
jobs: Record<string, { name?: string; if?: string }>;
};
const smokeLanes = fs
.readdirSync(directory)
.map((file) => parse(fs.readFileSync(path.join(directory, file), 'utf8')) as Workflow)
.flatMap((workflow) =>
Object.values(workflow.jobs)
.filter((job) => job.name === 'Smoke Tests' && workflow.on.pull_request)
.map((job) => ({ workflow, job })),
);
assert.deepEqual(smokeLanes.map(({ workflow }) => workflow.name).sort(), [
'Android',
'Linux',
'iOS',
'macOS',
]);
for (const { workflow, job } of smokeLanes) {
assert.deepEqual(
workflow.on.pull_request?.types,
['opened', 'synchronize', 'reopened', 'ready_for_review', 'converted_to_draft'],
workflow.name,
);
assert.equal(job.if, '${{ !github.event.pull_request.draft }}', workflow.name);
}
});
51 changes: 32 additions & 19 deletions scripts/mutation/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,10 @@ const USAGE = `Usage: pnpm mutation:run [options]
--summary <file> Also write the markdown report to <file>
--no-run Alias for --report with the default report path
--list-affected Print the PR lane's shard matrix as JSON and exit (empty
unless the diff touches the lane's own tooling)
unless the diff touches the lane's own sources)
--with-dependencies
Also select the canary for a diff touching only the lane's
dependency manifests
--fail-envelope <reason>
Write a failed lane envelope for a step that ran before (or
instead of) the sweep, e.g. a self-test failure
Expand Down Expand Up @@ -114,6 +117,7 @@ type Args = {
reportDir: string | undefined;
summary: string | undefined;
listAffected: boolean;
withDependencies: boolean;
failEnvelope: string | undefined;
shard: Shard | undefined;
expectShards: number | undefined;
Expand Down Expand Up @@ -159,6 +163,7 @@ function parseMutationArgs(argv: readonly string[]): Args {
summary: { type: 'string' },
'no-run': { type: 'boolean', default: false },
'list-affected': { type: 'boolean', default: false },
'with-dependencies': { type: 'boolean', default: false },
'fail-envelope': { type: 'string' },
shard: { type: 'string' },
'expect-shards': { type: 'string' },
Expand All @@ -175,6 +180,7 @@ function parseMutationArgs(argv: readonly string[]): Args {
reportDir: values['report-dir'],
summary: values.summary,
listAffected: Boolean(values['list-affected']),
withDependencies: Boolean(values['with-dependencies']),
failEnvelope: values['fail-envelope'],
shard: parseShard(values.shard),
expectShards: values['expect-shards'] ? Number(values['expect-shards']) : undefined,
Expand Down Expand Up @@ -605,7 +611,7 @@ async function sweep(args: Args, state: LaneState): Promise<number> {
// Same selection the PR matrix uses, so the reporting job can never score
// mutants the `select` job decided not to spend.
if (args.affected) {
state.modules = [...new Set(affectedMatrix(args.base).map((entry) => entry.module))];
state.modules = [...new Set(affectedMatrix(args).map((entry) => entry.module))];
}
if (state.modules.length === 0) {
process.stdout.write('mutation: no decision-kernel modules affected — nothing to mutate.\n');
Expand Down Expand Up @@ -645,26 +651,32 @@ async function sweep(args: Args, state: LaneState): Promise<number> {
return 0;
}

/**
* Sources of the lane itself: a change here must prove itself on real mutants.
* These are also the only paths that can produce a non-empty matrix, so
* `mutation-affected.yml` triggers on exactly them — asserted by
* `workflow.test.ts`, since a filter that missed one would let a harness change
* merge unproven, and any wider filter only buys a no-op job.
*/
export const LANE_TOOLING = [
/** Code and config the lane executes: a change here proves itself on real mutants on every PR. */
const LANE_SOURCES = [
'scripts/mutation/',
'scripts/lib/',
'stryker.config.json',
'vitest.mutation.config.ts',
'vitest.config.ts',
'.github/workflows/mutation-weekly.yml',
'.github/workflows/mutation-affected.yml',
'package.json',
'pnpm-lock.yaml',
'packages/*/package.json',
];

/**
* Manifests the lane resolves through. Unrelated PRs touch these routinely (a
* workspace export, a root script), so a diff confined to them spends mutants
* only under `--with-dependencies`, which the workflow's manual dispatch passes.
*/
const LANE_DEPENDENCIES = ['package.json', 'pnpm-lock.yaml', 'packages/*/package.json'];

/**
* `mutation-affected.yml` triggers on exactly these, asserted by
* `workflow.test.ts`: sources spend the canary, and a dependency-only diff runs
* the harness self-test. A filter that missed one would let that change merge
* unproven.
*/
export const LANE_TOOLING = [...LANE_SOURCES, ...LANE_DEPENDENCIES];

/**
* The PR lane's matrix: empty unless the diff touches the harness, otherwise the
* canary plus whatever kernels that same diff derives. The kernel report that
Expand All @@ -675,23 +687,24 @@ export const LANE_TOOLING = [
*/
export function affectedMatrixFor(
changed: readonly string[],
root: string = repoRoot,
{ withDependencies = false }: { withDependencies?: boolean } = {},
): ShardSpec[] {
const lanePaths = withDependencies ? LANE_TOOLING : LANE_SOURCES;
const touchesLane = changed.some((file) =>
LANE_TOOLING.some((pattern) =>
lanePaths.some((pattern) =>
path.matchesGlob(normalizePath(file), pattern.endsWith('/') ? `${pattern}**` : pattern),
),
);
if (!touchesLane) return [];
const modules = new Set(derivedAffectedModules(changed, root));
const modules = new Set(derivedAffectedModules(changed, repoRoot));
// Lane sources own no kernel, so a tooling-only diff derives nothing: without
// the canary the lane would select zero mutants and prove nothing.
modules.add(LANE_CANARY);
return shardMatrix(ALL_MODULE_IDS.filter((id) => modules.has(id)));
}

function affectedMatrix(base: string): ShardSpec[] {
return affectedMatrixFor(changedFiles(base));
function affectedMatrix({ base, withDependencies }: Args): ShardSpec[] {
return affectedMatrixFor(changedFiles(base), { withDependencies });
}

/**
Expand Down Expand Up @@ -726,7 +739,7 @@ async function main(argv = process.argv.slice(2)): Promise<number> {
// Selection only — no lane run, so no envelope. The shard jobs that consume
// this matrix each write their own.
const args = parseMutationArgs(argv);
process.stdout.write(`${JSON.stringify(affectedMatrix(args.base))}\n`);
process.stdout.write(`${JSON.stringify(affectedMatrix(args))}\n`);
return 0;
}
const state: LaneState = {
Expand Down
Loading
Loading