From 6a7d613918b456e4528c66729432d10d84d01625 Mon Sep 17 00:00:00 2001 From: XU Date: Sat, 26 Sep 2026 21:24:27 +0800 Subject: [PATCH] ci: require an absolute delta before flagging a benchmark regression --- .github/workflows/ci-performance.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-performance.yml b/.github/workflows/ci-performance.yml index 47ba3e0f14..d3da2d75db 100644 --- a/.github/workflows/ci-performance.yml +++ b/.github/workflows/ci-performance.yml @@ -199,6 +199,9 @@ jobs: console.log('| Benchmark | Baseline | PR | Change | Status |'); console.log('|-----------|----------|----|---------| ------ |'); + // Ignore relative changes below this absolute delta (milliseconds). + const ABSOLUTE_NOISE_FLOOR_MS = 2; + let hasRegression = false; let hasImprovement = false; @@ -214,14 +217,21 @@ jobs: const change = ((prValue - baseValue) / baseValue * 100); const changeStr = change > 0 ? \`+\${change.toFixed(1)}%\` : \`\${change.toFixed(1)}%\`; + // Benchmarks measured in fractions of a millisecond move by tens of + // percent on ordinary runner noise, so a relative threshold alone + // fails unrelated pull requests. Require the absolute change to + // clear a floor before the percentage decides the status. + const absoluteChange = prValue - baseValue; + const clearsNoiseFloor = Math.abs(absoluteChange) >= ABSOLUTE_NOISE_FLOOR_MS; + let status = '✅'; - if (change > 50) { + if (change > 50 && clearsNoiseFloor) { status = '❌ Much Slower'; hasRegression = true; - } else if (change > 25) { + } else if (change > 25 && clearsNoiseFloor) { status = '⚠️ Slower'; hasRegression = true; - } else if (change < -25) { + } else if (change < -25 && clearsNoiseFloor) { status = '🚀 Faster'; hasImprovement = true; }