From c2f3fdbcf1f64363eedaf670c2217bcce8062b2b Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Wed, 9 Sep 2026 18:43:00 +0200 Subject: [PATCH 1/3] feat: add ncu-ci resume command Resume failed or aborted PR CI jobs when Jenkins exposes a resume action and the CI-approved commit still matches the PR head. Refuse when available failure diagnostics reference files changed by the PR. Refs: https://github.com/nodejs/node-core-utils/pull/642 Co-authored-by: Moshe Atlow Signed-off-by: Filip Skokan Assisted-by: Codex --- bin/ncu-ci.js | 58 ++- docs/ncu-ci.md | 33 ++ lib/ci/ci_failure_parser.js | 225 ++++---- lib/ci/failure_file_scanner.js | 153 ++++++ lib/ci/resume_ci.js | 150 ++++++ lib/request.js | 16 +- test/fixtures/ci-resume-walk.json | 42 ++ test/unit/ci_failure_file_scanner.test.js | 193 +++++++ test/unit/ci_resume.test.js | 601 ++++++++++++++++++++++ 9 files changed, 1352 insertions(+), 119 deletions(-) create mode 100644 lib/ci/failure_file_scanner.js create mode 100644 lib/ci/resume_ci.js create mode 100644 test/fixtures/ci-resume-walk.json create mode 100644 test/unit/ci_failure_file_scanner.test.js create mode 100644 test/unit/ci_resume.test.js diff --git a/bin/ncu-ci.js b/bin/ncu-ci.js index 80803a7e..808f28f7 100755 --- a/bin/ncu-ci.js +++ b/bin/ncu-ci.js @@ -23,6 +23,7 @@ import { import { RunPRJob } from '../lib/ci/run_ci.js'; +import { ResumePRJob } from '../lib/ci/resume_ci.js'; import { writeJson, writeFile } from '../lib/file.js'; import { getMergedConfig } from '../lib/config.js'; import { runPromise } from '../lib/run.js'; @@ -134,6 +135,26 @@ const args = yargs(hideBin(process.argv)) }, handler }) + .command({ + command: 'resume ', + desc: 'Resume the latest CI run for given PR', + builder: (yargs) => { + yargs + .positional('prid', { + describe: 'ID of the PR or URL to the PR', + type: 'string' + }) + .option('owner', { + default: '', + describe: 'GitHub repository owner' + }) + .option('repo', { + default: '', + describe: 'GitHub repository name' + }); + }, + handler + }) .command({ command: 'url ', desc: 'Automatically detect CI type and show results', @@ -278,11 +299,13 @@ class RunPRJobCommand { return this.argv.prid; } - async start() { - const { - cli, request, prid, repo, owner - } = this; + validate() { + const { cli, prid, repo, owner } = this; let validArgs = true; + if (!Number.isSafeInteger(prid) || prid <= 0) { + validArgs = false; + cli.error('Pull request ID must be a positive integer'); + } if (!repo) { validArgs = false; cli.error('GitHub repository is missing, please set it via ncu-config ' + @@ -295,6 +318,13 @@ class RunPRJobCommand { } if (!validArgs) { this.cli.setExitCode(1); + } + return validArgs; + } + + async start() { + const { cli, request, prid, repo, owner } = this; + if (!this.validate()) { return; } const { certifySafe, checkForDuplicates } = this.argv; @@ -308,6 +338,20 @@ class RunPRJobCommand { } } +class ResumePRJobCommand extends RunPRJobCommand { + async start() { + const { cli, request, prid, repo, owner } = this; + if (!this.validate()) { + return; + } + const jobRunner = new ResumePRJob(cli, request, owner, repo, prid); + if (!(await jobRunner.resume())) { + cli.setExitCode(1); + process.exitCode = 1; + } + } +} + class CICommand { constructor(cli, request, argv) { this.cli = cli; @@ -565,7 +609,8 @@ async function main(command, argv) { let commandHandler; // Prepare queue. switch (command) { - case 'run': { + case 'run': + case 'resume': { const maybeURL = URL.parse(argv.prid); if (maybeURL?.host === 'github.com') { const [, owner, repo, , prid, , commit_sha] = maybeURL.pathname.split('/'); @@ -575,7 +620,8 @@ async function main(command, argv) { argv.prid = prid; } argv.prid = Number(argv.prid); - const jobRunner = new RunPRJobCommand(cli, request, argv); + const Command = command === 'run' ? RunPRJobCommand : ResumePRJobCommand; + const jobRunner = new Command(cli, request, argv); return jobRunner.start(); } case 'rate': { diff --git a/docs/ncu-ci.md b/docs/ncu-ci.md index 0f70a76f..57b682cd 100644 --- a/docs/ncu-ci.md +++ b/docs/ncu-ci.md @@ -17,6 +17,7 @@ Commands: runs ncu-ci walk Walk the CI and display the failures ncu-ci run Start a node-test-pull-request CI job for a PR + ncu-ci resume Resume the latest node-test-pull-request CI job for a PR ncu-ci url Automatically detect CI type and show results ncu-ci pr Show results of a node-test-pull-request CI job ncu-ci commit Show results of a node-test-commit CI job @@ -172,6 +173,38 @@ ncu-ci run https://github.com/nodejs/node/pull/34127/commits/35ea6ded7315cf9d058 If the PR has the `v8 engine` label, `ncu-ci run` also triggers the `node-test-commit-v8-linux` job after the main PR CI job is started successfully. +### `ncu-ci resume ` + +`ncu-ci resume ` resumes the latest `node-test-pull-request` CI run linked +in the PR description, comments, or reviews. The job must have finished with +`FAILURE` or `ABORTED` and expose Jenkins' resume action. Running jobs and jobs with +other results are not resumed. If no PR CI run is found, the command reports that +and exits unsuccessfully. + +The CI-approved commit (`COMMIT_SHA_CHECK`) must match the PR's current HEAD. +The command refuses to resume if they differ or the approved commit cannot be +determined. + +Before resuming, the command streams failed-job console output and compares +failure diagnostics with the PR's changed files. It refuses to resume if a failed +test or a file referenced in a failure diagnostic is changed by the PR. Logs are +scanned one at a time with bounded memory. HTTP compression is decoded as the +response arrives. A match cancels the download and skips remaining logs. Unknown +or unavailable failure details do not prevent resuming; the check uses the +available diagnostics. Failure to retrieve the PR's changed-file list prevents +resuming. + +Pass a PR number with repository information from config or flags, or a PR URL: + +```sh +ncu-ci resume 34127 --owner nodejs --repo node +ncu-ci resume https://github.com/nodejs/node/pull/34127 +``` + +This uses Jenkins' **Resume build** action on the existing job. It does not start +a fresh CI run for the current PR head. Jenkins credentials with permission to +resume the job are required. + ### `ncu-ci pr ` `ncu-ci pr ` returns information about the results of a `node-test-pull-request` job. diff --git a/lib/ci/ci_failure_parser.js b/lib/ci/ci_failure_parser.js index 3635fa00..4ae0677f 100644 --- a/lib/ci/ci_failure_parser.js +++ b/lib/ci/ci_failure_parser.js @@ -132,26 +132,26 @@ function failureMatcher(Failure, patterns, ctx, text) { return null; } +const fatalPattern = /fatal: .+/g; + // The elements are ranked by priority const FAILURE_FILTERS = [{ // NOTE(mmarchini): infra-related issues should have the highest priority, as // they can cause other issues to happen. - filter(ctx, text) { - const patterns = [{ - pattern: /Read-only file system/g, - context: { index: 0, contextBefore: 1, contextAfter: 0 } - }, - { - pattern: /Device or resource busy/g, - context: { index: 0, contextBefore: 1, contextAfter: 0 } - }, - { - pattern: /There is not enough space in the file system./g, - context: { index: 0, contextBefore: 1, contextAfter: 0 } - } - ]; - return failureMatcher(InfraFailure, patterns, ctx, text); + Failure: InfraFailure, + patterns: [{ + pattern: /Read-only file system/g, + context: { index: 0, contextBefore: 1, contextAfter: 0 } + }, + { + pattern: /Device or resource busy/g, + context: { index: 0, contextBefore: 1, contextAfter: 0 } + }, + { + pattern: /There is not enough space in the file system./g, + context: { index: 0, contextBefore: 1, contextAfter: 0 } } + ], }, { // TODO: match indentation to avoid skipping context with '...' filter(ctx, text) { @@ -169,89 +169,76 @@ const FAILURE_FILTERS = [{ ); } }, { - filter(ctx, text) { - const patterns = [{ - pattern: /\[ {2}FAILED {2}\].+/g, - context: { index: 0, contextBefore: 5, contextAfter: 0 } - }]; - return failureMatcher(CCTestFailure, patterns, ctx, text); - } + Failure: CCTestFailure, + patterns: [{ + pattern: /\[ {2}FAILED {2}\].+/g, + context: { index: 0, contextBefore: 5, contextAfter: 0 } + }], }, { // VS compilation error - filter(ctx, text) { - const patterns = [{ - pattern: /error C\d+:/mg, - context: { index: 0, contextBefore: 0, contextAfter: 5 } - }]; - return failureMatcher(BuildFailure, patterns, ctx, text); - } + Failure: BuildFailure, + patterns: [{ + pattern: /error C\d+:/mg, + context: { index: 0, contextBefore: 0, contextAfter: 5 } + }], }, { - filter(ctx, text) { - const patterns = [{ - pattern: /java\.io\.IOException.+/g, - context: { index: -1, contextBefore: 0, contextAfter: 5 } - }, { - pattern: /Build timed out/g, - context: { index: 0, contextBefore: 0, contextAfter: 1 } - }]; - return failureMatcher(JenkinsFailure, patterns, ctx, text); - } + Failure: JenkinsFailure, + patterns: [{ + pattern: /java\.io\.IOException.+/g, + context: { index: -1, contextBefore: 0, contextAfter: 5 } + }, { + pattern: /Build timed out/g, + context: { index: 0, contextBefore: 0, contextAfter: 1 } + }], }, { - filter(ctx, text) { - const patterns = [{ - pattern: + Failure: GitFailure, + patterns: [{ + pattern: /Changes not staged for commit:[\s\S]+no changes added to commit/mg, - context: { index: 0, contextBefore: 0, contextAfter: 0 } - }, { - pattern: + context: { index: 0, contextBefore: 0, contextAfter: 0 } + }, { + pattern: /error: Your local changes to the following files[\s\S]+Failed to merge in the changes./g, - context: { index: 0, contextBefore: 0, contextAfter: 0 } - }, { - pattern: /warning: failed to remove .+/g, - context: { index: 0, contextBefore: 0, contextAfter: 0 } - }]; - return failureMatcher(GitFailure, patterns, ctx, text); - } + context: { index: 0, contextBefore: 0, contextAfter: 0 } + }, { + pattern: /warning: failed to remove .+/g, + context: { index: 0, contextBefore: 0, contextAfter: 0 } + }], }, { - filter(ctx, text) { - const patterns = [{ - pattern: /ERROR: Error fetching .+/g, - context: { index: 0, contextBefore: 0, contextAfter: 5 } - }, { - pattern: /hudson\.plugins\.git\.GitException+/g, - context: { index: 0, contextBefore: 0, contextAfter: 5 } - }, { - pattern: /Cannot rebase: .+/g, - context: { index: 0, contextBefore: 0, contextAfter: 1 } - }]; - return failureMatcher(GitFailure, patterns, ctx, text); - } + Failure: GitFailure, + patterns: [{ + pattern: /ERROR: Error fetching .+/g, + context: { index: 0, contextBefore: 0, contextAfter: 5 } + }, { + pattern: /hudson\.plugins\.git\.GitException+/g, + context: { index: 0, contextBefore: 0, contextAfter: 5 } + }, { + pattern: /Cannot rebase: .+/g, + context: { index: 0, contextBefore: 0, contextAfter: 1 } + }], }, { - filter(ctx, text) { - const patterns = [{ - pattern: /sh: line /g, - context: { index: 0, contextBefore: 0, contextAfter: 1 } - }, { - pattern: /fatal error:/g, - context: { index: 0, contextBefore: 0, contextAfter: 1 } - }, { - pattern: /dtrace: failed to compile script/g, - context: { index: 0, contextBefore: 0, contextAfter: 1 } - }, { - pattern: /ERROR: .+/g, - // Pick the last one - context: { index: -1, contextBefore: 0, contextAfter: 5 } - }, { - // Pick the first one - pattern: /Error: .+/g, - context: { index: 0, contextBefore: 0, contextAfter: 5 } - }]; - return failureMatcher(BuildFailure, patterns, ctx, text); - } + Failure: BuildFailure, + patterns: [{ + pattern: /sh: line /g, + context: { index: 0, contextBefore: 0, contextAfter: 1 } + }, { + pattern: /fatal error:/g, + context: { index: 0, contextBefore: 0, contextAfter: 1 } + }, { + pattern: /dtrace: failed to compile script/g, + context: { index: 0, contextBefore: 0, contextAfter: 1 } + }, { + pattern: /ERROR: .+/g, + // Pick the last one + context: { index: -1, contextBefore: 0, contextAfter: 5 } + }, { + // Pick the first one + pattern: /Error: .+/g, + context: { index: 0, contextBefore: 0, contextAfter: 5 } + }], }, { filter(ctx, text) { - const pattern = /fatal: .+/g; - const matches = text.match(pattern); + const matches = text.match(fatalPattern); if (!matches) { return null; } @@ -259,30 +246,42 @@ const FAILURE_FILTERS = [{ return [new BuildFailure(ctx, reason)]; } }, { - filter(ctx, text) { - const patterns = [{ - pattern: /FATAL: .+/g, - context: { index: -1, contextBefore: 0, contextAfter: 5 } - }, { - pattern: /make.*: write error/mg, - context: { index: 0, contextBefore: 0, contextAfter: 3 } - }, { - pattern: /error: .+/g, - context: { index: 0, contextBefore: 0, contextAfter: 5 } - }, { - pattern: /Makefile:.+failed/g, - context: { index: 0, contextBefore: 0, contextAfter: 5 } - }, { - pattern: /make.*: .+ Error \d.*/g, - context: { index: 0, contextBefore: 0, contextAfter: 3 } - }, { - pattern: /warning: failed .+/g, - context: { index: 0, contextBefore: 0, contextAfter: 3 } - }]; - return failureMatcher(BuildFailure, patterns, ctx, text); - } + Failure: BuildFailure, + patterns: [{ + pattern: /FATAL: .+/g, + context: { index: -1, contextBefore: 0, contextAfter: 5 } + }, { + pattern: /make.*: write error/mg, + context: { index: 0, contextBefore: 0, contextAfter: 3 } + }, { + pattern: /error: .+/g, + context: { index: 0, contextBefore: 0, contextAfter: 5 } + }, { + pattern: /Makefile:.+failed/g, + context: { index: 0, contextBefore: 0, contextAfter: 5 } + }, { + pattern: /make.*: .+ Error \d.*/g, + context: { index: 0, contextBefore: 0, contextAfter: 3 } + }, { + pattern: /warning: failed .+/g, + context: { index: 0, contextBefore: 0, contextAfter: 3 } + }], }]; +// Share recognition patterns with streaming consumers without exposing the +// parser's constructors, filter priority, or context-selection machinery. +export const FAILURE_PATTERNS = { + infrastructure: FAILURE_FILTERS + .filter(({ Failure }) => Failure === InfraFailure) + .flatMap(({ patterns }) => patterns.map(({ pattern }) => pattern)), + diagnostic: [ + ...FAILURE_FILTERS + .filter(({ Failure }) => Failure && Failure !== InfraFailure && Failure !== CCTestFailure) + .flatMap(({ patterns }) => patterns.map(({ pattern }) => pattern)), + fatalPattern + ] +}; + export default class CIFailureParser { constructor(ctx, text) { this.ctx = ctx; @@ -291,8 +290,10 @@ export default class CIFailureParser { parse() { const text = this.text; - for (const { filter } of FAILURE_FILTERS) { - const result = filter(this.ctx, text); + for (const { filter, Failure, patterns } of FAILURE_FILTERS) { + const result = filter + ? filter(this.ctx, text) + : failureMatcher(Failure, patterns, this.ctx, text); // TODO: we may want to concat certain types of failures if (result) { return result; diff --git a/lib/ci/failure_file_scanner.js b/lib/ci/failure_file_scanner.js new file mode 100644 index 00000000..cdb6abd6 --- /dev/null +++ b/lib/ci/failure_file_scanner.js @@ -0,0 +1,153 @@ +import { StringDecoder } from 'node:string_decoder'; + +import { FAILURE_PATTERNS } from './ci_failure_parser.js'; + +function createMatcher(patterns) { + // Preserve matching flags without sharing RegExp.lastIndex between scans. + const matchers = patterns.map(pattern => + new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, ''))); + return text => matchers.some(pattern => pattern.test(text)); +} + +const diagnostic = createMatcher(FAILURE_PATTERNS.diagnostic); +const infrastructure = createMatcher(FAILURE_PATTERNS.infrastructure); + +function fileAliases(filename) { + const paths = new Set([filename]); + if (filename.startsWith('test/')) { + paths.add(filename.slice(5)); + paths.add(filename.slice(5).replace(/\.(?:js|mjs|cjs|out)$/, '')); + } else if (filename.startsWith('lib/')) { + paths.add(filename.slice(4)); + } + return [...paths]; +} + +// Emit overlapping, normalized windows rather than buffering whole log lines. +// Explicit line boundaries keep network chunk boundaries out of the matching rules. +async function * logWindows(source, overlap) { + const decoder = new StringDecoder('utf8'); + let tail = ''; + let lineStart = true; + let afterBackslash = false; + + function * consume(raw) { + raw = raw.replace(/\r/g, ''); + let text = raw.replace(/\\+/g, '/'); + if (afterBackslash && raw.startsWith('\\')) text = text.slice(1); + if (raw) afterBackslash = raw.endsWith('\\'); + for (let offset = 0; offset < text.length;) { + const newline = text.indexOf('\n', offset); + const end = Math.min(offset + 8192, newline < 0 ? text.length : newline + 1); + const window = tail + text.slice(offset, end); + const lineEnd = end === newline + 1; + yield { text: window, lineStart, lineEnd }; + if (lineEnd) { + tail = ''; + lineStart = true; + } else { + lineStart &&= window.length <= overlap; + tail = window.slice(-overlap); + } + offset = end; + } + } + + for await (const chunk of source) yield * consume(decoder.write(chunk)); + yield * consume(decoder.end()); + if (tail) yield { text: `${tail}\n`, lineStart, lineEnd: true }; +} + +// Each line is reduced to a filename and failure markers. Neither a giant line +// nor a giant TAP block needs to survive in memory. +async function * failureLines(windows, matchFile) { + let line = {}; + for await (const { text, lineStart, lineEnd } of windows) { + line.file ??= matchFile(text, lineStart); + line.diagnostic ||= diagnostic(text); + line.infrastructure ||= infrastructure(text); + line.cpp ||= /\[ {2}FAILED {2}\]/.test(text); + line.tapStart ||= lineStart && /^not ok \d+/.test(text); + line.tapEnd ||= lineEnd && / {2}\.\.\.\n$/.test(text); + line.todo ||= text.includes('# TODO :'); + line.gitStart ||= text.includes('Changes not staged for commit:') || + text.includes('error: Your local changes to the following files'); + line.gitEnd ||= text.includes('no changes added to commit') || + text.includes('Failed to merge in the changes.'); + if (lineEnd) { + yield line; + line = {}; + } + } +} + +async function findFailure(lines) { + let history = []; + let followingLines = 0; + let tap = null; + let git = null; + + for await (const line of lines) { + const precedingLines = history; + history = [...history, line.file].slice(-5); + if (line.tapStart) { + tap = {}; + followingLines = 0; + } + if (tap) { + tap.file ??= line.file; + tap.todo ||= line.todo; + // A later TODO can mark this as an expected failure; wait for the ending. + if (line.tapEnd) { + if (!tap.todo && tap.file) return tap.file; + tap = null; + } + continue; + } + + if (line.gitStart) git = {}; + if (git) { + git.file ??= line.file; + if (line.gitEnd) { + if (git.file) return git.file; + git = null; + } + } + if (line.infrastructure || line.cpp) { + const before = line.cpp ? 5 : 1; + const file = line.file || precedingLines.slice(-before).find(Boolean); + if (file) return file; + } + if (line.diagnostic) followingLines = 6; + if (followingLines > 0) { + followingLines--; + if (line.file) return line.file; + } + } +} + +export class FailureFileScanner { + constructor(filenames) { + const aliases = [...filenames].map(filename => ({ filename, paths: fileAliases(filename) })); + this.overlap = Math.max(256, + ...aliases.flatMap(({ paths }) => paths.map(path => path.length + 2))); + const matchers = aliases.map(({ filename, paths }) => ({ + filename, + patterns: paths.map(path => { + const escaped = path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + // Both delimiters must be real: a window boundary is not a path boundary. + return new RegExp(`[^\\w.-]${escaped}(?=[^\\w./-])`); + }) + })); + this.matchFile = (text, lineStart) => { + // The start of a real log line is also a valid left delimiter. + if (lineStart) text = `\n${text}`; + return matchers.find(({ patterns }) => + patterns.some(pattern => pattern.test(text)))?.filename; + }; + } + + scan(source) { + return findFailure(failureLines(logWindows(source, this.overlap), this.matchFile)); + } +} diff --git a/lib/ci/resume_ci.js b/lib/ci/resume_ci.js new file mode 100644 index 00000000..0fe679f0 --- /dev/null +++ b/lib/ci/resume_ci.js @@ -0,0 +1,150 @@ +import { JobParser, CI_TYPES_KEYS } from './ci_type_parser.js'; +import { CI_CRUMB_URL } from './run_ci.js'; +import { PRBuild } from './build-types/pr_build.js'; +import { getPrURL } from '../links.js'; +import { debuglog } from '../verbosity.js'; +import { FailureFileScanner } from './failure_file_scanner.js'; + +export class ResumePRJob { + constructor(cli, request, owner, repo, prid) { + this.cli = cli; + this.request = request; + this.owner = owner; + this.repo = repo; + this.prid = prid; + } + + async checkFailures(jobid) { + const { cli, request, owner, repo, prid } = this; + const filenames = new Set(); + for await (const file of request.getPullRequestFiles({ owner, repo, prid })) { + for (const filename of [file.filename, file.previous_filename].filter(Boolean)) { + filenames.add(filename); + } + } + const overlaps = new Set(); + const buildRequest = Object.create(request); + buildRequest.json = async(...args) => { + const data = await request.json(...args); + // Inspect failed descendants even when their parent job was aborted. + return data?.result === 'ABORTED' ? { ...data, result: 'FAILURE' } : data; + }; + // Reuse the build traversal, but consume console responses directly instead + // of handing whole logs to its summary parser. Serialize downloads so a + // match cancels this response and prevents queued logs from being opened. + let pending = Promise.resolve(); + buildRequest.text = (url) => { + const scan = pending.then(async() => { + if (!filenames.size || overlaps.size) return ''; + const scanner = new FailureFileScanner(filenames); + const match = await scanner.scan(request.stream(url)); + if (match) overlaps.add(match); + return ''; + }); + pending = scan.catch(debuglog); + return scan; + }; + const build = new PRBuild(cli, buildRequest, jobid, Infinity); + try { + await build.getResults(); + } catch (err) { + debuglog(err); + } + await pending; + if (overlaps.size) { + for (const filename of [...overlaps].sort()) { + cli.error(filename); + } + return false; + } + return true; + } + + async resume() { + const { cli, request, prid } = this; + let crumb; + cli.startSpinner('Validating Jenkins credentials'); + try { + ({ crumb } = await request.json(CI_CRUMB_URL)); + if (!crumb) { + throw new Error('Missing Jenkins crumb'); + } + } catch (err) { + debuglog(err); + cli.stopSpinner('Jenkins credentials invalid', cli.SPINNER_STATUS.FAILED); + return false; + } + cli.stopSpinner('Jenkins credentials valid'); + + try { + cli.startSpinner(`Looking for CI runs for pull request ${prid}`); + const parser = await JobParser.fromPR(getPrURL(this), cli, request); + const job = parser.parse().get(CI_TYPES_KEYS.PR); + if (!job) { + cli.stopSpinner(`No CI run detected from pull request ${prid}`, + cli.SPINNER_STATUS.FAILED); + return false; + } + cli.stopSpinner(`Found PR CI job ${job.jobid}`); + + const build = new PRBuild(cli, request, job.jobid, undefined, + 'result,building,actions[_class,parameters[name,value]]'); + const { result, building, actions = [] } = await build.getBuildData(); + if (building || (result !== 'FAILURE' && result !== 'ABORTED')) { + const status = building ? 'RUNNING' : result ?? 'RUNNING'; + cli.error(`CI job ${job.jobid} is in status ${status}, skipping resume`); + return false; + } + if (!actions.some(action => + action._class === 'com.tikal.jenkins.plugins.multijob.MultiJobResumeBuild')) { + cli.error(`CI job ${job.jobid} is not resumable`); + return false; + } + + const approvedSHAs = new Set(actions.flatMap(action => action.parameters ?? []) + .filter(parameter => parameter.name === 'COMMIT_SHA_CHECK') + .map(parameter => parameter.value)); + const [approvedSHA] = approvedSHAs; + if (approvedSHAs.size !== 1 || typeof approvedSHA !== 'string' || !approvedSHA) { + cli.error(`Refusing to resume CI job ${job.jobid}: cannot determine its approved commit`); + return false; + } + + cli.startSpinner('Checking failures against changed PR files'); + if (!(await this.checkFailures(job.jobid))) { + cli.stopSpinner('Refusing to resume CI: failures reference files changed by this PR', + cli.SPINNER_STATUS.FAILED); + return false; + } + cli.stopSpinner('No changed PR files found in available failure details'); + + // Read HEAD after inspecting failures so the comparison is fresh when resuming. + const pr = await request.getPullRequest(getPrURL(this)); + if (pr.head?.sha !== approvedSHA) { + cli.error(`Refusing to resume CI job ${job.jobid}: ` + + 'its approved commit does not match the current PR HEAD'); + return false; + } + + cli.startSpinner(`Resuming PR CI job ${job.jobid}`); + const response = await request.fetch(`${build.jobUrl}resume`, { + method: 'POST', + headers: { + 'Jenkins-Crumb': crumb + } + }); + if (response.status !== 200) { + cli.stopSpinner( + `Failed to resume PR CI: ${response.status} ${response.statusText}`, + cli.SPINNER_STATUS.FAILED); + return false; + } + cli.stopSpinner('PR CI job successfully resumed'); + } catch (err) { + debuglog(err); + cli.stopSpinner('Failed to resume CI', cli.SPINNER_STATUS.FAILED); + return false; + } + return true; + } +} diff --git a/lib/request.js b/lib/request.js index 0bafd73f..6bac06c7 100644 --- a/lib/request.js +++ b/lib/request.js @@ -58,6 +58,17 @@ export default class Request { return res.text(); } + async * stream(url, options = {}) { + const res = await this.fetch(url, options); + if (!res.ok) { + await res.body?.cancel(); + throw new Error(`Unable to stream ${url}: ${res.status} ${res.statusText}`); + } + // Undici decompresses HTTP content encodings as it reads. Returning early + // from this iterator also cancels the response body and its HTTP download. + yield * res.body; + } + async json(url, options = {}) { options.headers = options.headers || {}; const text = await this.text(url, options); @@ -106,7 +117,10 @@ export default class Request { const url = `/repos/${owner}/${repo}/pulls/${prid}/files?per_page=100&page=${page}`; const batch = await this.json(url); - if (!Array.isArray(batch) || batch.length === 0) { + if (!Array.isArray(batch)) { + throw new Error('Unable to retrieve pull request files'); + } + if (batch.length === 0) { break; } yield * batch; diff --git a/test/fixtures/ci-resume-walk.json b/test/fixtures/ci-resume-walk.json new file mode 100644 index 00000000..ab7d0cef --- /dev/null +++ b/test/fixtures/ci-resume-walk.json @@ -0,0 +1,42 @@ +[ + { + "filename": "test/fixtures/typescript/ts/test-mock-module.ts", + "failure": { + "type": "JS_TEST_FAILURE", + "file": "es-module/test-typescript", + "url": "https://ci.nodejs.org/job/node-test-binary-windows-js-suites/RUN_SUBSET=0,nodes=win10-COMPILED_BY-vs2022_clang/43103/console", + "upstream": "https://ci.nodejs.org/job/node-test-pull-request/77225/", + "reason": "not ok 141 es-module/test-typescript\n '✖ c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\fixtures\\\\typescript\\\\ts\\\\test-mock-module.ts (45.1525ms)\\n' +" + } + }, + { + "filename": "test/fixtures/rc/test.js", + "failure": { + "type": "JS_TEST_FAILURE", + "file": "parallel/test-config-file", + "url": "https://ci.nodejs.org/job/node-test-binary-windows-js-suites/RUN_SUBSET=0,nodes=win10-COMPILED_BY-vs2022_clang/43103/console", + "upstream": "https://ci.nodejs.org/job/node-test-pull-request/77225/", + "reason": "not ok 298 parallel/test-config-file\n + '✖ c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\fixtures\\\\rc\\\\test.js (106.6917ms)\\n' +" + } + }, + { + "filename": "test/es-module/test-typescript.mjs", + "failure": { + "type": "JS_TEST_FAILURE", + "file": "es-module/test-typescript", + "url": "https://ci.nodejs.org/job/node-test-binary-windows-js-suites/RUN_SUBSET=0,nodes=win10-COMPILED_BY-vs2022_clang/43103/console", + "upstream": "https://ci.nodejs.org/job/node-test-pull-request/77225/", + "reason": "not ok 141 es-module/test-typescript\n Location: test\\es-module\\test-typescript.mjs:240:1" + } + }, + { + "filename": "lib/internal/vfs/file_handle.js", + "failure": { + "type": "JS_TEST_FAILURE", + "file": "/home/iojs/build/workspace/node-test-linter/lib/internal/vfs/file_handle.js", + "url": "https://ci.nodejs.org/job/node-test-linter/67148/console", + "upstream": "https://ci.nodejs.org/job/node-test-pull-request/77227/", + "reason": "not ok 1056 - /home/iojs/build/workspace/node-test-linter/lib/internal/vfs/file_handle.js" + } + } +] diff --git a/test/unit/ci_failure_file_scanner.test.js b/test/unit/ci_failure_file_scanner.test.js new file mode 100644 index 00000000..0c59df5a --- /dev/null +++ b/test/unit/ci_failure_file_scanner.test.js @@ -0,0 +1,193 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import { createServer } from 'node:http'; +import { setImmediate } from 'node:timers/promises'; +import { gzipSync } from 'node:zlib'; +import { describe, it } from 'node:test'; +import { fetch } from 'undici'; +import { FailureFileScanner } from '../../lib/ci/failure_file_scanner.js'; +import CIFailureParser from '../../lib/ci/ci_failure_parser.js'; +import Request from '../../lib/request.js'; + +const filename = 'test/parallel/test-example.js'; +const tap = (text) => `not ok 1 parallel/test-example\n ---\n${text}\n ...\n`; + +async function scan(text, files = [filename], size = 8192) { + const buffer = Buffer.from(text); + async function * source() { + for (let offset = 0; offset < buffer.length; offset += size) { + yield buffer.subarray(offset, offset + size); + } + } + return new FailureFileScanner(files).scan(source()); +} + +describe('Streaming failure file scanner', () => { + for (const message of [ + 'src/node.cc: Read-only file system', + 'error C2143: src/node.cc', + 'java.io.IOException: src/node.cc', + 'fatal: src/node.cc', + 'ERROR: src/node.cc' + ]) { + it(`reuses diagnostic patterns across parsers without regex state leaking: ${message}`, + async() => { + const log = `Build started\n${message}\n`; + for (let i = 0; i < 3; i++) { + assert.equal(await scan(log, ['src/node.cc'], 1), 'src/node.cc'); + const failures = new CIFailureParser({}, log).parse(); + assert.equal(failures.length, 1); + assert.match(failures[0].reason, /src\/node\.cc/); + } + }); + } + + it('handles UTF-8, CRLF, escaped Windows paths, and markers split at every byte', async() => { + const path = 'test/fixtures/é-example.js'; + const log = tap(' c:\\\\workspace\\\\test\\\\fixtures\\\\é-example.js:42:1') + .replaceAll('\n', '\r\n'); + assert.equal(await scan(log, [path], 1), path); + }); + + it('requires a real path boundary across chunks', async() => { + assert.equal(await scan(tap('').replace('test-example', 'test-example-long'), + [filename], 1), undefined); + assert.equal(await scan('src/node.cc-extra: error: failure\n', ['src/node.cc'], 1), + undefined); + }); + + it('does not turn a sliding window boundary into a path boundary', async() => { + const log = `xsrc/node.cc${' '.repeat(245)}error: failure\n`; + assert.equal(await scan(log, ['src/node.cc'], 256), undefined); + }); + + it('ignores successful tests and ordinary path mentions', async() => { + assert.equal(await scan(`ok 1 parallel/test-example\n${filename}\n`), undefined); + }); + + it('ignores expected failures even when TODO appears late in a large block', async() => { + const log = tap(` ${'x'.repeat(200000)}\n # TODO : expected failure`); + assert.equal(await scan(log, [filename], 31), undefined); + }); + + it('recognizes a later failure after a TODO block', async() => { + assert.equal(await scan(tap(' # TODO : expected') + tap(' actual failure')), filename); + }); + + it('handles indented and output-prefixed TAP endings captured by ncu-ci walk', async() => { + for (const ending of [' ...\n', ' [out] ...\n']) { + assert.equal(await scan(`not ok 1 parallel/test-example\n${ending}`, [filename], 1), + filename); + } + }); + + it('retains a filename until a diagnostic at the other end of a giant line', async() => { + assert.equal(await scan(`src/node.cc ${'x'.repeat(200000)} error: failure\n`, + ['src/node.cc']), 'src/node.cc'); + }); + + it('does not treat an incomplete TAP block as a confirmed failure', async() => { + assert.equal(await scan('not ok 1 parallel/test-example\n partial output'), undefined); + }); + + it('does not carry an unfinished failure into another log when reused', async() => { + const scanner = new FailureFileScanner([filename]); + assert.equal(await scanner.scan([Buffer.from('not ok 1 parallel/test-example\n')]), + undefined); + assert.equal(await scanner.scan([Buffer.from('unrelated output\n ...\n')]), undefined); + assert.equal(await scanner.scan([Buffer.from(tap(' actual failure'))]), filename); + }); + + it('matches a compiler diagnostic without a final newline', async() => { + assert.equal(await scan('../src/node.cc:42: error: failure', ['src/node.cc'], 1), + 'src/node.cc'); + }); + + it('keeps compiler and C++ failure context', async() => { + assert.equal(await scan('error: compilation failed\n src/node.cc:42\n', ['src/node.cc']), + 'src/node.cc'); + assert.equal(await scan('test/cctest/test-example.cc:42\nmessage\n[ FAILED ] Example\n', + ['test/cctest/test-example.cc']), 'test/cctest/test-example.cc'); + }); + + it('does not retain diagnostic context indefinitely', async() => { + assert.equal(await scan(`error: unrelated\n${'unrelated\n'.repeat(6)}${filename}\n`), + undefined); + }); + + it('scans 256 MiB of giant lines and TAP output with a 32 MiB heap', () => { + const scannerURL = new URL('../../lib/ci/failure_file_scanner.js', import.meta.url).href; + const result = spawnSync(process.execPath, ['--max-old-space-size=32', + '--input-type=module', '--eval', ` + import assert from 'node:assert/strict'; + import { FailureFileScanner } from ${JSON.stringify(scannerURL)}; + const chunk = Buffer.alloc(65536, 120); + async function * source() { + // A 128 MiB line with its filename and diagnostic at opposite ends. + yield Buffer.from('src/unrelated.cc '); + for (let i = 0; i < 2048; i++) yield chunk; + yield Buffer.from(' error: compilation failed\\n'); + yield Buffer.from('not ok 1 parallel/test-example\\n ---\\n'); + for (let i = 0; i < 2048; i++) yield chunk; + yield Buffer.from('\\n ...\\n'); + } + assert.equal(await new FailureFileScanner([${JSON.stringify(filename)}]).scan(source()), + ${JSON.stringify(filename)}); + `], { encoding: 'utf8', timeout: 30000 }); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stdout + result.stderr); + }); +}); + +describe('Streaming HTTP logs', () => { + async function server(t, handler) { + const instance = createServer(handler); + instance.listen(0, '127.0.0.1'); + await once(instance, 'listening'); + t.after(() => { + instance.closeAllConnections(); + instance.close(); + }); + return `http://127.0.0.1:${instance.address().port}/consoleText`; + } + + const request = Object.assign(Object.create(Request.prototype), { fetch }); + + it('cancels the HTTP response without downloading its remaining body', async(t) => { + let responseClosed; + let sent = 0; + const url = await server(t, async(_req, res) => { + responseClosed = once(res, 'close'); + res.write(tap(' failure')); + const chunk = Buffer.alloc(65536); + while (!res.destroyed && sent < 1024 * 1024 * 1024) { + if (!res.write(chunk)) { + await Promise.race([once(res, 'drain'), responseClosed]); + } + sent += chunk.length; + await setImmediate(); + } + }); + assert.equal(await new FailureFileScanner([filename]).scan(request.stream(url)), filename); + await responseClosed; + assert.ok(sent < 1024 * 1024 * 1024, `Downloaded ${sent} trailing bytes`); + }); + + it('scans a gzip-encoded HTTP response without double decompression', async(t) => { + const url = await server(t, (req, res) => { + assert.match(req.headers['accept-encoding'], /gzip/); + res.writeHead(200, { 'Content-Encoding': 'gzip' }); + res.end(gzipSync(tap(' failure'))); + }); + assert.equal(await new FailureFileScanner([filename]).scan(request.stream(url)), filename); + }); + + it('cancels an error response instead of scanning its body', async(t) => { + const url = await server(t, (_req, res) => { + res.writeHead(404); + res.end(tap(' failure')); + }); + await assert.rejects(new FailureFileScanner([filename]).scan(request.stream(url)), /404/); + }); +}); diff --git a/test/unit/ci_resume.test.js b/test/unit/ci_resume.test.js new file mode 100644 index 00000000..d462c927 --- /dev/null +++ b/test/unit/ci_resume.test.js @@ -0,0 +1,601 @@ +import { describe, it, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as sinon from 'sinon'; + +import { ResumePRJob } from '../../lib/ci/resume_ci.js'; +import { CI_CRUMB_URL } from '../../lib/ci/run_ci.js'; +import TestCLI from '../fixtures/test_cli.js'; +import Request from '../../lib/request.js'; +import { PRBuild } from '../../lib/ci/build-types/pr_build.js'; + +const approvedSHA = 'a'.repeat(40); +const resumeTree = 'result,building,actions[_class,parameters[name,value]]'; +const resumeBuildData = { + result: 'FAILURE', + building: false, + actions: [ + { _class: 'com.tikal.jenkins.plugins.multijob.MultiJobResumeBuild' }, + { parameters: [{ name: 'COMMIT_SHA_CHECK', value: approvedSHA }] } + ] +}; +const failureLog = 'not ok 1 parallel/test-example\n' + + ' ---\n severity: fail\n stack: |-\n AssertionError\n ...\n'; +const failureBuildData = { + result: 'FAILURE', + actions: [{ parameters: [] }], + changeSet: { items: [] }, + subBuilds: [{ + buildNumber: 1, + build: { + subBuilds: [{ + jobName: 'node-test-commit-linux-freestyle', + result: 'FAILURE', + url: 'https://ci.nodejs.org/job/node-test-commit-linux-freestyle/1/' + }] + } + }] +}; + +// Diagnostic excerpts captured with ncu-ci walk pr on 2026-09-09. +const walkFailures = JSON.parse(readFileSync( + new URL('../fixtures/ci-resume-walk.json', import.meta.url), 'utf8')); + +describe('Resume file checks against real CI diagnostics', () => { + for (const { filename, failure } of walkFailures) { + it(`detects a PR change to ${filename}`, async() => { + const request = { + async json() { return failureBuildData; }, + async * stream() { yield Buffer.from(`${failure.reason}\n ...\n`); }, + async * getPullRequestFiles() { yield { filename }; } + }; + const cli = new TestCLI(); + const runner = new ResumePRJob(cli, request, 'nodejs', 'node', 1); + assert.equal(await runner.checkFailures(1), false); + assert.deepEqual(cli._calls.error, [[filename]]); + + request.getPullRequestFiles = async function * () { + yield { filename: `${filename}.unrelated` }; + }; + assert.equal(await runner.checkFailures(1), true); + }); + } +}); + +describe('Jenkins resume', () => { + const owner = 'nodejs'; + const repo = 'node-auto-test'; + const prid = 123456; + const jobid = 654321; + const crumb = 'asdf1234'; + const jobURL = `https://ci.nodejs.org/job/node-test-pull-request/${jobid}/`; + const apiURL = `${jobURL}api/json?tree=${encodeURIComponent(resumeTree)}`; + const fullAPIURL = new PRBuild(null, null, jobid).apiUrl; + const filesURL = `/repos/${owner}/${repo}/pulls/${prid}/files?per_page=100&page=1`; + const prURL = `/repos/${owner}/${repo}/pulls/${prid}`; + const comment = (bodyText, publishedAt = '2026-09-09T12:00:00Z') => + ({ bodyText, publishedAt }); + let cli; + let request; + let jobRunner; + + beforeEach(() => { + cli = new TestCLI(); + request = { + json: sinon.stub().rejects(new Error('Unexpected JSON request')), + gql: sinon.stub().rejects(new Error('Unexpected GraphQL request')), + text: sinon.stub().resolves(failureLog), + async * stream(url) { yield Buffer.from(await request.text(url)); }, + getPullRequestFiles: Request.prototype.getPullRequestFiles, + getPullRequest: Request.prototype.getPullRequest, + fetch: sinon.stub().resolves({ status: 200 }) + }; + request.json.withArgs(CI_CRUMB_URL).resolves({ crumb }); + request.json.withArgs(apiURL).resolves(resumeBuildData); + request.json.withArgs(prURL).resolves({ head: { sha: approvedSHA } }); + request.json.withArgs(fullAPIURL).resolves(failureBuildData); + request.json.withArgs(filesURL).resolves([{ filename: 'README.md' }]); + request.gql.withArgs('PR').resolves({ + repository: { + pullRequest: { bodyText: '', createdAt: '2026-09-08T12:00:00Z' } + } + }); + request.gql.withArgs('Reviews').resolves([]); + request.gql.withArgs('PRComments').resolves([comment(jobURL)]); + jobRunner = new ResumePRJob(cli, request, owner, repo, prid); + }); + + it('resumes the PR job with a Jenkins crumb', async() => { + assert.equal(await jobRunner.resume(), true); + sinon.assert.calledOnceWithExactly(request.fetch, `${jobURL}resume`, { + method: 'POST', + headers: { 'Jenkins-Crumb': crumb } + }); + assert.equal(request.gql.callCount, 3); + for (const call of request.gql.getCalls()) { + assert.deepEqual(call.args[1], { owner, repo, prid }); + } + assert.deepEqual(cli._calls.stopSpinner.at(-1), ['PR CI job successfully resumed']); + }); + + it('uses the latest PR CI link across the whole thread', async() => { + request.gql.withArgs('PRComments').resolves([ + comment('https://ci.nodejs.org/job/node-test-commit/987654/', '2026-09-10T12:00:00Z'), + comment('https://ci.nodejs.org/job/node-test-pull-request/123456/', + '2026-09-08T13:00:00Z') + ]); + request.gql.withArgs('Reviews').resolves([comment(jobURL)]); + assert.equal(await jobRunner.resume(), true); + assert.equal(request.fetch.firstCall.args[0], `${jobURL}resume`); + }); + + it('finds CI links in the PR description', async() => { + request.gql.withArgs('PRComments').resolves([]); + request.gql.withArgs('PR').resolves({ + repository: { + pullRequest: { bodyText: jobURL, createdAt: '2026-09-08T12:00:00Z' } + } + }); + assert.equal(await jobRunner.resume(), true); + assert.equal(request.fetch.firstCall.args[0], `${jobURL}resume`); + }); + + for (const comments of [[], [comment('https://ci.nodejs.org/job/node-test-commit/123456/')]]) { + it(`fails gracefully with ${comments.length ? 'only non-PR CI links' : 'no CI links'}`, + async() => { + request.gql.withArgs('PRComments').resolves(comments); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + sinon.assert.calledOnceWithExactly(request.json, CI_CRUMB_URL); + assert.deepEqual(cli._calls.stopSpinner.at(-1), [ + `No CI run detected from pull request ${prid}`, cli.SPINNER_STATUS.FAILED + ]); + }); + } + + for (const result of [null, 'SUCCESS', 'UNSTABLE', 'NOT_BUILT']) { + it(`does not resume a job with result ${result}`, async() => { + request.json.withArgs(apiURL).resolves({ result, building: result === null }); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + assert.deepEqual(cli._calls.error, [ + [`CI job ${jobid} is in status ${result ?? 'RUNNING'}, skipping resume`] + ]); + }); + } + + it('does not resume a running job even if its result is FAILURE', async() => { + request.json.withArgs(apiURL).resolves({ result: 'FAILURE', building: true }); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('resumes an aborted job with a resume action', async() => { + request.json.withArgs(apiURL).resolves({ ...resumeBuildData, result: 'ABORTED' }); + request.json.withArgs(fullAPIURL).resolves({ ...failureBuildData, result: 'ABORTED' }); + assert.equal(await jobRunner.resume(), true); + sinon.assert.calledOnce(request.fetch); + }); + + it('checks failed tests inside an aborted job before resuming', async() => { + request.json.withArgs(apiURL).resolves({ ...resumeBuildData, result: 'ABORTED' }); + request.json.withArgs(fullAPIURL).resolves({ ...failureBuildData, result: 'ABORTED' }); + request.json.withArgs(filesURL).resolves([{ filename: 'test/parallel/test-example.js' }]); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + assert.deepEqual(cli._calls.error, [['test/parallel/test-example.js']]); + }); + + for (const result of ['FAILURE', 'ABORTED']) { + it(`refuses a ${result} job without a resume action`, async() => { + request.json.withArgs(apiURL).resolves({ ...resumeBuildData, result, actions: [] }); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + assert.deepEqual(cli._calls.error, [[`CI job ${jobid} is not resumable`]]); + }); + } + + it('does not resume an aborted job that is still building', async() => { + request.json.withArgs(apiURL).resolves({ + ...resumeBuildData, result: 'ABORTED', building: true + }); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + for (const result of ['FAILURE', 'ABORTED']) { + it(`refuses a ${result} job approved for a different PR HEAD`, async() => { + request.json.withArgs(apiURL).resolves({ ...resumeBuildData, result }); + request.json.withArgs(prURL).resolves({ head: { sha: 'b'.repeat(40) } }); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + assert.match(cli._calls.error.at(-1)[0], /does not match the current PR HEAD/); + }); + } + + for (const value of [undefined, '', false]) { + it(`refuses a job without an approved commit: ${value}`, async() => { + request.json.withArgs(apiURL).resolves({ + ...resumeBuildData, + actions: [resumeBuildData.actions[0], { + parameters: [{ name: 'COMMIT_SHA_CHECK', value }] + }] + }); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + assert.match(cli._calls.error.at(-1)[0], /cannot determine its approved commit/); + }); + } + + it('accepts repeated parameters for the same approved commit', async() => { + request.json.withArgs(apiURL).resolves({ + ...resumeBuildData, + actions: [...resumeBuildData.actions, resumeBuildData.actions[1]] + }); + assert.equal(await jobRunner.resume(), true); + }); + + it('refuses conflicting approved commit parameters', async() => { + request.json.withArgs(apiURL).resolves({ + ...resumeBuildData, + actions: [...resumeBuildData.actions, { + parameters: [{ name: 'COMMIT_SHA_CHECK', value: 'b'.repeat(40) }] + }] + }); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('refuses when the current PR HEAD cannot be retrieved', async() => { + request.json.withArgs(prURL).rejects(new Error('Unavailable')); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('does not fall back to an older failed job when the latest job is successful', async() => { + request.gql.withArgs('PRComments').resolves([ + comment('https://ci.nodejs.org/job/node-test-pull-request/123456/', + '2026-09-08T13:00:00Z'), + comment(jobURL) + ]); + request.json.withArgs(apiURL).resolves({ result: 'SUCCESS', building: false }); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + assert.equal(request.json.callCount, 2); + assert.equal(request.json.lastCall.args[0], apiURL); + }); + + for (const invalidCrumb of [undefined, '', false]) { + it(`rejects an invalid Jenkins crumb: ${invalidCrumb}`, async() => { + request.json.withArgs(CI_CRUMB_URL).resolves({ crumb: invalidCrumb }); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.gql); + sinon.assert.notCalled(request.fetch); + }); + } + + it('fails if Jenkins credentials cannot be validated', async() => { + request.json.withArgs(CI_CRUMB_URL).rejects(new Error('Unauthorized')); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.gql); + sinon.assert.notCalled(request.fetch); + }); + + it('fails if the PR cannot be loaded', async() => { + request.gql.withArgs('PR').rejects(new Error('Not found')); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('fails if build data cannot be loaded', async() => { + request.json.withArgs(apiURL).rejects(new Error('Not found')); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('fails if the resume request throws', async() => { + request.fetch.rejects(new Error('Connection reset')); + assert.equal(await jobRunner.resume(), false); + assert.deepEqual(cli._calls.stopSpinner.at(-1), [ + 'Failed to resume CI', cli.SPINNER_STATUS.FAILED + ]); + }); + + it('reports a failed resume request', async() => { + request.fetch.resolves({ status: 403, statusText: 'Forbidden' }); + assert.equal(await jobRunner.resume(), false); + assert.deepEqual(cli._calls.stopSpinner.at(-1), [ + 'Failed to resume PR CI: 403 Forbidden', cli.SPINNER_STATUS.FAILED + ]); + }); + + for (const filename of ['test/parallel/test-example.js', 'test/parallel/test-example.mjs']) { + it(`refuses to resume a failed test changed by the PR: ${filename}`, async() => { + request.json.withArgs(filesURL).resolves([{ filename }]); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + assert.deepEqual(cli._calls.error, [[filename]]); + }); + } + + it('refuses when a fixture rename leaves a broken import in an unchanged test', async() => { + request.text.resolves(failureLog.replace('AssertionError', + "Error: Cannot find module '../fixtures/old-name.js'")); + request.json.withArgs(filesURL).resolves([{ + filename: 'test/fixtures/new-name.js', + previous_filename: 'test/fixtures/old-name.js' + }]); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + assert.deepEqual(cli._calls.error, [['test/fixtures/old-name.js']]); + }); + + for (const path of ['/workspace/src/node.cc:42:5', 'C:\\workspace\\src\\node.cc:42:5']) { + it(`checks source paths in diagnostics: ${path}`, async() => { + request.text.resolves(failureLog.replace('AssertionError', `${path}: AssertionError`)); + request.json.withArgs(filesURL).resolves([{ filename: 'src/node.cc' }]); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + } + + it('checks compilation failures against changed source files', async() => { + request.text.resolves('../src/node.cc:42:5: error: no matching function\n'); + request.json.withArgs(filesURL).resolves([{ filename: 'src/node.cc' }]); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('does not match test names that only share a prefix', async() => { + request.json.withArgs(filesURL).resolves([{ filename: 'test/parallel/test-exam.js' }]); + assert.equal(await jobRunner.resume(), true); + sinon.assert.calledOnce(request.fetch); + }); + + it('checks all failed tests', async() => { + request.text.resolves(failureLog + failureLog.replace('test-example', 'test-second')); + request.json.withArgs(filesURL).resolves([{ filename: 'test/parallel/test-second.js' }]); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('checks failed tests even when an infrastructure error takes precedence', async() => { + request.text.resolves('Read-only file system\n' + failureLog); + request.json.withArgs(filesURL).resolves([{ filename: 'test/parallel/test-example.js' }]); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('checks available failures even when another build log cannot be downloaded', async() => { + const data = structuredClone(failureBuildData); + const url = 'https://ci.nodejs.org/job/node-test-commit-linux-freestyle/2/'; + data.subBuilds[0].build.subBuilds.push({ + jobName: 'node-test-commit-linux-freestyle', result: 'FAILURE', url + }); + request.json.withArgs(fullAPIURL).resolves(data); + request.text.withArgs(`${url}consoleText`).rejects(new Error('Unavailable')); + request.json.withArgs(filesURL).resolves([{ filename: 'test/parallel/test-example.js' }]); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('cancels a matching log and never opens queued logs', async() => { + const data = structuredClone(failureBuildData); + data.subBuilds[0].build.subBuilds.push({ + jobName: 'node-test-commit-linux-freestyle', result: 'FAILURE', + url: 'https://ci.nodejs.org/job/node-test-commit-linux-freestyle/2/' + }); + request.json.withArgs(fullAPIURL).resolves(data); + request.json.withArgs(filesURL).resolves([{ filename: 'test/parallel/test-example.js' }]); + let opened = 0; + let cancelled = false; + request.stream = async function * () { + opened++; + try { + yield Buffer.from(failureLog); + assert.fail('Must not read the rest of the log'); + } finally { + cancelled = true; + } + }; + assert.equal(await jobRunner.resume(), false); + assert.equal(opened, 1); + assert.equal(cancelled, true); + sinon.assert.notCalled(request.fetch); + }); + + it('continues to a matching log after a stream fails midway', async() => { + const data = structuredClone(failureBuildData); + data.subBuilds[0].build.subBuilds.push({ + jobName: 'node-test-commit-linux-freestyle', result: 'FAILURE', + url: 'https://ci.nodejs.org/job/node-test-commit-linux-freestyle/2/' + }); + request.json.withArgs(fullAPIURL).resolves(data); + request.json.withArgs(filesURL).resolves([{ filename: 'test/parallel/test-example.js' }]); + let active = false; + let opened = 0; + request.stream = async function * () { + assert.equal(active, false); + active = true; + try { + if (++opened === 1) { + yield Buffer.from('incomplete output'); + throw new Error('Connection reset'); + } + yield Buffer.from(failureLog); + } finally { + active = false; + } + }; + assert.equal(await jobRunner.resume(), false); + assert.equal(opened, 2); + sinon.assert.notCalled(request.fetch); + }); + + it('checks later pages of changed files', async() => { + request.json.withArgs(filesURL).resolves( + Array.from({ length: 100 }, (_, i) => ({ filename: `doc/file-${i}.md` }))); + request.json.withArgs(filesURL.replace('&page=1', '&page=2')).resolves([ + { filename: 'test/parallel/test-example.js' } + ]); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('refuses to resume if fetching changed files fails', async() => { + request.json.withArgs(filesURL).rejects(new Error('Unavailable')); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('refuses to resume if GitHub returns an error response for changed files', async() => { + request.json.withArgs(filesURL).resolves({ message: 'Not Found' }); + assert.equal(await jobRunner.resume(), false); + sinon.assert.notCalled(request.fetch); + }); + + it('allows resuming if failure logs cannot be downloaded', async() => { + request.text.rejects(new Error('Unavailable')); + assert.equal(await jobRunner.resume(), true); + sinon.assert.calledOnce(request.fetch); + }); + + it('allows resuming if failures cannot be parsed', async() => { + request.text.resolves('Unrecognized failure output'); + assert.equal(await jobRunner.resume(), true); + sinon.assert.calledOnce(request.fetch); + }); + + it('allows resuming if detailed build data cannot be downloaded', async() => { + request.json.withArgs(fullAPIURL).rejects(new Error('Unavailable')); + assert.equal(await jobRunner.resume(), true); + sinon.assert.calledOnce(request.fetch); + }); +}); + +describe('ncu-ci resume CLI', () => { + const binary = fileURLToPath(new URL('../../bin/ncu-ci.js', import.meta.url)); + const requestURL = new URL('../../lib/request.js', import.meta.url).href; + const jobURL = 'https://ci.nodejs.org/job/node-test-pull-request/654321/'; + const fullAPIURL = new PRBuild(null, null, 654321).apiUrl; + const apiURL = `${jobURL}api/json?tree=${encodeURIComponent(resumeTree)}`; + + function run(t, args, hasCI = true, changedFile = 'README.md', + buildData = resumeBuildData, headSHA = approvedSHA) { + const dir = mkdtempSync(join(tmpdir(), 'ncu-ci-resume-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + writeFileSync(join(dir, 'ncurc'), JSON.stringify({ username: 'test', token: 'test' })); + const script = ` + import assert from 'node:assert/strict'; + import Request from ${JSON.stringify(requestURL)}; + Request.prototype.gql = async (name, variables) => { + assert.deepEqual(variables, { owner: 'nodejs', repo: 'node', prid: 123456 }); + if (name === 'PR') { + return { repository: { pullRequest: { bodyText: '', createdAt: '2026-09-08' } } }; + } + if (name === 'PRComments' && ${hasCI}) { + return [{ bodyText: ${JSON.stringify(jobURL)}, publishedAt: '2026-09-09' }]; + } + return []; + }; + Request.prototype.json = async (url) => { + if (url.endsWith('/crumbIssuer/api/json')) return { crumb: 'test-crumb' }; + if (url === '/repos/nodejs/node/pulls/123456') { + return { head: { sha: ${JSON.stringify(headSHA)} } }; + } + if (url.startsWith('/repos/nodejs/node/pulls/123456/files?')) { + return [{ filename: ${JSON.stringify(changedFile)} }]; + } + if (url === ${JSON.stringify(fullAPIURL)}) return ${JSON.stringify(failureBuildData)}; + assert.equal(url, ${JSON.stringify(apiURL)}); + return ${JSON.stringify(buildData)}; + }; + Request.prototype.stream = async function * () { + yield Buffer.from(${JSON.stringify(failureLog)}); + }; + Request.prototype.fetch = async (url, options) => { + assert.equal(url, ${JSON.stringify(`${jobURL}resume`)}); + assert.equal(options.method, 'POST'); + assert.equal(options.headers['Jenkins-Crumb'], 'test-crumb'); + return { status: 200 }; + }; + process.argv = [process.execPath, ${JSON.stringify(binary)}, ...${JSON.stringify(args)}]; + await import(${JSON.stringify(new URL('../../bin/ncu-ci.js', import.meta.url).href)}); + `; + const result = spawnSync(process.execPath, ['--input-type=module', '--eval', script], { + cwd: dir, + env: { ...process.env, XDG_CONFIG_HOME: dir }, + encoding: 'utf8', + timeout: 10000 + }); + assert.ifError(result.error); + return { status: result.status, output: result.stdout + result.stderr }; + } + + for (const args of [ + ['123456', '--owner', 'nodejs', '--repo', 'node'], + ['https://github.com/nodejs/node/pull/123456'] + ]) { + it(`accepts ${args[0]}`, (t) => { + const { status, output } = run(t, ['resume', ...args]); + assert.equal(status, 0, output); + assert.match(output, /PR CI job successfully resumed/); + }); + } + + it('exits unsuccessfully when no CI run is found', (t) => { + const { status, output } = run(t, ['resume', 'https://github.com/nodejs/node/pull/123456'], + false); + assert.equal(status, 1, output); + assert.match(output, /No CI run detected from pull request 123456/); + assert.doesNotMatch(output, /TypeError/); + }); + + it('rejects invalid PR IDs', (t) => { + const { status, output } = run(t, ['resume', 'invalid', '--owner', 'nodejs', '--repo', 'node']); + assert.equal(status, 1, output); + assert.match(output, /Pull request ID must be a positive integer/); + }); + + it('requires repository information for numeric PR IDs', (t) => { + const { status, output } = run(t, ['resume', '123456']); + assert.equal(status, 1, output); + assert.match(output, /GitHub repository is missing/); + assert.match(output, /GitHub owner is missing/); + }); + + it('exits unsuccessfully when a failed test is changed by the PR', (t) => { + const { status, output } = run(t, ['resume', 'https://github.com/nodejs/node/pull/123456'], + true, 'test/parallel/test-example.js'); + assert.equal(status, 1, output); + assert.match(output, /Refusing to resume CI: failures reference files changed by this PR/); + assert.match(output, /test\/parallel\/test-example.js/); + assert.doesNotMatch(output, /PR CI job successfully resumed/); + }); + + it('resumes an aborted job when Jenkins exposes the resume action', (t) => { + const { status, output } = run(t, ['resume', 'https://github.com/nodejs/node/pull/123456'], + true, 'README.md', { ...resumeBuildData, result: 'ABORTED' }); + assert.equal(status, 0, output); + assert.match(output, /PR CI job successfully resumed/); + }); + + it('exits 1 when an aborted job is not resumable', (t) => { + const { status, output } = run(t, ['resume', 'https://github.com/nodejs/node/pull/123456'], + true, 'README.md', { ...resumeBuildData, result: 'ABORTED', actions: [] }); + assert.equal(status, 1, output); + assert.match(output, /is not resumable/); + }); + + it('exits 1 when the CI-approved commit differs from the PR HEAD', (t) => { + const { status, output } = run(t, ['resume', 'https://github.com/nodejs/node/pull/123456'], + true, 'README.md', resumeBuildData, 'b'.repeat(40)); + assert.equal(status, 1, output); + assert.match(output, /does not match the current PR HEAD/); + assert.doesNotMatch(output, /PR CI job successfully resumed/); + }); +}); From 10d6e1a4952dde8579da72bfcb09366d6ff91e3f Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Thu, 10 Sep 2026 11:05:35 +0200 Subject: [PATCH 2/3] fixup! feat: add ncu-ci resume command --- lib/cli.js | 8 +++++++- test/unit/cli.test.js | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/cli.js b/lib/cli.js index af849b73..e679f5ec 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -154,7 +154,13 @@ export default class CLI { case INFO: symbol = info; } - this.spinner.stop(`${symbol} ${rawText}`); + const text = `${symbol} ${rawText}`; + if (this.spinner.isSpinning) { + this.spinner.stop(text); + } else { + // A nested operation may already have stopped the shared spinner. + this.log(text); + } } write(text) { diff --git a/test/unit/cli.test.js b/test/unit/cli.test.js index fc13ec7d..060ac337 100644 --- a/test/unit/cli.test.js +++ b/test/unit/cli.test.js @@ -55,6 +55,13 @@ describe('cli', () => { assert.strictEqual(cli.spinner.text, 'bar'); }); + it('prints the result after a nested operation has stopped the spinner', () => { + cli.stopSpinner('Data downloaded'); + cli.stopSpinner('Refusing to resume CI', cli.SPINNER_STATUS.FAILED); + assert.ok(logResult().endsWith( + `${success} Data downloaded\n${error} Refusing to resume CI\n`)); + }); + afterEach(() => { cli.stopSpinner('stop', 'info'); }); From a8becd3676f38ba68a5b8ebd538a847f10d3129f Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Thu, 10 Sep 2026 11:12:04 +0200 Subject: [PATCH 3/3] fixup! feat: add ncu-ci resume command --- lib/ci/ci_failure_parser.js | 38 ++++++++++++++++------- lib/ci/failure_file_scanner.js | 18 +++++------ test/unit/ci_failure_file_scanner.test.js | 9 ++++-- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/lib/ci/ci_failure_parser.js b/lib/ci/ci_failure_parser.js index 4ae0677f..1fe7b2c2 100644 --- a/lib/ci/ci_failure_parser.js +++ b/lib/ci/ci_failure_parser.js @@ -134,6 +134,26 @@ function failureMatcher(Failure, patterns, ctx, text) { const fatalPattern = /fatal: .+/g; +// Shared by the whole-log parser and the streaming file scanner. +export const FAILURE_MARKERS = { + tap: { + start: /^not ok \d+/, + end: / {2}\.\.\.\r?\n/, + todo: '# TODO :' + }, + cpp: /\[ {2}FAILED {2}\]/, + git: [{ + start: /Changes not staged for commit:/, + end: /no changes added to commit/ + }, { + start: /error: Your local changes to the following files/, + end: /Failed to merge in the changes./ + }] +}; + +const tapPattern = new RegExp( + `${FAILURE_MARKERS.tap.start.source}[\\s\\S]+?${FAILURE_MARKERS.tap.end.source}`, 'mg'); + // The elements are ranked by priority const FAILURE_FILTERS = [{ // NOTE(mmarchini): infra-related issues should have the highest priority, as @@ -155,12 +175,11 @@ const FAILURE_FILTERS = [{ }, { // TODO: match indentation to avoid skipping context with '...' filter(ctx, text) { - const pattern = /^not ok \d+[\s\S]+? {2}\.\.\.\r?\n/mg; - const matches = text.match(pattern); + const matches = text.match(tapPattern); if (!matches) { return null; } - const nonFlaky = matches.filter((m) => !m.includes('# TODO :')); + const nonFlaky = matches.filter((m) => !m.includes(FAILURE_MARKERS.tap.todo)); if (!nonFlaky.length) { return null; } @@ -171,7 +190,7 @@ const FAILURE_FILTERS = [{ }, { Failure: CCTestFailure, patterns: [{ - pattern: /\[ {2}FAILED {2}\].+/g, + pattern: new RegExp(`${FAILURE_MARKERS.cpp.source}.+`, 'g'), context: { index: 0, contextBefore: 5, contextAfter: 0 } }], }, { @@ -192,15 +211,10 @@ const FAILURE_FILTERS = [{ }], }, { Failure: GitFailure, - patterns: [{ - pattern: - /Changes not staged for commit:[\s\S]+no changes added to commit/mg, - context: { index: 0, contextBefore: 0, contextAfter: 0 } - }, { - pattern: - /error: Your local changes to the following files[\s\S]+Failed to merge in the changes./g, + patterns: [...FAILURE_MARKERS.git.map(({ start, end }) => ({ + pattern: new RegExp(`${start.source}[\\s\\S]+${end.source}`, 'mg'), context: { index: 0, contextBefore: 0, contextAfter: 0 } - }, { + })), { pattern: /warning: failed to remove .+/g, context: { index: 0, contextBefore: 0, contextAfter: 0 } }], diff --git a/lib/ci/failure_file_scanner.js b/lib/ci/failure_file_scanner.js index cdb6abd6..cac3feeb 100644 --- a/lib/ci/failure_file_scanner.js +++ b/lib/ci/failure_file_scanner.js @@ -1,6 +1,6 @@ import { StringDecoder } from 'node:string_decoder'; -import { FAILURE_PATTERNS } from './ci_failure_parser.js'; +import { FAILURE_MARKERS, FAILURE_PATTERNS } from './ci_failure_parser.js'; function createMatcher(patterns) { // Preserve matching flags without sharing RegExp.lastIndex between scans. @@ -11,6 +11,8 @@ function createMatcher(patterns) { const diagnostic = createMatcher(FAILURE_PATTERNS.diagnostic); const infrastructure = createMatcher(FAILURE_PATTERNS.infrastructure); +const gitStart = createMatcher(FAILURE_MARKERS.git.map(({ start }) => start)); +const gitEnd = createMatcher(FAILURE_MARKERS.git.map(({ end }) => end)); function fileAliases(filename) { const paths = new Set([filename]); @@ -66,14 +68,12 @@ async function * failureLines(windows, matchFile) { line.file ??= matchFile(text, lineStart); line.diagnostic ||= diagnostic(text); line.infrastructure ||= infrastructure(text); - line.cpp ||= /\[ {2}FAILED {2}\]/.test(text); - line.tapStart ||= lineStart && /^not ok \d+/.test(text); - line.tapEnd ||= lineEnd && / {2}\.\.\.\n$/.test(text); - line.todo ||= text.includes('# TODO :'); - line.gitStart ||= text.includes('Changes not staged for commit:') || - text.includes('error: Your local changes to the following files'); - line.gitEnd ||= text.includes('no changes added to commit') || - text.includes('Failed to merge in the changes.'); + line.cpp ||= FAILURE_MARKERS.cpp.test(text); + line.tapStart ||= lineStart && FAILURE_MARKERS.tap.start.test(text); + line.tapEnd ||= lineEnd && FAILURE_MARKERS.tap.end.test(text); + line.todo ||= text.includes(FAILURE_MARKERS.tap.todo); + line.gitStart ||= gitStart(text); + line.gitEnd ||= gitEnd(text); if (lineEnd) { yield line; line = {}; diff --git a/test/unit/ci_failure_file_scanner.test.js b/test/unit/ci_failure_file_scanner.test.js index 0c59df5a..78a24186 100644 --- a/test/unit/ci_failure_file_scanner.test.js +++ b/test/unit/ci_failure_file_scanner.test.js @@ -29,9 +29,14 @@ describe('Streaming failure file scanner', () => { 'error C2143: src/node.cc', 'java.io.IOException: src/node.cc', 'fatal: src/node.cc', - 'ERROR: src/node.cc' + 'ERROR: src/node.cc', + 'src/node.cc:42\n[ FAILED ] Example', + tap(' severity: fail\n src/node.cc:42'), + 'Changes not staged for commit:\n modified: src/node.cc\nno changes added to commit', + 'error: Your local changes to the following files\n src/node.cc\n' + + 'Failed to merge in the changes.' ]) { - it(`reuses diagnostic patterns across parsers without regex state leaking: ${message}`, + it(`reuses failure patterns across parsers without regex state leaking: ${message}`, async() => { const log = `Build started\n${message}\n`; for (let i = 0; i < 3; i++) {