From da2df338c9b79e70fa4400f3bbb52aa5eacde329 Mon Sep 17 00:00:00 2001 From: Raoul Biagioni Date: Mon, 14 Sep 2026 08:13:24 +0100 Subject: [PATCH] feat: accept a local directory as the source repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An absolute path to an already-checked-out working tree is now a valid source argument. parseRepoUrl maps it (parent dir = owner, last path segment = name) and clone-repo copies the tree into the build dir — skipping .git, node_modules and build/cache cruft — instead of git-cloning a URL. Same downstream contract (real working tree at clonePath, a default branch, reachable: true), and the INV-21 source-identity pin still applies. No network, no credentials. This lets the pipeline run against repos with no reachable clone URL (private/internal hosts, air-gapped machines, or a folder you already have locally). --- src/orchestrator.mjs | 13 +++++ tests/local-source.test.mjs | 99 +++++++++++++++++++++++++++++++++++++ tools/clone-repo.mjs | 58 ++++++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 tests/local-source.test.mjs diff --git a/src/orchestrator.mjs b/src/orchestrator.mjs index c3b00f4..86ac610 100644 --- a/src/orchestrator.mjs +++ b/src/orchestrator.mjs @@ -49,6 +49,19 @@ const step = (n, total, label, kind) => log(`\n${C.bold}${C.cyan}[${n}/${total}] export function parseRepoUrl(raw) { let s = String(raw || '').trim(); if (!s) return null; + // LOCAL SOURCE: an absolute path to an already-checked-out working tree — no host, + // no owner (the parent dir stands in), identity = the tree's own name. Downstream (clone-repo) + // recognises the same shape and copies the tree instead of cloning a URL. + if (s.startsWith('/')) { + let u; try { u = new URL(s); } catch { /* not a file:// URL */ } + const isFilePath = !u || (u.protocol !== 'file:'); + if (isFilePath) { + const name = s.replace(/\/+$/, '').split('/').filter(Boolean).pop() || 'repo'; + const owner = s.replace(/\/+$/, '').split('/').filter(Boolean).slice(-2, -1)[0] || 'local'; + return { owner, name, url: s }; + } + s = u.pathname; + } const scp = s.match(/^git@([^:]+):(.+)$/); if (scp) s = `https://${scp[1]}/${scp[2]}`; if (!/^[a-z]+:\/\//i.test(s)) { diff --git a/tests/local-source.test.mjs b/tests/local-source.test.mjs new file mode 100644 index 0000000..e98ad5f --- /dev/null +++ b/tests/local-source.test.mjs @@ -0,0 +1,99 @@ +// tests/local-source.test.mjs — a local filesystem path as the source repo. +// +// A repo with no reachable clone URL (private infra, air-gapped, or "I have the folder, not a +// public link") must still run the pipeline: an ABSOLUTE PATH is a valid source. parseRepoUrl +// maps it (parent dir = owner, last segment = name) and clone-repo COPIES the working tree +// (skipping .git and dependency cruft) instead of cloning a URL, keeping the INV-21 identity +// pin and the same contract output (reachable: true, clonePath, slug, build.json merge). +// +// No cloud calls, no network. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseRepoUrl } from '../src/orchestrator.mjs'; + +const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +test('local source — parseRepoUrl accepts an absolute path (parent dir = owner, last segment = name)', () => { + const r = parseRepoUrl('/home/user/projects/my-service'); + assert.equal(r.name, 'my-service'); + assert.equal(r.owner, 'projects'); + assert.equal(r.url, '/home/user/projects/my-service'); +}); + +test('local source — trailing slashes and a single-segment path are tolerated', () => { + const r = parseRepoUrl('/tmp/somewhere/my-repo/'); + assert.equal(r.name, 'my-repo'); + assert.equal(r.owner, 'somewhere'); + const r2 = parseRepoUrl('/repo-only'); + assert.equal(r2.name, 'repo-only'); + assert.equal(r2.owner, 'local', 'no parent dir → the "local" stand-in'); +}); + +test('local source — ordinary URLs are routed to the normal (non-local) grammar', () => { + // A path-like string that is really a URL must not be treated as a local path. + const r = parseRepoUrl('https://github.com/owner/cool-lib'); + assert.equal(r.owner, 'owner'); + assert.equal(r.name, 'cool-lib'); + assert.match(r.url, /^https:\/\/github\.com\/owner\/cool-lib$/); +}); + +test('local source — clone-repo COPIES the working tree, skipping .git and dependency cruft', () => { + // The full local contract in one tmp tree: files arrive, cruft does not, the repo slot is the + // same shape the URL clone produces (reachable: true, clonePath, slug), and build.json is merged. + const srcRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'llsrc-')); + fs.mkdirSync(path.join(srcRoot, 'sub')); + fs.writeFileSync(path.join(srcRoot, 'README.md'), '# local repo\n'); + fs.writeFileSync(path.join(srcRoot, 'sub', 'module.py'), 'x = 1\n'); + fs.writeFileSync(path.join(srcRoot, 'sub', 'junk.pyc'), 'x'); + fs.mkdirSync(path.join(srcRoot, '.git')); + fs.writeFileSync(path.join(srcRoot, '.git', 'HEAD'), 'ref: refs/heads/main\n'); + fs.mkdirSync(path.join(srcRoot, 'node_modules', 'left-pad'), { recursive: true }); + fs.writeFileSync(path.join(srcRoot, 'node_modules', 'left-pad', 'index.js'), 'module.exports=1;'); + + const buildDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llbuild-')); + fs.writeFileSync(path.join(buildDir, 'build.json'), + JSON.stringify({ repo: { url: srcRoot } }, null, 2)); + const r = spawnSync(process.execPath, [path.join(REPO, 'tools', 'clone-repo.mjs'), buildDir], { + encoding: 'utf8', + env: { ...process.env, EXPLAINER_SUBMITTED_REPO: '' }, + }); + const out = JSON.parse(r.stdout); + assert.equal(out.ok, true, `local-source copy must succeed: ${r.stdout}\n${r.stderr}`); + const repo = out.outputs.repo; + assert.equal(repo.name, path.basename(srcRoot)); + assert.equal(repo.reachable, true, 'the contract output must be honest: a real tree now exists'); + assert.equal(repo.localSource, srcRoot); + assert.ok(fs.existsSync(path.join(buildDir, 'repo', 'README.md')), 'real files are copied'); + assert.ok(fs.existsSync(path.join(buildDir, 'repo', 'sub', 'module.py')), 'nested files are copied'); + assert.ok(!fs.existsSync(path.join(buildDir, 'repo', '.git')), '.git must never be copied (the KB corpus would index it)'); + assert.ok(!fs.existsSync(path.join(buildDir, 'repo', 'node_modules')), 'dependency cruft must not bloat the corpus'); + assert.ok(!fs.existsSync(path.join(buildDir, 'repo', 'sub', 'junk.pyc')), 'build artefacts are skipped'); + // build.json is merged (repo slot + buildId), not clobbered. + const merged = JSON.parse(fs.readFileSync(path.join(buildDir, 'build.json'), 'utf8')); + assert.equal(merged.repo.url, srcRoot); + assert.ok(merged.buildId, 'buildId must be set on a local-source run too'); +}); + +test('local source — clone-repo still enforces the INV-21 identity pin', () => { + // The source-identity guard must apply to LOCAL sources too: a swapped-in tree that does not + // match the submitted repo is refused before anything is copied. + const srcRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'llsrc-pinned-')); + fs.writeFileSync(path.join(srcRoot, 'README.md'), '# not the submitted repo\n'); + const buildDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llbuild-pinned-')); + fs.writeFileSync(path.join(buildDir, 'build.json'), + JSON.stringify({ repo: { url: srcRoot } }, null, 2)); + const r = spawnSync(process.execPath, [path.join(REPO, 'tools', 'clone-repo.mjs'), buildDir], { + encoding: 'utf8', + env: { ...process.env, EXPLAINER_SUBMITTED_REPO: 'someone/else-repo' }, + }); + assert.notEqual(r.status, 0, 'a mismatched local source must exit non-zero'); + assert.match(r.stdout, /SOURCE-IDENTITY VIOLATION/); + assert.match(r.stdout, /someone\/else-repo/, 'the refusal must name the submitted repo'); + assert.ok(!fs.existsSync(path.join(buildDir, 'repo', 'README.md')), 'nothing may be copied before the pin is verified'); +}); diff --git a/tools/clone-repo.mjs b/tools/clone-repo.mjs index 5022a99..ec8321a 100644 --- a/tools/clone-repo.mjs +++ b/tools/clone-repo.mjs @@ -45,6 +45,33 @@ function parseRepoUrl(raw) { return { host: u.host, owner, name, cloneUrl: `https://${u.host}/${owner}/${name}.git` }; } +// ---------- LOCAL SOURCE (2026-09-11): an absolute directory instead of a URL ------------------- +// repo.url may be an absolute path to an already-checked-out working tree (e.g. a Gerrit repo that +// is already cloned locally — private infra, no reachable clone URL). We copy that tree (minus .git +// and dependency/build cruft that would bloat the KB corpus) into /repo, keeping the same +// CONTRACT the downstream stations rely on: a real working tree at repo.clonePath, a default branch +// name, an honest reachable=true. No network, no credentials. +function copyLocalSource(src, dest) { + fs.mkdirSync(dest, { recursive: true }); + const SKIP = new Set(['.git', 'node_modules', 'dist', 'build', 'target', 'coverage', '.next', '.cache', '.venv', 'venv', '__pycache__']); + let files = 0; + (function walk(dir) { + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + if (SKIP.has(e.name)) continue; + const sp = path.join(dir, e.name); + const dp = path.join(dest, path.relative(src, sp)); + if (e.isDirectory()) { fs.mkdirSync(dp, { recursive: true }); walk(sp); } + else if (e.isFile()) { + if (e.name.endsWith('.o') || e.name.endsWith('.pyc') || e.name === 'package-lock.json') continue; + fs.mkdirSync(path.dirname(dp), { recursive: true }); + fs.copyFileSync(sp, dp); + files++; + } + } + })(src); + return files; +} + // ---------- git helpers (token never logged) ---------- function runGit(args, { capture = true, timeout = 120000 } = {}) { try { @@ -85,6 +112,37 @@ async function main() { const url = ctx?.repo?.url; if (!url) fail('build.json has no repo.url — clone-repo requires repo.url as its declared input'); + // LOCAL SOURCE: an absolute path to an already-checked-out working tree replaces the whole + // probe/clone half of this station (no host, no credentials, no network). Same CONTRACT output: + // a working tree at /repo + a repo slot downstream stations can rely on. + const localSrc = (!/^[a-z][a-z0-9+.-]*:\/\//i.test(String(url)) && String(url).startsWith('/') && fs.existsSync(String(url)) && fs.statSync(String(url)).isDirectory()) + ? String(url) : null; + if (localSrc) { + const name = path.basename(localSrc).replace(/\.git$/i, ''); + const owner = path.basename(path.dirname(localSrc)) || 'local'; + const pinned = (process.env.EXPLAINER_SUBMITTED_REPO || '').trim().toLowerCase(); + if (pinned && `${owner}/${name}`.toLowerCase() !== pinned) { + fail(`SOURCE-IDENTITY VIOLATION (INV-21): local source resolves to ${owner}/${name} but this build was submitted for ${pinned}.`); + } + const dest = path.join(buildDirAbs, 'repo'); + try { fs.rmSync(dest, { recursive: true, force: true }); } + catch (e) { fail(`could not clear prior clone at ${dest}: ${e.message}`); } + let files; + try { files = copyLocalSource(localSrc, dest); } + catch (e) { fail(`could not copy local source ${localSrc} -> ${dest}: ${e.message}`); } + if (files === 0) fail(`local source at ${localSrc} copied 0 files — is it an empty working tree?`); + let defaultBranch = 'main'; + const gh = runGit(['-C', localSrc, 'rev-parse', '--abbrev-ref', 'HEAD']); + if (gh.ok && gh.stdout.trim()) defaultBranch = gh.stdout.trim(); + const repoSlot = { url, owner, name, slug: name, private: true, defaultBranch, clonePath: dest, reachable: true, author: null, localSource: localSrc }; + ctx.buildId = ctx.buildId || randomUUID(); + ctx.repo = { ...(ctx.repo || {}), ...repoSlot }; + try { fs.writeFileSync(buildJsonPath, JSON.stringify(ctx, null, 2) + '\n'); } + catch (e) { fail(`could not write build.json: ${e.message}`); } + console.error(`[clone-repo] OK local source ${localSrc} -> ${dest} (${files} files, branch ${defaultBranch})`); + done({ slot: 'repo', buildId: ctx.buildId, repo: repoSlot, clonePath: dest }); + } + const parsed = parseRepoUrl(url); if (!parsed) fail(`could not parse owner/name from repo.url "${url}" (expected https://host/owner/name or git@host:owner/name)`); const { host, owner, name, cloneUrl } = parsed;