From 42763dbc39ce2940325b1b2435f652114764533a Mon Sep 17 00:00:00 2001 From: Sean Pryor Date: Wed, 23 Sep 2026 15:12:28 -0400 Subject: [PATCH] AIPCC-31724: Add airgapped prefetch bundles Add a connected prefetch phase that captures the resolved dependency graph, prepared sources, build requirements, wheels, and constraints in a checksummed bundle. Extend build-sequence to validate and consume it with network access disabled. Reject stale prepared-source workspaces, rebuild sdists from prefetched sources for post-build hooks, and exclude nested generated artifacts from source archives. Document the two-stage workflow and test bundle validation, offline behavior, and Cargo vendoring. Co-authored-by: OpenAI Codex Signed-off-by: Sean Pryor --- docs/concepts/architecture-overview.rst | 20 +- docs/concepts/bootstrap-vs-build.rst | 4 + docs/reference/files.md | 28 + docs/using.md | 27 + src/fromager/bootstrapper/_bootstrapper.py | 19 + src/fromager/bootstrapper/_build.py | 3 + src/fromager/bootstrapper/_prepare_source.py | 23 +- .../bootstrapper/_process_install_deps.py | 31 +- src/fromager/bootstrapper/_work_item.py | 3 + src/fromager/build_environment.py | 52 +- src/fromager/commands/bootstrap.py | 40 + src/fromager/commands/build.py | 267 +++++-- src/fromager/context.py | 37 + src/fromager/finders.py | 35 +- src/fromager/packagesettings/_settings.py | 16 + src/fromager/prefetch.py | 755 ++++++++++++++++++ src/fromager/sources.py | 50 +- src/fromager/tarballs.py | 21 +- src/fromager/vendor_rust.py | 38 +- tests/test_bootstrap.py | 72 ++ tests/test_bootstrapper.py | 60 ++ tests/test_bootstrapper_iterative.py | 25 + tests/test_build_environment.py | 50 ++ tests/test_commands.py | 1 + tests/test_finders.py | 15 + tests/test_prefetch.py | 597 ++++++++++++++ tests/test_sources.py | 125 +++ tests/test_vendor_rust.py | 48 ++ 28 files changed, 2365 insertions(+), 97 deletions(-) create mode 100644 src/fromager/prefetch.py create mode 100644 tests/test_prefetch.py create mode 100644 tests/test_vendor_rust.py diff --git a/docs/concepts/architecture-overview.rst b/docs/concepts/architecture-overview.rst index d81e3ab05..44fe157ca 100644 --- a/docs/concepts/architecture-overview.rst +++ b/docs/concepts/architecture-overview.rst @@ -97,6 +97,21 @@ The ``build`` command uses the same source acquisition and build subsystems but skips resolution and recursion -- it compiles a single package given a name, version, and source URL. +Prefetch Bundles +---------------- + +.. versionadded:: 0.92.0 + +``bootstrap --prefetch-dir`` separates network-enabled discovery from an +airgapped production build. It runs the sdist-only bootstrap pipeline to +dependency closure, then exports prepared source archives, bootstrap wheels, prebuilt +wheels, ``graph.json``, and ``build-order.json`` into a versioned bundle. + +``build-sequence --prefetch-dir`` verifies the bundle manifest, skips source +resolution, downloading, and source preparation, and builds from the prepared +source archives. Build-environment dependencies are installed from local wheelhouse +directories with the package index disabled. + Extension Points ---------------- @@ -126,7 +141,7 @@ breakdown. Key Data Structures ------------------- -Four data structures flow between subsystems: +Five data structures flow between subsystems: .. list-table:: :header-rows: 1 @@ -148,3 +163,6 @@ Four data structures flow between subsystems: * - ``BuildEnvironment`` - Isolated virtual environment for building one package. Created per source build, cleaned up after completion. + * - ``PrefetchBundle`` + - Verified manifest and local artifacts used to move dependency discovery + and source preparation ahead of an airgapped build. diff --git a/docs/concepts/bootstrap-vs-build.rst b/docs/concepts/bootstrap-vs-build.rst index 6ff7f4fc3..65cde486e 100644 --- a/docs/concepts/bootstrap-vs-build.rst +++ b/docs/concepts/bootstrap-vs-build.rst @@ -107,3 +107,7 @@ Typical Workflow ``build-order.json`` from bootstrap to rebuild deterministically 3. **Fixes:** Use ``build`` to rebuild individual packages after applying patches + +4. **Airgapped builds:** Use ``bootstrap --prefetch-dir`` while connected, copy + the resulting bundle into the isolated environment, then run + ``build-sequence --prefetch-dir``. diff --git a/docs/reference/files.md b/docs/reference/files.md index 117377c8c..4e118b96f 100644 --- a/docs/reference/files.md +++ b/docs/reference/files.md @@ -114,6 +114,34 @@ The following example shows `graph.json` file for the top-level dependency `whee } ``` +## Prefetch bundle + +`bootstrap --prefetch-dir` exports the files needed by a later airgapped build: + +```text +prefetch-output/ +├── manifest.json +├── graph.json +├── build-order.json +├── constraints.txt +├── merged-constraints.txt +├── sdists/ +├── wheels/ +└── prebuilt/ +``` + +`sdists/` contains source archives after `prepare_source` hooks, project +overrides, patches, and Rust vendoring have run. When a hook prepares files +beside the source root, the archive preserves those sibling directories too. +`wheels/` contains wheels needed to create build environments, and `prebuilt/` +contains packages intentionally used as binary distributions. + +`manifest.json` records the format version, build variant, Python environment, +effective settings fingerprint, package-to-artifact mapping, exact build +requirements, file sizes, and SHA-256 digests. All paths in the manifest are +relative to the bundle directory. `build-sequence --prefetch-dir` validates the +manifest before starting any builds. + ## Output Directories During the wheel building process, fromager generates multiple output directories namely `sdists-repo`, `wheels-repo` and `work-dir`. These directories contain important information related to the wheel build. diff --git a/docs/using.md b/docs/using.md index 7439ed529..a39f3225d 100644 --- a/docs/using.md +++ b/docs/using.md @@ -127,6 +127,33 @@ Production builds use separate commands for the steps described as part of bootstrapping, and accept arguments to control the servers that are used for downloading source or built wheels. +### Airgapped builds + +Use `bootstrap --prefetch-dir` to prepare a dependency graph and all source +artifacts while network access is available. Prefetch mode uses the sdist-only +bootstrap path, runs source preparation hooks, and exports prepared source +archives (including patched sdists and any prepared sibling directories), +bootstrap wheels, prebuilt wheels, the dependency graph, and the build order. + +```bash +fromager -O bootstrap-output bootstrap \ + --prefetch-dir prefetch-output \ + -r requirements.txt +``` + +Copy `prefetch-output` into the isolated environment, then build directly from +the bundle. The build validates all recorded file sizes and SHA-256 digests, +does not resolve or download sources, and installs build dependencies from the +local wheel directories using the exact versions selected during prefetch. + +```bash +fromager -O build-output build-sequence --prefetch-dir prefetch-output +``` + +The build variant and package settings must match the prefetch run. The first +implementation supports `build-sequence`; parallel bundle builds are not yet +supported. + Two commands support building wheels from source. ### The build command diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index af5df5424..bb3974fa1 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -65,6 +65,7 @@ def __init__( test_mode: bool = False, multiple_versions: bool = False, num_bg_threads: int = DEFAULT_BG_THREADS, + capture_build_requirements: bool = False, ) -> None: if test_mode and sdist_only: raise ValueError( @@ -78,6 +79,7 @@ def __init__( self.sdist_only = sdist_only self.test_mode = test_mode self.multiple_versions = multiple_versions + self.capture_build_requirements = capture_build_requirements self.why: list[tuple[RequirementType, Requirement, Version]] = [] self._num_bg_threads = max(1, num_bg_threads) self._bg_pool: concurrent.futures.ThreadPoolExecutor | None = ( @@ -699,6 +701,9 @@ def add_to_build_order( source_type: SourceType, prebuilt: bool = False, constraint: Requirement | None = None, + build_requirements: typing.Mapping[str, Version] | None = None, + prepared_source_root: str | None = None, + prepared_source_workspace: str | None = None, ) -> None: """Append a package to the build-order list if not already present. @@ -715,6 +720,11 @@ def add_to_build_order( prebuilt: True if the wheel was taken from a prebuilt cache rather than compiled locally. constraint: The version constraint active for this package, if any. + build_requirements: Exact distributions installed in the build environment. + prepared_source_root: Source root relative to ``ctx.work_dir`` when + capturing a prefetch build. + prepared_source_workspace: Source workspace relative to ``ctx.work_dir`` + when capturing a prefetch build. """ # We only care if this version of this package has been built, # and don't want to trigger building it twice. The "extras" @@ -735,6 +745,15 @@ def add_to_build_order( "source_url": source_url, "source_url_type": str(source_type), } + if build_requirements is not None: + info["build_requirements"] = [ + f"{canonicalize_name(name)}=={version}" + for name, version in sorted(build_requirements.items()) + ] + if prepared_source_root is not None: + info["prepared_source_root"] = prepared_source_root + if prepared_source_workspace is not None: + info["prepared_source_workspace"] = prepared_source_workspace self._build_stack.append(info) def _check_write_error(self, fut: concurrent.futures.Future[int]) -> None: diff --git a/src/fromager/bootstrapper/_build.py b/src/fromager/bootstrapper/_build.py index 250b99ac2..304b1f374 100644 --- a/src/fromager/bootstrapper/_build.py +++ b/src/fromager/bootstrapper/_build.py @@ -58,6 +58,9 @@ def run(self, bt: Bootstrapper) -> list[Phase]: if remaining_deps.isdisjoint(wi.build_system_deps): wi.build_env.install(remaining_deps) + if bt.capture_build_requirements: + wi.resolved_build_requirements = dict(wi.build_env.get_distributions()) + wheel_filename, sdist_filename = self.do_build(bt.ctx, bt.explain) source_type = sources.get_source_type(bt.ctx, wi.req) diff --git a/src/fromager/bootstrapper/_prepare_source.py b/src/fromager/bootstrapper/_prepare_source.py index a0ba1da73..f9f1d319b 100644 --- a/src/fromager/bootstrapper/_prepare_source.py +++ b/src/fromager/bootstrapper/_prepare_source.py @@ -28,6 +28,7 @@ def _bg_prepare_source( req: Requirement, resolved_version: Version, source_url: str, + use_wheel_cache: bool = True, ) -> PreparedSourceData: """Background-safe source download+unpack: no Bootstrapper state accessed.""" # Thread-safe: _seen_requirements in the main thread prevents the same @@ -35,14 +36,15 @@ def _bg_prepare_source( # Paths from download_source() and prepare_source() include {name}-{version}, # making them unique across concurrent threads processing different packages. logger.info("preparing source") - cached_wheel, unpacked = _cache.find_cached_wheel( - ctx, cache_wheel_server_url, req, resolved_version - ) - if unpacked is not None: - return PreparedSourceData( - sdist_root_dir=unpacked / unpacked.stem, - cached_wheel_filename=cached_wheel, + if use_wheel_cache: + cached_wheel, unpacked = _cache.find_cached_wheel( + ctx, cache_wheel_server_url, req, resolved_version ) + if unpacked is not None: + return PreparedSourceData( + sdist_root_dir=unpacked / unpacked.stem, + cached_wheel_filename=cached_wheel, + ) source_filename = sources.download_source( ctx=ctx, req=req, version=resolved_version, download_url=source_url ) @@ -96,7 +98,12 @@ def do_prepare_prebuilt() -> PreparedSourceData: def do_prepare_source() -> PreparedSourceData: with req_ctxvar_context(req, resolved_version): return _bg_prepare_source( - ctx, cache_wheel_server_url, req, resolved_version, source_url + ctx, + cache_wheel_server_url, + req, + resolved_version, + source_url, + use_wheel_cache=not bt.capture_build_requirements, ) return do_prepare_source diff --git a/src/fromager/bootstrapper/_process_install_deps.py b/src/fromager/bootstrapper/_process_install_deps.py index a6ed4b58d..064893825 100644 --- a/src/fromager/bootstrapper/_process_install_deps.py +++ b/src/fromager/bootstrapper/_process_install_deps.py @@ -149,13 +149,32 @@ def run(self, bt: Bootstrapper) -> list[Phase]: pbi = bt.ctx.package_build_info(wi.req) constraint = bt.ctx.constraints.get_constraint(wi.req.name) + build_order_kwargs: dict[str, typing.Any] = { + "req": wi.req, + "version": wi.resolved_version, + "source_url": wi.source_url, + "source_type": wi.build_result.source_type, + "prebuilt": pbi.pre_built, + "constraint": constraint, + "build_requirements": ( + wi.resolved_build_requirements + if bt.capture_build_requirements + else None + ), + } + if bt.capture_build_requirements and wi.build_result.sdist_root_dir is not None: + build_order_kwargs.update( + { + "prepared_source_root": str( + wi.build_result.sdist_root_dir.relative_to(bt.ctx.work_dir) + ), + "prepared_source_workspace": str( + wi.build_result.unpack_dir.relative_to(bt.ctx.work_dir) + ), + } + ) bt.add_to_build_order( - req=wi.req, - version=wi.resolved_version, - source_url=wi.source_url, - source_type=wi.build_result.source_type, - prebuilt=pbi.pre_built, - constraint=constraint, + **build_order_kwargs, ) dep_items: list[Phase] = bt.create_unresolved_work_items( diff --git a/src/fromager/bootstrapper/_work_item.py b/src/fromager/bootstrapper/_work_item.py index 301633948..b772f156a 100644 --- a/src/fromager/bootstrapper/_work_item.py +++ b/src/fromager/bootstrapper/_work_item.py @@ -50,6 +50,9 @@ class WorkItem: build_system_deps: set[Requirement] = dataclasses.field(default_factory=set) build_backend_deps: set[Requirement] = dataclasses.field(default_factory=set) build_sdist_deps: set[Requirement] = dataclasses.field(default_factory=set) + resolved_build_requirements: dict[str, Version] = dataclasses.field( + default_factory=dict + ) def is_build_requirement_context(self) -> bool: """Return True if this item is being processed as part of a build requirement. diff --git a/src/fromager/build_environment.py b/src/fromager/build_environment.py index 1be2fe5bd..a86d9bdc1 100644 --- a/src/fromager/build_environment.py +++ b/src/fromager/build_environment.py @@ -48,6 +48,9 @@ def __init__( self.all_reqs = all_reqs resolutions = [] for r in all_reqs: + if ctx.offline: + resolutions.append(f"{r} -> missing from prefetch bundle") + continue try: _, version = resolver.resolve( ctx=ctx, @@ -154,6 +157,17 @@ def get_venv_environ( venv_environ["UV_NO_MANAGED_PYTHON"] = "true" venv_environ["UV_PYTHON_DOWNLOADS"] = "never" venv_environ["UV_PYTHON"] = str(self.python) + if self._ctx.offline: + venv_environ["UV_OFFLINE"] = "true" + venv_environ["UV_NO_CACHE"] = "true" + venv_environ["PIP_NO_INDEX"] = "1" + venv_environ["PIP_FIND_LINKS"] = " ".join( + os.fspath(path) for path in self._ctx.wheelhouse_dirs + ) + if self._ctx.offline_constraints_file is not None: + venv_environ["PIP_CONSTRAINT"] = os.fspath( + self._ctx.offline_constraints_file + ) return venv_environ @@ -233,7 +247,9 @@ def install(self, reqs: typing.Iterable[Requirement]) -> None: self.run( cmd, cwd=str(self.path.parent), - network_isolation=False, + network_isolation=( + self._ctx.network_isolation if self._ctx.offline else False + ), ) logger.info( "installed dependencies %s into build environment in %s", @@ -340,6 +356,40 @@ def prepare_build_environment( return build_env +@metrics.timeit(description="prepare prefetched build environment") +def prepare_build_environment_from_prefetch( + *, + ctx: context.WorkContext, + req: Requirement, + sdist_root_dir: pathlib.Path, + build_requirements: typing.Iterable[Requirement], +) -> BuildEnvironment: + """Create a build environment from graph-resolved requirements. + + Args: + ctx: Offline work context. + req: Requirement being built. + sdist_root_dir: Root directory of the unpacked prepared source archive. + build_requirements: Exact requirements captured during prefetch. + + Returns: + Prepared build environment. + """ + build_env = BuildEnvironment( + ctx=ctx, + req=req, + sdist_root_dir=sdist_root_dir, + ) + _safe_install( + ctx=ctx, + req=req, + build_env=build_env, + deps=set(build_requirements), + dep_req_type=RequirementType.BUILD_SYSTEM, + ) + return build_env + + def _safe_install( ctx: context.WorkContext, req: Requirement, diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index ddc3f7d29..ee7f8a305 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -1,4 +1,5 @@ import logging +import pathlib import time import typing from datetime import timedelta @@ -12,9 +13,11 @@ from .. import ( bootstrapper, + clickext, context, dependency_graph, metrics, + prefetch, progress, requirements_file, resolver, @@ -125,6 +128,12 @@ def _get_requirements_from_args( show_default=True, help="Number of background threads for parallel I/O pre-fetching (min 1).", ) +@click.option( + "--prefetch-dir", + type=clickext.ClickPath(), + default=None, + help="Export prepared build inputs for a later offline build.", +) @click.argument("toplevel", nargs=-1) @click.pass_obj def bootstrap( @@ -138,6 +147,7 @@ def bootstrap( multiple_versions: bool, max_release_age: int | None, num_bg_threads: int, + prefetch_dir: pathlib.Path | None, toplevel: list[str], ) -> None: """Compute and build the dependencies of a set of requirements recursively @@ -147,9 +157,15 @@ def bootstrap( .. versionadded:: 0.89.0 ``--bg-threads`` option for parallel I/O pre-fetching. + + .. versionadded:: 0.92.0 + ``--prefetch-dir`` exports inputs for an offline build. """ logger.info(f"cache wheel server url: {cache_wheel_server_url}") + if prefetch_dir is not None and test_mode: + raise click.UsageError("--prefetch-dir cannot be combined with --test-mode") + to_build = _get_requirements_from_args(toplevel, requirements_files) if not to_build: raise RuntimeError( @@ -164,6 +180,24 @@ def bootstrap( logger.info("no previous bootstrap data") prev_graph = None + if prefetch_dir is not None and not sdist_only: + logger.info("prefetch mode enables sdist-only bootstrap") + sdist_only = True + + if prefetch_dir is not None: + existing_prepared_sources = sorted( + directory.name + for directory in wkctx.work_dir.iterdir() + if directory.is_dir() and (directory / directory.name).is_dir() + ) + if existing_prepared_sources: + raise click.UsageError( + "--prefetch-dir requires a clean work directory; found existing " + "prepared source workspaces: " + ", ".join(existing_prepared_sources) + ) + logger.info("prefetch mode keeps prepared source trees for bundle export") + wkctx.cleanup = False + if sdist_only: logger.info("sdist-only (fast mode), getting metadata from sdists") else: @@ -210,6 +244,7 @@ def bootstrap( test_mode=test_mode, multiple_versions=multiple_versions, num_bg_threads=num_bg_threads, + capture_build_requirements=prefetch_dir is not None, ) as bt: # Resolve and bootstrap all top-level dependencies and their transitive # dependencies. Context management and error handling are handled internally @@ -233,6 +268,10 @@ def bootstrap( f"Could not produce a pip compatible constraints file. Please review {constraints_filename} for more details" ) + if prefetch_dir is not None: + bundle = prefetch.export_prefetch_bundle(wkctx, prefetch_dir) + logger.info("wrote prefetch bundle to %s", bundle.root) + logger.debug("match_py_req LRU cache: %r", resolver.match_py_req.cache_info()) metrics.summarize(wkctx, "Bootstrapping") @@ -561,6 +600,7 @@ def bootstrap_parallel( multiple_versions=multiple_versions, max_release_age=max_release_age, num_bg_threads=num_bg_threads, + prefetch_dir=None, toplevel=toplevel, ) diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 67f96439a..7675ba23f 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -4,6 +4,7 @@ import json import logging import pathlib +import shutil import sys import threading import typing @@ -27,6 +28,7 @@ hooks, metrics, overrides, + prefetch, progress, read, server, @@ -142,27 +144,59 @@ def build( "cache_wheel_server_url", help="url to a wheel server from where fromager can check if it had already built the wheel", ) -@click.argument("build_order_file") +@click.option( + "--prefetch-dir", + type=clickext.ClickPath(exists=True, file_okay=False), + default=None, + help="Use prepared sources and wheels from a prefetch bundle.", +) +@click.argument("build_order_file", required=False) @click.pass_obj def build_sequence( wkctx: context.WorkContext, - build_order_file: str, + build_order_file: str | None, force: bool, cache_wheel_server_url: str | None, + prefetch_dir: pathlib.Path | None, ) -> None: """Build a sequence of wheels in order - BUILD_ORDER_FILE is the build-order.json files to build - - SDIST_SERVER_URL is the URL for a PyPI-compatible package index hosting sdists + BUILD_ORDER_FILE is the build-order.json file to build. It is optional when + ``--prefetch-dir`` supplies a verified build order and local artifacts. Performs the equivalent of the 'build' command for each item in the build order file. """ - server.start_wheel_server(wkctx) + prefetch_bundle: prefetch.PrefetchBundle | None = None + if prefetch_dir is not None: + if cache_wheel_server_url is not None: + raise click.UsageError( + "--cache-wheel-server-url cannot be used with --prefetch-dir" + ) + prefetch_bundle = prefetch.load_prefetch_bundle( + prefetch_dir, + expected_variant=wkctx.variant, + ) + prefetch_bundle.validate_configuration(wkctx) + wkctx.enable_offline_build( + prefetch_bundle.wheelhouse_dirs, + constraints_file=prefetch_bundle.constraints_file, + ) + wkctx.network_isolation = True + build_order_file = str(prefetch_bundle.build_order_file) + logger.info("using prefetch bundle %s", prefetch_bundle.root) + elif build_order_file is None: + raise click.UsageError("pass BUILD_ORDER_FILE or use --prefetch-dir") + + if prefetch_bundle is not None: + logger.info("using local wheel directories for offline builds") + else: + server.start_wheel_server(wkctx) - if force: + if prefetch_bundle is not None: + logger.info("rebuilding prefetched packages as needed") + elif force: logger.info( "rebuilding all wheels even if they exist in " f"{wkctx.wheel_server_url=}, {cache_wheel_server_url=}" @@ -193,6 +227,7 @@ def build_sequence( source_download_url=source_download_url, force=force, cache_wheel_server_url=cache_wheel_server_url, + prefetch_bundle=prefetch_bundle, ) if entry.prebuilt: logger.info( @@ -312,6 +347,108 @@ def _create_table( return table +def _stage_prefetched_artifact( + source: pathlib.Path, destination_dir: pathlib.Path +) -> pathlib.Path: + destination = destination_dir / source.name + destination_dir.mkdir(parents=True, exist_ok=True) + if source.resolve() != destination.resolve(): + shutil.copy2(source, destination) + return destination + + +@dataclasses.dataclass(frozen=True) +class _PreparedBuildSource: + sdist_filename: pathlib.Path + source_root_dir: pathlib.Path + build_env: build_environment.BuildEnvironment + workspace_dir: pathlib.Path | None = None + + +def _prepare_prefetched_source( + wkctx: context.WorkContext, + bundle: prefetch.PrefetchBundle, + req: Requirement, + resolved_version: Version, +) -> _PreparedBuildSource: + prefetched_sdist = bundle.package_artifact( + req.name, str(resolved_version), prebuilt=False + ) + source_root_dir, _ = sources.unpack_source( + ctx=wkctx, + req=req, + version=resolved_version, + source_filename=prefetched_sdist, + reuse_existing=False, + ) + sources.write_build_meta( + source_root_dir.parent, req, prefetched_sdist, resolved_version + ) + sources.normalize_source_timestamps(source_root_dir) + build_env = build_environment.prepare_build_environment_from_prefetch( + ctx=wkctx, + req=req, + sdist_root_dir=source_root_dir, + build_requirements=bundle.build_requirements(req.name, str(resolved_version)), + ) + sdist_filename = sources.build_sdist( + ctx=wkctx, + req=req, + version=resolved_version, + sdist_root_dir=source_root_dir, + build_env=build_env, + ) + return _PreparedBuildSource( + sdist_filename, + source_root_dir, + build_env, + workspace_dir=source_root_dir.parent, + ) + + +def _prepare_online_source( + wkctx: context.WorkContext, + req: Requirement, + resolved_version: Version, + source_download_url: str, +) -> _PreparedBuildSource: + source_filename = sources.download_source( + ctx=wkctx, + req=req, + version=resolved_version, + download_url=source_download_url, + ) + source_root_dir = sources.prepare_source( + ctx=wkctx, + req=req, + source_filename=source_filename, + version=resolved_version, + ) + build_env = build_environment.prepare_build_environment( + ctx=wkctx, + req=req, + version=resolved_version, + sdist_root_dir=source_root_dir, + ) + sdist_filename = sources.build_sdist( + ctx=wkctx, + req=req, + version=resolved_version, + sdist_root_dir=source_root_dir, + build_env=build_env, + ) + return _PreparedBuildSource(sdist_filename, source_root_dir, build_env) + + +def _clean_prepared_source( + wkctx: context.WorkContext, prepared: _PreparedBuildSource +) -> None: + if prepared.workspace_dir is not None and wkctx.cleanup: + shutil.rmtree(prepared.workspace_dir) + return + wkctx.clean_build_dirs(prepared.source_root_dir, prepared.build_env) + + def _build( wkctx: context.WorkContext, resolved_version: Version, @@ -319,6 +456,39 @@ def _build( source_download_url: str, force: bool, cache_wheel_server_url: str | None, + prefetch_bundle: prefetch.PrefetchBundle | None = None, +) -> BuildSequenceEntry: + """Build one package and always release its per-package log handler.""" + root_logger = logging.getLogger(None) + module_name = overrides.pkgname_to_override_module(req.name) + wheel_log = wkctx.logs_dir / f"{module_name}-{resolved_version}.log" + file_handler = logging.FileHandler(filename=str(wheel_log)) + file_handler.setFormatter(logging.Formatter(VERBOSE_LOG_FMT)) + file_handler.addFilter(ThreadLogFilter(threading.current_thread().name)) + root_logger.addHandler(file_handler) + try: + return _build_unlogged( + wkctx=wkctx, + resolved_version=resolved_version, + req=req, + source_download_url=source_download_url, + force=force, + cache_wheel_server_url=cache_wheel_server_url, + prefetch_bundle=prefetch_bundle, + ) + finally: + root_logger.removeHandler(file_handler) + file_handler.close() + + +def _build_unlogged( + wkctx: context.WorkContext, + resolved_version: Version, + req: Requirement, + source_download_url: str, + force: bool, + cache_wheel_server_url: str | None, + prefetch_bundle: prefetch.PrefetchBundle | None = None, ) -> BuildSequenceEntry: """Handle one version of one wheel. @@ -334,20 +504,12 @@ def _build( # We attach a handler to the root logger so that all messages are logged to # the file, and we add a filter to the handler so that only messages from # the current thread are logged for when we build in parallel. - root_logger = logging.getLogger(None) - module_name = overrides.pkgname_to_override_module(req.name) - wheel_log = wkctx.logs_dir / f"{module_name}-{resolved_version}.log" - file_handler = logging.FileHandler(filename=str(wheel_log)) - file_handler.setFormatter(logging.Formatter(VERBOSE_LOG_FMT)) - file_handler.addFilter(ThreadLogFilter(threading.current_thread().name)) - root_logger.addHandler(file_handler) - logger.info("starting processing") pbi = wkctx.package_build_info(req) prebuilt = pbi.pre_built # See if we can reuse an existing wheel. - if not force: + if not force and prefetch_bundle is None: wheel_filename = _is_wheel_built( wkctx, req.name, @@ -361,12 +523,21 @@ def _build( # Handle prebuilt wheels. if prebuilt: if not wheel_filename: - logger.info("downloading prebuilt wheel") - wheel_filename = wheels.download_wheel( - req=req, - wheel_url=source_download_url, - output_directory=wkctx.wheels_build, - ) + if prefetch_bundle is not None: + prefetched_wheel = prefetch_bundle.package_artifact( + req.name, str(resolved_version), prebuilt=True + ) + wheel_filename = _stage_prefetched_artifact( + prefetched_wheel, wkctx.wheels_build + ) + logger.info("using prefetched wheel %s", wheel_filename) + else: + logger.info("downloading prebuilt wheel") + wheel_filename = wheels.download_wheel( + req=req, + wheel_url=source_download_url, + output_directory=wkctx.wheels_build, + ) else: # already downloaded prebuilt wheel use_exiting_wheel = True @@ -384,51 +555,28 @@ def _build( # If we get here and still don't have a wheel filename, then we need to # build the wheel. if not wheel_filename: - source_filename = sources.download_source( - ctx=wkctx, - req=req, - version=resolved_version, - download_url=source_download_url, - ) + if prefetch_bundle is not None: + prepared = _prepare_prefetched_source( + wkctx, prefetch_bundle, req, resolved_version + ) + else: + prepared = _prepare_online_source( + wkctx, req, resolved_version, source_download_url + ) logger.debug( "saved sdist of version %s from %s to %s", resolved_version, source_download_url, - source_filename, - ) - - # Prepare source - source_root_dir = sources.prepare_source( - ctx=wkctx, - req=req, - source_filename=source_filename, - version=resolved_version, - ) - - # Build environment - build_env = build_environment.prepare_build_environment( - ctx=wkctx, - req=req, - version=resolved_version, - sdist_root_dir=source_root_dir, - ) - - # Make a new source distribution, in case we patched the code. - sdist_filename = sources.build_sdist( - ctx=wkctx, - req=req, - version=resolved_version, - sdist_root_dir=source_root_dir, - build_env=build_env, + prepared.sdist_filename, ) # Build wheel_filename = wheels.build_wheel( ctx=wkctx, req=req, - sdist_root_dir=source_root_dir, + sdist_root_dir=prepared.source_root_dir, version=resolved_version, - build_env=build_env, + build_env=prepared.build_env, ) hooks.run_post_build_hooks( @@ -436,14 +584,11 @@ def _build( req=req, dist_name=canonicalize_name(req.name), dist_version=str(resolved_version), - sdist_filename=sdist_filename, + sdist_filename=prepared.sdist_filename, wheel_filename=wheel_filename, ) - wkctx.clean_build_dirs(source_root_dir, build_env) - - root_logger.removeHandler(file_handler) - file_handler.close() + _clean_prepared_source(wkctx, prepared) server.update_wheel_mirror(wkctx) diff --git a/src/fromager/context.py b/src/fromager/context.py index 8ce9dd71d..443f005ef 100644 --- a/src/fromager/context.py +++ b/src/fromager/context.py @@ -94,6 +94,9 @@ def __init__( self.time_description_store: dict[str, str] = {} self._parallel_builds = False + self._offline = False + self._offline_constraints_file: pathlib.Path | None = None + self._wheelhouse_dirs: tuple[pathlib.Path, ...] = () self.cooldown: candidate.Cooldown | None = cooldown self._max_release_age: datetime.timedelta | None = max_release_age @@ -124,14 +127,48 @@ def wheels_build(self) -> pathlib.Path: @property def pip_wheel_server_args(self) -> list[str]: + if self._offline: + args = ["--no-index", "--offline"] + for directory in self.wheelhouse_dirs: + args.extend(["--find-links", os.fspath(directory)]) + return args args = ["--index-url", self.wheel_server_url] parsed = urlparse(self.wheel_server_url) if parsed.scheme != "https" and parsed.hostname: args = args + ["--trusted-host", parsed.hostname] return args + @property + def offline(self) -> bool: + """Whether builds must use only local prefetched artifacts.""" + return self._offline + + @property + def offline_constraints_file(self) -> pathlib.Path | None: + """Constraints file supplied by the prefetch bundle.""" + return self._offline_constraints_file + + @property + def wheelhouse_dirs(self) -> tuple[pathlib.Path, ...]: + """Directories available for offline wheel installation.""" + return (self.wheels_downloads, *self._wheelhouse_dirs) + + def enable_offline_build( + self, + wheelhouse_dirs: typing.Iterable[pathlib.Path], + constraints_file: pathlib.Path | None = None, + ) -> None: + """Use local wheel directories for dependency installation.""" + self._offline = True + self._offline_constraints_file = ( + constraints_file.resolve() if constraints_file is not None else None + ) + self._wheelhouse_dirs = tuple(path.resolve() for path in wheelhouse_dirs) + @property def pip_constraint_args(self) -> list[str]: + if self._offline_constraints_file is not None: + return ["--constraint", os.fspath(self._offline_constraints_file)] if not self.constraints: return [] return ["--constraint", os.fspath(self.merged_constraints)] diff --git a/src/fromager/finders.py b/src/fromager/finders.py index 69c31e331..11bf46d7f 100644 --- a/src/fromager/finders.py +++ b/src/fromager/finders.py @@ -6,7 +6,13 @@ import typing from packaging.requirements import Requirement -from packaging.utils import BuildTag, canonicalize_name +from packaging.utils import ( + BuildTag, + InvalidWheelFilename, + canonicalize_name, + parse_wheel_filename, +) +from packaging.version import Version from . import overrides, resolver from .constraints import Constraints @@ -195,6 +201,33 @@ def find_wheel( return None +def find_exact_wheel( + download_dirs: typing.Iterable[pathlib.Path], + req: Requirement, + dist_version: str, + build_tag: BuildTag = (), +) -> pathlib.Path | None: + """Find a wheel matching an exact name, version, and build tag.""" + expected_name = canonicalize_name(req.name) + expected_version = Version(dist_version) + expected_build_tag = build_tag or (0, "") + for directory in download_dirs: + for candidate in sorted(directory.glob("*.whl")): + try: + name, version, candidate_build_tag, _ = parse_wheel_filename( + candidate.name + ) + except InvalidWheelFilename: + continue + if ( + name == expected_name + and version == expected_version + and (candidate_build_tag or (0, "")) == expected_build_tag + ): + return candidate + return None + + def default_expected_source_directory_name(req: Requirement, dist_version: str) -> str: raise NotImplementedError diff --git a/src/fromager/packagesettings/_settings.py b/src/fromager/packagesettings/_settings.py index 888e6b225..5b491140d 100644 --- a/src/fromager/packagesettings/_settings.py +++ b/src/fromager/packagesettings/_settings.py @@ -225,6 +225,22 @@ def list_pre_built(self) -> set[Package]: if self.package_build_info(name).pre_built ) + def configuration_data( + self, packages: typing.Iterable[str] + ) -> dict[str, typing.Any]: + """Return stable configuration data for selected packages.""" + package_names = {Package(canonicalize_name(name)) for name in packages} + return { + "global": self._settings.model_dump(mode="json"), + "max_jobs": self.max_jobs, + "packages": { + str(name): settings.model_dump(mode="json") + for name, settings in sorted(self._package_settings.items()) + if settings.has_config and name in package_names + }, + "variant": self.variant, + } + def list_overrides(self) -> set[Package]: """List packages with overrides diff --git a/src/fromager/prefetch.py b/src/fromager/prefetch.py new file mode 100644 index 000000000..07bb20d6e --- /dev/null +++ b/src/fromager/prefetch.py @@ -0,0 +1,755 @@ +"""Create and consume self-contained prefetch bundles.""" + +from __future__ import annotations + +import dataclasses +import hashlib +import importlib.metadata +import json +import os +import pathlib +import platform +import shutil +import sys +import sysconfig +import tarfile +import tempfile +import typing + +import pydantic +from packaging.requirements import Requirement +from packaging.utils import ( + InvalidSdistFilename, + canonicalize_name, + parse_sdist_filename, +) +from packaging.version import Version + +from . import context, downloads, finders, tarballs + +MANIFEST_FILENAME = "manifest.json" +FORMAT_VERSION: typing.Literal[1] = 1 +_GENERATED_SOURCE_FILES = frozenset( + { + "build-backend-requirements.txt", + "build-meta.json", + "build-sdist-requirements.txt", + "build-system-requirements.txt", + "requirements.txt", + } +) +_GENERATED_SOURCE_DIRECTORIES = frozenset( + {"__pycache__", "_skbuild", "build", "target"} +) + + +def _exact_requirement_coordinate(requirement: str) -> tuple[str, str]: + parsed = Requirement(requirement) + specifiers = list(parsed.specifier) + if ( + parsed.url + or len(specifiers) != 1 + or specifiers[0].operator != "==" + or "*" in specifiers[0].version + ): + raise ValueError(f"build requirement is not an exact pin: {requirement}") + return canonicalize_name(parsed.name), specifiers[0].version + + +class PrefetchArtifact(pydantic.BaseModel): + """Describe one file stored in a prefetch bundle.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + path: str + sha256: str = pydantic.Field(pattern=r"^[0-9a-f]{64}$") + size: int = pydantic.Field(ge=0) + + @pydantic.field_validator("path") + @classmethod + def validate_path(cls, value: str) -> str: + """Require a safe path relative to the bundle root.""" + path = pathlib.PurePosixPath(value) + if not value or path.is_absolute() or ".." in path.parts: + raise ValueError(f"invalid prefetch artifact path {value!r}") + return value + + +class PrefetchPackage(pydantic.BaseModel): + """Describe the prepared artifact for one package build.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + dist: str + version: str + prebuilt: bool + artifact: PrefetchArtifact + build_requirements: tuple[str, ...] = () + + @pydantic.model_validator(mode="after") + def validate_artifact_type(self) -> PrefetchPackage: + """Require a wheel for prebuilt packages and a source archive otherwise.""" + if self.prebuilt and not self.artifact.path.endswith(".whl"): + raise ValueError(f"prebuilt package {self.dist} does not reference a wheel") + if not self.prebuilt and not self.artifact.path.endswith((".tar.gz", ".zip")): + raise ValueError( + f"source package {self.dist} does not reference a source archive" + ) + for requirement in self.build_requirements: + _exact_requirement_coordinate(requirement) + return self + + +class PrefetchWheel(pydantic.BaseModel): + """Describe an exact wheel available to offline build environments.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + dist: str + version: str + artifact: PrefetchArtifact + + +class PrefetchManifest(pydantic.BaseModel): + """Describe all inputs stored in a prefetch bundle.""" + + model_config = pydantic.ConfigDict(extra="forbid", frozen=True) + + format_version: typing.Literal[1] = FORMAT_VERSION + fromager_version: str + python_version: str + python_implementation: str + platform: str + variant: str + settings_fingerprint: str = pydantic.Field(pattern=r"^[0-9a-f]{64}$") + graph: PrefetchArtifact + build_order: PrefetchArtifact + constraints: PrefetchArtifact | None = None + merged_constraints: PrefetchArtifact | None = None + packages: tuple[PrefetchPackage, ...] + wheels: tuple[PrefetchWheel, ...] + + @pydantic.model_validator(mode="after") + def validate_package_keys(self) -> PrefetchManifest: + """Reject duplicate package and version records.""" + seen: set[tuple[str, str]] = set() + for package in self.packages: + key = (canonicalize_name(package.dist), package.version) + if key in seen: + raise ValueError(f"duplicate prefetch package {key[0]}=={key[1]}") + seen.add(key) + return self + + +@dataclasses.dataclass(frozen=True) +class PrefetchBundle: + """Provide validated access to a loaded prefetch bundle.""" + + root: pathlib.Path + manifest: PrefetchManifest + _verified: set[str] = dataclasses.field( + default_factory=set, + init=False, + repr=False, + compare=False, + ) + + @property + def build_order_file(self) -> pathlib.Path: + """Return the validated build-order file.""" + return self.artifact_path(self.manifest.build_order) + + @property + def graph_file(self) -> pathlib.Path: + """Return the validated dependency graph file.""" + return self.artifact_path(self.manifest.graph) + + @property + def constraints_file(self) -> pathlib.Path | None: + """Return input constraints captured from the prefetch run, when present.""" + artifact = self.manifest.merged_constraints + return self.artifact_path(artifact) if artifact is not None else None + + @property + def wheelhouse_dirs(self) -> tuple[pathlib.Path, ...]: + """Return local directories containing prefetched wheels.""" + directories = { + self.artifact_path(wheel.artifact).parent for wheel in self.manifest.wheels + } + return tuple(sorted(directories)) + + @property + def package_coordinates(self) -> tuple[tuple[str, str], ...]: + """Return package names and versions recorded in the bundle.""" + return tuple( + (package.dist, package.version) for package in self.manifest.packages + ) + + def artifact_path(self, artifact: PrefetchArtifact) -> pathlib.Path: + """Resolve an artifact path inside the bundle.""" + path = (self.root / artifact.path).resolve() + if not path.is_relative_to(self.root): + raise ValueError(f"prefetch artifact escapes bundle root: {artifact.path}") + return path + + def package_artifact( + self, + dist: str, + version: str, + *, + prebuilt: bool, + ) -> pathlib.Path: + """Return the prepared artifact for a package build.""" + name = canonicalize_name(dist) + for package in self.manifest.packages: + if ( + canonicalize_name(package.dist) == name + and package.version == version + and package.prebuilt == prebuilt + ): + return self.verify_artifact(package.artifact) + raise KeyError( + f"prefetch bundle has no {'wheel' if prebuilt else 'source archive'} " + f"for {name}=={version}" + ) + + def build_requirements(self, dist: str, version: str) -> set[Requirement]: + """Return exact requirements for one package's build environment.""" + package = self._package(dist, version) + requirements = {Requirement(value) for value in package.build_requirements} + for requirement in requirements: + self._verify_wheel(requirement) + return requirements + + def verify_artifact(self, artifact: PrefetchArtifact) -> pathlib.Path: + """Verify and return one artifact path.""" + filename = self.artifact_path(artifact) + if artifact.path in self._verified: + return filename + if not filename.is_file(): + raise FileNotFoundError(f"prefetch artifact is missing: {filename}") + if filename.stat().st_size != artifact.size: + raise ValueError(f"prefetch artifact has wrong size: {filename}") + if _sha256(filename) != artifact.sha256: + raise ValueError(f"prefetch artifact has wrong SHA-256: {filename}") + self._verified.add(artifact.path) + return filename + + def validate_configuration(self, ctx: context.WorkContext) -> None: + """Verify that build settings match the prefetch environment.""" + actual = configuration_fingerprint(ctx, self.package_coordinates) + if self.manifest.settings_fingerprint != actual: + raise ValueError( + "prefetch settings and patches do not match the build configuration" + ) + + def _package(self, dist: str, version: str) -> PrefetchPackage: + name = canonicalize_name(dist) + for package in self.manifest.packages: + if canonicalize_name(package.dist) == name and package.version == version: + return package + raise KeyError(f"prefetch bundle has no package {name}=={version}") + + def _verify_wheel(self, requirement: Requirement) -> pathlib.Path: + name, version = _exact_requirement_coordinate(str(requirement)) + for wheel in self.manifest.wheels: + if canonicalize_name(wheel.dist) == name and wheel.version == version: + return self.verify_artifact(wheel.artifact) + raise KeyError(f"prefetch bundle has no wheel for {name}=={version}") + + +def _sha256(filename: pathlib.Path) -> str: + digest = hashlib.sha256() + with filename.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def configuration_fingerprint( + ctx: context.WorkContext, + packages: typing.Iterable[tuple[str, str]], +) -> str: + """Fingerprint settings, patches, and plugins that influence build output.""" + digest = hashlib.sha256() + package_versions = tuple(sorted(packages)) + package_names = {canonicalize_name(name) for name, _ in package_versions} + plugin_data: list[dict[str, str]] = [] + for group in ("fromager.hooks", "fromager.project_overrides"): + for entry_point in importlib.metadata.entry_points(group=group): + if ( + group == "fromager.project_overrides" + and canonicalize_name(entry_point.name) not in package_names + ): + continue + distribution = entry_point.dist + plugin_data.append( + { + "distribution": distribution.name if distribution else "", + "group": group, + "name": entry_point.name, + "value": entry_point.value, + "version": distribution.version if distribution else "", + } + ) + configuration = { + "plugins": sorted( + plugin_data, + key=lambda value: ( + value["group"], + value["name"], + value["distribution"], + value["version"], + value["value"], + ), + ), + "settings": ctx.settings.configuration_data(package_names), + } + settings_data = json.dumps( + configuration, + sort_keys=True, + separators=(",", ":"), + ) + digest.update(settings_data.encode("utf-8")) + + patches_dir = ctx.settings.patches_dir.resolve() + patches: set[pathlib.Path] = set() + for name, version in package_versions: + patches.update(ctx.package_build_info(name).get_patches(Version(version))) + for patch in sorted(patches): + patch = patch.resolve() + digest.update(patch.relative_to(patches_dir).as_posix().encode("utf-8")) + digest.update(_sha256(patch).encode("ascii")) + return digest.hexdigest() + + +def _fromager_version() -> str: + try: + return importlib.metadata.version("fromager") + except importlib.metadata.PackageNotFoundError: + return "unknown" + + +def _artifact_record(root: pathlib.Path, filename: pathlib.Path) -> PrefetchArtifact: + relative = filename.resolve().relative_to(root).as_posix() + return PrefetchArtifact( + path=relative, + sha256=_sha256(filename), + size=filename.stat().st_size, + ) + + +def _copy_artifact( + root: pathlib.Path, + source: pathlib.Path, + relative_destination: pathlib.Path, +) -> PrefetchArtifact: + destination = root / relative_destination + destination.parent.mkdir(parents=True, exist_ok=True) + resolved_parent = destination.parent.resolve() + if not resolved_parent.is_relative_to(root): + raise ValueError(f"prefetch destination escapes bundle root: {destination}") + destination = resolved_parent / destination.name + if destination.is_symlink(): + raise ValueError(f"prefetch destination is a symlink: {destination}") + if source.resolve() == destination.resolve(): + return _artifact_record(root, destination) + + digest = hashlib.sha256() + size = 0 + temporary_path: pathlib.Path | None = None + try: + with ( + source.open("rb") as source_file, + tempfile.NamedTemporaryFile( + dir=resolved_parent, + prefix=f".{destination.name}.", + delete=False, + ) as temporary_file, + ): + temporary_path = pathlib.Path(temporary_file.name) + for chunk in iter(lambda: source_file.read(1024 * 1024), b""): + temporary_file.write(chunk) + digest.update(chunk) + size += len(chunk) + temporary_file.flush() + os.fsync(temporary_file.fileno()) + shutil.copystat(source, temporary_path, follow_symlinks=False) + temporary_path.replace(destination) + except BaseException: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + raise + + return PrefetchArtifact( + path=destination.relative_to(root).as_posix(), + sha256=digest.hexdigest(), + size=size, + ) + + +def _write_manifest(root: pathlib.Path, manifest: PrefetchManifest) -> None: + destination = root / MANIFEST_FILENAME + if destination.is_symlink(): + raise ValueError(f"prefetch manifest is a symlink: {destination}") + temporary_path: pathlib.Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=root, + prefix=f".{MANIFEST_FILENAME}.", + delete=False, + ) as temporary_file: + temporary_path = pathlib.Path(temporary_file.name) + temporary_file.write(manifest.model_dump_json(indent=2) + "\n") + temporary_file.flush() + os.fsync(temporary_file.fileno()) + temporary_path.replace(destination) + except BaseException: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + raise + + +def _load_build_order(filename: pathlib.Path) -> list[dict[str, typing.Any]]: + try: + value = json.loads(filename.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as err: + raise ValueError(f"could not read build order {filename}: {err}") from err + if not isinstance(value, list) or not all( + isinstance(entry, dict) for entry in value + ): + raise ValueError(f"build order {filename} is not a list of objects") + return typing.cast(list[dict[str, typing.Any]], value) + + +def _find_built_sdist( + ctx: context.WorkContext, req: Requirement, version: str +) -> pathlib.Path | None: + source = finders.find_sdist(ctx, ctx.sdists_builds, req, version) + if source is not None: + return source + + expected_name = canonicalize_name(req.name) + expected_version = Version(version) + for candidate in sorted(ctx.sdists_builds.glob("*")): + try: + name, candidate_version = parse_sdist_filename(candidate.name) + except InvalidSdistFilename: + continue + if name == expected_name and candidate_version == expected_version: + return candidate + return None + + +def _source_workspace( + ctx: context.WorkContext, source_root: str, workspace: str +) -> tuple[pathlib.Path, pathlib.Path]: + source_root_path = (ctx.work_dir / pathlib.PurePosixPath(source_root)).resolve() + workspace_path = (ctx.work_dir / pathlib.PurePosixPath(workspace)).resolve() + if not workspace_path.is_relative_to(ctx.work_dir): + raise ValueError( + f"prepared source workspace escapes work directory: {workspace}" + ) + if not source_root_path.is_relative_to(workspace_path): + raise ValueError(f"prepared source root escapes work directory: {source_root}") + if not source_root_path.is_dir(): + raise FileNotFoundError( + f"prepared source workspace is missing: {source_root_path}" + ) + return workspace_path, source_root_path + + +def _is_generated_source_entry( + package_dir: pathlib.Path, source_root: pathlib.Path, path: pathlib.Path +) -> bool: + if ( + path.parent == package_dir + and path != source_root + and ( + path.name.startswith("build-") + or path.name in _GENERATED_SOURCE_FILES + or path.suffix == ".log" + ) + ): + return True + + try: + relative_path = path.relative_to(source_root) + except ValueError: + return False + + current_path = source_root + for part in relative_path.parts: + current_path /= part + if part.endswith(".egg-info") or current_path.suffix in {".pyc", ".pyo"}: + return True + if part in _GENERATED_SOURCE_DIRECTORIES and current_path.is_dir(): + return True + return False + + +def _write_prepared_source_archive( + *, + root: pathlib.Path, + ctx: context.WorkContext, + source_root: str, + workspace: str, + filename: str, +) -> pathlib.Path: + package_dir, source_root_path = _source_workspace(ctx, source_root, workspace) + + destination_dir = root / "sdists" + destination_dir.mkdir(parents=True, exist_ok=True) + archive_filename = ( + f"{filename[:-4]}.tar.gz" if filename.endswith(".zip") else filename + ) + destination = destination_dir / archive_filename + if destination.is_symlink(): + raise ValueError(f"prefetch destination is a symlink: {destination}") + temporary_path: pathlib.Path | None = None + try: + with tempfile.NamedTemporaryFile( + dir=destination_dir, + prefix=f".{destination.name}.", + delete=False, + ) as temporary_file: + temporary_path = pathlib.Path(temporary_file.name) + with tarfile.open(temporary_path, "w:gz", format=tarfile.PAX_FORMAT) as archive: + tarballs.tar_reproducible( + tar=archive, + basedir=package_dir, + prefix=package_dir, + exclude_vcs=True, + include_basedir=False, + exclude=lambda path: _is_generated_source_entry( + package_dir, source_root_path, path + ), + ) + temporary_path.replace(destination) + except BaseException: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + raise + return destination + + +def export_prefetch_bundle( + ctx: context.WorkContext, + destination_dir: pathlib.Path, +) -> PrefetchBundle: + """Export bootstrap outputs as a self-contained prefetch bundle. + + Args: + ctx: Completed bootstrap work context. + destination_dir: Directory to populate with the bundle. + + Returns: + The validated bundle. + """ + root = destination_dir.resolve() + root.mkdir(parents=True, exist_ok=True) + + build_order_source = ctx.work_dir / "build-order.json" + graph_source = ctx.graph_file + build_order = _load_build_order(build_order_source) + + graph = _copy_artifact(root, graph_source, pathlib.Path("graph.json")) + build_order_artifact = _copy_artifact( + root, build_order_source, pathlib.Path("build-order.json") + ) + + constraints_source = ctx.work_dir / "constraints.txt" + constraints = ( + _copy_artifact(root, constraints_source, pathlib.Path("constraints.txt")) + if constraints_source.is_file() + else None + ) + merged_constraints_source = ctx.merged_constraints + merged_constraints = ( + _copy_artifact( + root, + merged_constraints_source, + pathlib.Path("merged-constraints.txt"), + ) + if merged_constraints_source.is_file() + else None + ) + + packages: list[PrefetchPackage] = [] + required_wheels: set[tuple[str, str]] = set() + for entry in build_order: + try: + dist = str(entry["dist"]) + version = str(entry["version"]) + prebuilt = bool(entry["prebuilt"]) + source_url = str(entry["source_url"]) + raw_build_requirements = entry["build_requirements"] + except KeyError as err: + raise ValueError(f"build-order entry is missing {err.args[0]!r}") from err + if not isinstance(raw_build_requirements, list) or not all( + isinstance(requirement, str) for requirement in raw_build_requirements + ): + raise ValueError(f"invalid build requirements for {dist}=={version}") + build_requirements = tuple(sorted(set(raw_build_requirements))) + + req = Requirement(f"{dist}=={version}") + source: pathlib.Path | None + if prebuilt: + expected_source = ctx.wheels_prebuilt / downloads.extract_filename_from_url( + source_url + ) + source = expected_source if expected_source.is_file() else None + if source is None: + source = finders.find_wheel(ctx.wheels_prebuilt, req, version) + subdir = pathlib.Path("prebuilt") + else: + source = _find_built_sdist(ctx, req, version) + subdir = pathlib.Path("sdists") + if source is None: + kind = "prebuilt wheel" if prebuilt else "patched sdist" + raise FileNotFoundError(f"could not find {kind} for {req}") + + node_key = f"{canonicalize_name(dist)}=={version}" + if node_key not in ctx.dependency_graph.nodes: + raise ValueError(f"build order references missing graph node {node_key}") + required_wheels.update( + _exact_requirement_coordinate(requirement) + for requirement in build_requirements + ) + + if not prebuilt: + prepared_source_root = entry.get("prepared_source_root") + prepared_source_workspace = entry.get("prepared_source_workspace") + if not isinstance(prepared_source_root, str) or not isinstance( + prepared_source_workspace, str + ): + raise ValueError( + f"build-order entry has no prepared source workspace for {req}" + ) + prepared_source = _write_prepared_source_archive( + root=root, + ctx=ctx, + source_root=prepared_source_root, + workspace=prepared_source_workspace, + filename=source.name, + ) + artifact = _artifact_record(root, prepared_source) + else: + artifact = _copy_artifact(root, source, subdir / source.name) + packages.append( + PrefetchPackage( + dist=dist, + version=version, + prebuilt=prebuilt, + artifact=artifact, + build_requirements=build_requirements, + ) + ) + + wheel_records: dict[str, PrefetchWheel] = {} + for dist, version in sorted(required_wheels): + req = Requirement(f"{dist}=={version}") + build_tag = ctx.package_build_info(req).build_tag(Version(version)) + source = finders.find_exact_wheel( + (ctx.wheels_downloads, ctx.wheels_prebuilt), + req, + version, + build_tag, + ) + if source is None: + raise FileNotFoundError( + f"could not find build-environment wheel for {dist}=={version}" + ) + wheel_subdir = pathlib.Path( + "prebuilt" if source.parent == ctx.wheels_prebuilt else "wheels" + ) + artifact = _copy_artifact(root, source, wheel_subdir / source.name) + wheel_records[artifact.path] = PrefetchWheel( + dist=dist, + version=version, + artifact=artifact, + ) + + manifest = PrefetchManifest( + fromager_version=_fromager_version(), + python_version=platform.python_version(), + python_implementation=sys.implementation.name, + platform=sysconfig.get_platform(), + variant=ctx.variant, + settings_fingerprint=configuration_fingerprint( + ctx, ((package.dist, package.version) for package in packages) + ), + graph=graph, + build_order=build_order_artifact, + constraints=constraints, + merged_constraints=merged_constraints, + packages=tuple(packages), + wheels=tuple(wheel_records[path] for path in sorted(wheel_records)), + ) + _write_manifest(root, manifest) + return load_prefetch_bundle(root, expected_variant=ctx.variant) + + +def load_prefetch_bundle( + directory: pathlib.Path, + *, + expected_variant: str | None = None, + verify_environment: bool = True, +) -> PrefetchBundle: + """Load and verify a prefetch bundle. + + Args: + directory: Bundle directory containing ``manifest.json``. + expected_variant: Required variant, when supplied. + verify_environment: Verify the current Python and platform. + + Returns: + The verified bundle. + """ + root = directory.resolve() + manifest_path = root / MANIFEST_FILENAME + try: + manifest = PrefetchManifest.model_validate_json( + manifest_path.read_text(encoding="utf-8") + ) + except (OSError, pydantic.ValidationError) as err: + raise ValueError( + f"could not load prefetch manifest {manifest_path}: {err}" + ) from err + + if expected_variant is not None and manifest.variant != expected_variant: + raise ValueError( + f"prefetch variant {manifest.variant!r} does not match " + f"build variant {expected_variant!r}" + ) + + if verify_environment: + expected_environment = { + "fromager_version": _fromager_version(), + "python_version": platform.python_version(), + "python_implementation": sys.implementation.name, + "platform": sysconfig.get_platform(), + } + for field, expected in expected_environment.items(): + actual = getattr(manifest, field) + if actual != expected: + raise ValueError( + f"prefetch {field} {actual!r} does not match " + f"build environment {expected!r}" + ) + + bundle = PrefetchBundle(root=root, manifest=manifest) + artifacts: list[PrefetchArtifact] = [manifest.graph, manifest.build_order] + if manifest.constraints is not None: + artifacts.append(manifest.constraints) + if manifest.merged_constraints is not None: + artifacts.append(manifest.merged_constraints) + artifacts.extend(package.artifact for package in manifest.packages) + artifacts.extend(wheel.artifact for wheel in manifest.wheels) + + for artifact in artifacts: + bundle.verify_artifact(artifact) + + return bundle diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 5409a181f..581682f73 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -3,8 +3,10 @@ import inspect import json import logging +import os import pathlib import shutil +import stat import tarfile import typing import warnings @@ -39,6 +41,10 @@ logger = logging.getLogger(__name__) +# ZIP stores local timestamps and cannot represent dates before 1980. Use the +# following day so this remains valid in timezones west of UTC. +ZIP_SAFE_EPOCH = 315619200 + def download_url( *, @@ -250,11 +256,34 @@ def _takes_arg(f: typing.Callable, arg_name: str) -> bool: return arg_name in sig.parameters +def _extract_zip_safely(archive: zipfile.ZipFile, destination: pathlib.Path) -> None: + """Extract a ZIP archive without allowing path traversal or symlinks.""" + destination = destination.resolve() + for member in archive.infolist(): + member_path = pathlib.PurePosixPath(member.filename) + if member_path.is_absolute() or ".." in member_path.parts: + raise ValueError(f"unsafe path in source archive: {member.filename}") + target = (destination / pathlib.Path(*member_path.parts)).resolve() + if not target.is_relative_to(destination): + raise ValueError(f"unsafe path in source archive: {member.filename}") + mode = (member.external_attr >> 16) & 0o170000 + if stat.S_ISLNK(mode): + raise ValueError(f"symlink in source archive: {member.filename}") + if member.is_dir(): + target.mkdir(parents=True, exist_ok=True) + continue + target.parent.mkdir(parents=True, exist_ok=True) + with archive.open(member) as source, target.open("wb") as output: + shutil.copyfileobj(source, output) + + def unpack_source( ctx: context.WorkContext, req: Requirement, version: Version, source_filename: pathlib.Path, + *, + reuse_existing: bool = True, ) -> tuple[pathlib.Path, bool]: """Extracts a downloaded source archive (.tar.gz or .zip file) into a standardized directory.""" # sdist names are less standardized and the names of the directories they @@ -267,7 +296,7 @@ def unpack_source( # build process, including the unpacked source in a subdirectory. unpack_dir = ctx.work_dir / expected_name if unpack_dir.exists(): - if ctx.cleanup: + if ctx.cleanup or not reuse_existing: logger.debug("cleaning up %s", unpack_dir) shutil.rmtree(unpack_dir) else: @@ -285,7 +314,7 @@ def unpack_source( t.extractall(unpack_dir) elif str(source_filename).endswith(".zip"): with zipfile.ZipFile(source_filename) as zf: - zf.extractall(path=unpack_dir) + _extract_zip_safely(zf, unpack_dir) else: raise ValueError(f"Do not know how to unpack source archive {source_filename}") @@ -293,8 +322,14 @@ def unpack_source( # not be the same name as the root directory of the content in the sdist # (due to case, punctuation, etc.), so after we unpack it look for what was # created and ensure the extracted directory matches the override module - # name and version of the requirement. - unpacked_root_dir = next(iter(unpack_dir.glob("*"))) + # name and version of the requirement. Prepared-source archives may also + # contain sibling directories, so prefer the expected root when present. + expected_root_dir = unpack_dir / expected_name + unpacked_root_dir = ( + expected_root_dir + if expected_root_dir.is_dir() + else next(iter(unpack_dir.glob("*"))) + ) if unpacked_root_dir.name != expected_name: desired_name = unpacked_root_dir.parent / expected_name try: @@ -311,6 +346,13 @@ def unpack_source( return (unpacked_root_dir, True) +def normalize_source_timestamps(source_root_dir: pathlib.Path) -> None: + """Set source mtimes to a reproducible value accepted by ZIP files.""" + paths = [source_root_dir, *source_root_dir.rglob("*")] + for path in paths: + os.utime(path, (ZIP_SAFE_EPOCH, ZIP_SAFE_EPOCH), follow_symlinks=False) + + def patch_source( ctx: context.WorkContext, source_root_dir: pathlib.Path, diff --git a/src/fromager/tarballs.py b/src/fromager/tarballs.py index d425a2283..3aa5a92e1 100644 --- a/src/fromager/tarballs.py +++ b/src/fromager/tarballs.py @@ -4,6 +4,7 @@ import pathlib import stat import tarfile +import typing VCS_DIRS = {".bzr", ".git", ".hg", ".svn"} @@ -30,6 +31,8 @@ def tar_reproducible( prefix: pathlib.Path | None = None, *, exclude_vcs: bool = False, + exclude: typing.Callable[[pathlib.Path], bool] | None = None, + include_basedir: bool = True, ) -> None: """Create reproducible tar file @@ -38,14 +41,30 @@ def tar_reproducible( If ``exclude_vcs`` is True, then Bazaar, git, Mercurial, and subversion directories and files are excluded. + + ``exclude`` can remove additional paths from the archive. + + If ``include_basedir`` is false, archive the contents of ``basedir`` + without adding the directory itself. """ - content = [str(basedir)] # convert from pathlib.Path, if that's what we have + content = [str(basedir)] if include_basedir else [] for root, dirs, files in os.walk(basedir): if exclude_vcs: # modify lists in-place, so os.walk does not descent into the # excluded entries. git submodules have a `.git` file. dirs[:] = [directory for directory in dirs if directory not in VCS_DIRS] files[:] = [filename for filename in files if filename not in VCS_DIRS] + if exclude is not None: + dirs[:] = [ + directory + for directory in dirs + if not exclude(pathlib.Path(root) / directory) + ] + files[:] = [ + filename + for filename in files + if not exclude(pathlib.Path(root) / filename) + ] for directory in dirs: content.append(os.path.join(root, directory)) for filename in files: diff --git a/src/fromager/vendor_rust.py b/src/fromager/vendor_rust.py index d88268a5b..0f6ffef66 100644 --- a/src/fromager/vendor_rust.py +++ b/src/fromager/vendor_rust.py @@ -16,13 +16,6 @@ VENDOR_DIR = "vendor" -CARGO_CONFIG = { - "source": { - "crates-io": {"replace-with": "vendored-sources"}, - "vendored-sources": {"directory": VENDOR_DIR}, - }, -} - class RustBuildSystem(enum.StrEnum): maturin = "maturin" @@ -33,15 +26,30 @@ def _cargo_vendor( req: Requirement, manifests: list[pathlib.Path], project_dir: pathlib.Path, -) -> typing.Iterable[pathlib.Path]: +) -> tuple[typing.Iterable[pathlib.Path], tomlkit.TOMLDocument]: """Run cargo vendor""" logger.info(f"updating vendored rust dependencies in {project_dir}") args = ["cargo", "vendor", f"--manifest-path={manifests[0]}"] for manifest in manifests[1:]: args.append(f"--sync={manifest}") args.append(os.fspath(project_dir / VENDOR_DIR)) - external_commands.run(args, network_isolation=False) - return sorted(project_dir.joinpath(VENDOR_DIR).iterdir()) + output = external_commands.run(args, network_isolation=False) + cargo_config = _parse_cargo_vendor_config(output) + return sorted(project_dir.joinpath(VENDOR_DIR).iterdir()), cargo_config + + +def _parse_cargo_vendor_config(output: str) -> tomlkit.TOMLDocument: + """Extract Cargo's source replacement configuration.""" + config_start = output.find("[source.") + if config_start < 0: + raise ValueError("cargo vendor did not emit source configuration") + + try: + cargo_config = tomlkit.parse(output[config_start:]) + cargo_config["source"]["vendored-sources"]["directory"] = VENDOR_DIR + except (KeyError, tomlkit.exceptions.ParseError) as err: + raise ValueError("cargo vendor emitted invalid source configuration") from err + return cargo_config def _cargo_shrink(crate_dir: pathlib.Path) -> None: @@ -70,7 +78,9 @@ def _cargo_shrink(crate_dir: pathlib.Path) -> None: json.dump(checksums, f) -def _cargo_config(project_dir: pathlib.Path) -> None: +def _cargo_config( + project_dir: pathlib.Path, cargo_config: tomlkit.TOMLDocument +) -> None: """create .cargo/config.toml""" dotcargo = project_dir / ".cargo" config_toml = dotcargo / "config.toml" @@ -83,7 +93,7 @@ def _cargo_config(project_dir: pathlib.Path) -> None: dotcargo.mkdir(exist_ok=True) cfg = tomlkit.parse("") - cfg.update(CARGO_CONFIG) + cfg.update(cargo_config) with open(config_toml, "w", encoding="utf-8") as f: tomlkit.dump(cfg, f) @@ -127,7 +137,7 @@ def vendor_generic_rust_package( # default to Cargo.toml in root dir manifests = [root_dir / "Cargo.toml"] # fetch and vendor Rust crates - vendored = _cargo_vendor(req, manifests, root_dir) + vendored, cargo_config = _cargo_vendor(req, manifests, root_dir) logger.debug(f"vendored crates: {sorted(d.name for d in vendored)}") # remove unnecessary pre-compiled files for Windows, macOS, and iOS. @@ -136,7 +146,7 @@ def vendor_generic_rust_package( _cargo_shrink(crate_dir) # update or create .cargo/config.toml - _cargo_config(root_dir) + _cargo_config(root_dir, cargo_config) return vendored diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index c57ab26f6..56e795cf3 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -542,6 +542,78 @@ def test_skip_constraints_cli_option() -> None: assert "Skip generating constraints.txt file" in result.output +@patch("fromager.commands.bootstrap.prefetch.export_prefetch_bundle") +@patch("fromager.commands.bootstrap.write_constraints_file", return_value=True) +@patch("fromager.commands.bootstrap.bootstrapper.Bootstrapper") +@patch("fromager.commands.bootstrap.server.start_wheel_server") +@patch("fromager.commands.bootstrap.progress.progress_context") +@patch("fromager.commands.bootstrap.metrics.summarize") +def test_prefetch_dir_enables_sdist_only_bootstrap( + mock_metrics: Mock, + mock_progress: Mock, + mock_server: Mock, + mock_bootstrapper: Mock, + mock_write_constraints: Mock, + mock_export: Mock, + tmp_context: context.WorkContext, +) -> None: + mock_progress.return_value.__enter__.return_value = Mock() + mock_progress.return_value.__exit__.return_value = None + mock_bt_instance = Mock() + mock_bt_instance.__enter__ = Mock(return_value=mock_bt_instance) + mock_bt_instance.__exit__ = Mock(return_value=None) + mock_bt_instance.finalize.return_value = 0 + mock_bootstrapper.return_value = mock_bt_instance + mock_export.return_value.root = pathlib.Path("prefetch") + + result = CliRunner().invoke( + bootstrap.bootstrap, + ["--prefetch-dir", "prefetch", "demo"], + obj=tmp_context, + ) + + assert result.exit_code == 0 + assert tmp_context.cleanup is False + assert mock_bootstrapper.call_args.kwargs["sdist_only"] is True + assert mock_bootstrapper.call_args.kwargs["capture_build_requirements"] is True + mock_export.assert_called_once_with(tmp_context, pathlib.Path("prefetch")) + + +def test_prefetch_dir_rejects_test_mode( + tmp_context: context.WorkContext, +) -> None: + with patch("fromager.commands.bootstrap.server.start_wheel_server") as mock_server: + result = CliRunner().invoke( + bootstrap.bootstrap, + ["--prefetch-dir", "prefetch", "--test-mode", "demo"], + obj=tmp_context, + ) + + assert result.exit_code == 2 + assert "--prefetch-dir cannot be combined with --test-mode" in result.output + mock_server.assert_not_called() + + +def test_prefetch_dir_rejects_existing_prepared_sources( + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, +) -> None: + stale_source = tmp_context.work_dir / "demo-1.0" / "demo-1.0" + stale_source.mkdir(parents=True) + + with patch("fromager.commands.bootstrap.bootstrapper.Bootstrapper") as mock_bt: + result = CliRunner().invoke( + bootstrap.bootstrap, + ["--prefetch-dir", str(tmp_path / "prefetch"), "demo"], + obj=tmp_context, + ) + + assert result.exit_code == 2 + assert "requires a clean work directory" in result.output + mock_bt.assert_not_called() + assert tmp_context.cleanup is True + + @patch("fromager.commands.bootstrap.bootstrapper.Bootstrapper") @patch("fromager.commands.bootstrap.server.start_wheel_server") @patch("fromager.commands.bootstrap.progress.progress_context") diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index e640cfb07..79daa17a2 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -193,6 +193,30 @@ def test_build_order_repeats(tmp_context: WorkContext) -> None: assert expected == contents +def test_build_order_records_captured_build_requirements( + tmp_context: WorkContext, +) -> None: + bt = bootstrapper.Bootstrapper(tmp_context) + bt.add_to_build_order( + req=Requirement("buildme>1.0"), + version=Version("6.0"), + source_url="url", + source_type=SourceType.SDIST, + build_requirements={ + "flit_core": Version("3.12.0"), + "packaging": Version("24.0"), + }, + ) + + bt.finalize() + + contents = json.loads(bt._build_order_filename.read_text()) + assert contents[0]["build_requirements"] == [ + "flit-core==3.12.0", + "packaging==24.0", + ] + + def test_build_order_name_canonicalization(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) bt.add_to_build_order( @@ -902,6 +926,11 @@ def capture_run(self: Resolve, bt_arg: bootstrapper.Bootstrapper) -> list[Phase] return [] with ( + patch.object( + bt, + "resolve_versions", + return_value=[("https://example.test/testpkg-1.0.tar.gz", Version("1.0"))], + ), patch.object( bt, "_resolve_and_add_top_level", @@ -924,6 +953,7 @@ def test_bootstrap_skips_failed_resolution(tmp_context: WorkContext) -> None: req = Requirement("badpkg") with ( + patch.object(bt, "resolve_versions", return_value=[]), patch.object(bt, "_resolve_and_add_top_level", return_value=None), patch.object(Resolve, "run") as mock_run, patch.object(bt, "_record_stack_state"), @@ -946,6 +976,7 @@ def capture_run(self: Resolve, bt_arg: bootstrapper.Bootstrapper) -> list[Phase] return [] with ( + patch.object(bt, "resolve_versions", return_value=[]), patch.object( bt, "_resolve_and_add_top_level", @@ -1089,6 +1120,35 @@ def test_bg_prepare_source_log_prefix_includes_version( ) +def test_bg_prepare_source_skips_wheel_cache_for_prefetch( + tmp_context: WorkContext, +) -> None: + req = Requirement("mypkg==1.2.3") + version = Version("1.2.3") + source_filename = pathlib.Path("mypkg-1.2.3.tar.gz") + source_root = tmp_context.work_dir / "mypkg-1.2.3" / "mypkg-1.2.3" + + with ( + patch("fromager.bootstrapper._cache.find_cached_wheel") as find_cached, + patch( + "fromager.sources.download_source", return_value=source_filename + ) as download_source, + patch("fromager.sources.prepare_source", return_value=source_root), + ): + result = _bg_prepare_source( + ctx=tmp_context, + cache_wheel_server_url="https://cache.test/simple/", + req=req, + resolved_version=version, + source_url="https://pkg.test/mypkg-1.2.3.tar.gz", + use_wheel_cache=False, + ) + + find_cached.assert_not_called() + download_source.assert_called_once() + assert result.sdist_root_dir == source_root + + def test_bg_prepare_prebuilt_log_prefix_includes_version( tmp_context: WorkContext, caplog: pytest.LogCaptureFixture, diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index d68d45d4e..ec97cacb7 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -1875,6 +1875,30 @@ def test_overlapping_deps_skips_install(self, tmp_context: WorkContext) -> None: mock_env.install.assert_not_called() + def test_captures_installed_build_requirements( + self, tmp_context: WorkContext + ) -> None: + bt = bootstrapper.Bootstrapper(tmp_context, capture_build_requirements=True) + item = self._make_build_phase_item(tmp_context) + build_env = item.work_item.build_env + assert build_env is not None + distributions = { + "flit_core": Version("3.12.0"), + "packaging": Version("24.0"), + } + + with ( + patch.object(item, "do_build", return_value=(None, None)), + patch.object(build_env, "get_distributions", return_value=distributions), + patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), + ): + item.run(bt) + + assert item.work_item.resolved_build_requirements == { + "flit_core": Version("3.12.0"), + "packaging": Version("24.0"), + } + def test_partial_overlap_deps_skips_install(self, tmp_context: WorkContext) -> None: """isdisjoint is False on partial overlap, so install is skipped entirely.""" bt = bootstrapper.Bootstrapper(tmp_context) @@ -2212,6 +2236,7 @@ def test_build_order_called_with_correct_args( source_type=SourceType.SDIST, prebuilt=True, constraint=constraint, + build_requirements=None, ) diff --git a/tests/test_build_environment.py b/tests/test_build_environment.py index 28c504af1..5d083790f 100644 --- a/tests/test_build_environment.py +++ b/tests/test_build_environment.py @@ -42,6 +42,24 @@ def test_missing_dependency_format( assert "flit_core -> 3.9.0" in s +@patch("fromager.resolver.resolve") +def test_missing_dependency_offline_does_not_resolve( + resolve_dist: Mock, + tmp_context: WorkContext, +) -> None: + tmp_context.enable_offline_build([]) + + ex = build_environment.MissingDependency( + tmp_context, + RequirementType.BUILD_BACKEND, + "setuptools", + [Requirement("setuptools>=40.8.0")], + ) + + assert "setuptools>=40.8.0 -> missing from prefetch bundle" in str(ex) + resolve_dist.assert_not_called() + + def test_missing_dependency_pattern() -> None: msg = textwrap.dedent(""" DEBUG uv 0.8.4 @@ -104,3 +122,35 @@ def test_get_distributions_invalid_json_raises() -> None: with pytest.raises(json.JSONDecodeError): build_environment.BuildEnvironment.get_distributions(env) + + +@patch("fromager.build_environment._safe_install") +@patch("fromager.build_environment.BuildEnvironment") +def test_prepare_build_environment_from_prefetch_uses_exact_requirements( + build_environment_class: Mock, + safe_install: Mock, + tmp_context: WorkContext, +) -> None: + req = Requirement("demo==1.0") + build_requirements = { + Requirement("flit-core==3.12.0"), + Requirement("packaging==24.0"), + } + sdist_root = tmp_context.work_dir / "demo-1.0" + expected_build_environment = build_environment_class.return_value + + result = build_environment.prepare_build_environment_from_prefetch( + ctx=tmp_context, + req=req, + sdist_root_dir=sdist_root, + build_requirements=build_requirements, + ) + + assert result is expected_build_environment + safe_install.assert_called_once_with( + ctx=tmp_context, + req=req, + build_env=expected_build_environment, + deps=build_requirements, + dep_req_type=RequirementType.BUILD_SYSTEM, + ) diff --git a/tests/test_commands.py b/tests/test_commands.py index 5678d3d96..ce73999c5 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -18,5 +18,6 @@ def test_bootstrap_parallel_options() -> None: expected.discard("sdist_only") expected.discard("graph_file") expected.discard("test_mode") + expected.discard("prefetch_dir") assert set(get_option_names(bootstrap.bootstrap_parallel)) == expected diff --git a/tests/test_finders.py b/tests/test_finders.py index 110ccfa1f..7c607753c 100644 --- a/tests/test_finders.py +++ b/tests/test_finders.py @@ -84,6 +84,21 @@ def test_find_wheel_returns_none_when_only_non_wheel_files( assert finders.find_wheel(downloads, req, "1.2", ()) is None +def test_find_exact_wheel_matches_build_tag(tmp_path: pathlib.Path) -> None: + downloads = tmp_path / "downloads" + downloads.mkdir() + expected = downloads / "mypkg-1.2-2-py3-none-any.whl" + expected.touch() + (downloads / "mypkg-1.2-1-py3-none-any.whl").touch() + (downloads / "mypkg-2.0-2-py3-none-any.whl").touch() + + result = finders.find_exact_wheel( + (downloads,), Requirement("mypkg"), "1.2", (2, "") + ) + + assert result == expected + + @pytest.mark.parametrize( "dist_name,version_string,unpack_base,source_base", [ diff --git a/tests/test_prefetch.py b/tests/test_prefetch.py new file mode 100644 index 000000000..502efc902 --- /dev/null +++ b/tests/test_prefetch.py @@ -0,0 +1,597 @@ +"""Tests for prefetch bundle export and validation.""" + +from __future__ import annotations + +import json +import pathlib +import tarfile +from unittest.mock import Mock, patch + +import pytest +from click.testing import CliRunner +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version + +from fromager import build_environment, context, prefetch +from fromager.commands import build +from fromager.requirements_file import RequirementType + + +def _write_bootstrap_outputs(ctx: context.WorkContext) -> None: + ctx.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("demo==1.0"), + req_version=Version("1.0"), + ) + ctx.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("binary-demo==2.0"), + req_version=Version("2.0"), + pre_built=True, + ) + ctx.dependency_graph.add_dependency( + parent_name=canonicalize_name("demo"), + parent_version=Version("1.0"), + req_type=RequirementType.BUILD_SYSTEM, + req=Requirement("setuptools>=70"), + req_version=Version("80.0"), + ) + ctx.dependency_graph.add_dependency( + parent_name=canonicalize_name("setuptools"), + parent_version=Version("80.0"), + req_type=RequirementType.INSTALL, + req=Requirement("packaging>=23"), + req_version=Version("24.0"), + ) + with ctx.graph_file.open("w", encoding="utf-8") as f: + ctx.dependency_graph.serialize(f) + build_order = [ + { + "req": "packaging>=23", + "constraint": "", + "dist": "packaging", + "version": "24.0", + "prebuilt": False, + "source_url": "https://pkg.test/packaging-24.0.tar.gz", + "source_url_type": "sdist", + "build_requirements": [], + "prepared_source_root": "packaging-24.0/packaging-24.0", + "prepared_source_workspace": "packaging-24.0", + }, + { + "req": "setuptools>=70", + "constraint": "", + "dist": "setuptools", + "version": "80.0", + "prebuilt": False, + "source_url": "https://pkg.test/setuptools-80.0.tar.gz", + "source_url_type": "sdist", + "build_requirements": [], + "prepared_source_root": "setuptools-80.0/setuptools-80.0", + "prepared_source_workspace": "setuptools-80.0", + }, + { + "req": "demo==1.0", + "constraint": "", + "dist": "demo", + "version": "1.0", + "prebuilt": False, + "source_url": "https://pkg.test/demo-1.0.tar.gz", + "source_url_type": "sdist", + "build_requirements": ["packaging==24.0", "setuptools==80.0"], + "prepared_source_root": "demo-1.0/demo-1.0", + "prepared_source_workspace": "demo-1.0", + }, + { + "req": "binary-demo==2.0", + "constraint": "", + "dist": "binary-demo", + "version": "2.0", + "prebuilt": True, + "source_url": "https://pkg.test/binary_demo-2.0-py3-none-any.whl", + "source_url_type": "prebuilt", + "build_requirements": [], + }, + ] + ctx.work_dir.joinpath("build-order.json").write_text( + json.dumps(build_order), encoding="utf-8" + ) + ctx.work_dir.joinpath("constraints.txt").write_text("demo==1.0\n", encoding="utf-8") + ctx.sdists_builds.joinpath("demo-1.0.tar.gz").write_bytes(b"patched source") + ctx.sdists_builds.joinpath("setuptools-80.0.tar.gz").write_bytes( + b"patched setuptools" + ) + ctx.sdists_builds.joinpath("packaging-24.0.tar.gz").write_bytes( + b"patched packaging" + ) + for dist, version in ( + ("demo", "1.0"), + ("packaging", "24.0"), + ("setuptools", "80.0"), + ): + source_root = ctx.work_dir / f"{dist}-{version}" / f"{dist}-{version}" + source_root.mkdir(parents=True) + source_root.joinpath("prepared.txt").write_text(dist) + ctx.wheels_prebuilt.joinpath("binary_demo-2.0-py3-none-any.whl").write_bytes( + b"prebuilt wheel" + ) + ctx.wheels_downloads.joinpath("setuptools-80.0-py3-none-any.whl").write_bytes( + b"bootstrap wheel" + ) + ctx.wheels_downloads.joinpath("packaging-24.0-py3-none-any.whl").write_bytes( + b"transitive bootstrap wheel" + ) + ctx.wheels_downloads.joinpath("unrelated-9.0-py3-none-any.whl").write_bytes( + b"stale wheel" + ) + + +def test_export_prefetch_bundle( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + + assert bundle.manifest.format_version == 1 + assert bundle.manifest.variant == "cpu" + assert bundle.build_order_file.name == "build-order.json" + with tarfile.open( + bundle.package_artifact("demo", "1.0", prebuilt=False) + ) as archive: + prepared_file = archive.extractfile("demo-1.0/prepared.txt") + assert prepared_file is not None + assert prepared_file.read() == b"demo" + assert ( + bundle.package_artifact("binary-demo", "2.0", prebuilt=True).read_bytes() + == b"prebuilt wheel" + ) + assert {path.name for path in bundle.wheelhouse_dirs} == {"wheels"} + demo = next( + package for package in bundle.manifest.packages if package.dist == "demo" + ) + assert set(demo.build_requirements) == { + "packaging==24.0", + "setuptools==80.0", + } + assert { + pathlib.Path(wheel.artifact.path).name for wheel in bundle.manifest.wheels + } == { + "packaging-24.0-py3-none-any.whl", + "setuptools-80.0-py3-none-any.whl", + } + + +def test_export_prefetch_bundle_includes_prepared_siblings( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + package_dir = tmp_context.work_dir / "demo-1.0" + source_dir = package_dir / "demo-1.0" + source_dir.joinpath("prepared.txt").unlink() + (source_dir / "setup.py").write_text("# setup") + (package_dir / "prepared-dependency").mkdir() + (package_dir / "prepared-dependency" / "README").write_text("prepared") + (package_dir / "build-3.12.14").mkdir() + + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + + with tarfile.open( + bundle.package_artifact("demo", "1.0", prebuilt=False) + ) as archive: + names = set(archive.getnames()) + assert "demo-1.0/setup.py" in names + assert "prepared-dependency/README" in names + assert not any("build-3.12.14" in name for name in names) + + +def test_export_prefetch_bundle_preserves_build_named_source_root( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + workspace = tmp_context.work_dir / "demo-1.0" + source_root = workspace / "demo-1.0" + source_root.rename(workspace / "build-1.0") + workspace.rename(tmp_context.work_dir / "build-1.0") + build_order_path = tmp_context.work_dir / "build-order.json" + build_order = json.loads(build_order_path.read_text(encoding="utf-8")) + demo = next(entry for entry in build_order if entry["dist"] == "demo") + demo["prepared_source_root"] = "build-1.0/build-1.0" + demo["prepared_source_workspace"] = "build-1.0" + build_order_path.write_text(json.dumps(build_order), encoding="utf-8") + + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + + with tarfile.open( + bundle.package_artifact("demo", "1.0", prebuilt=False) + ) as archive: + assert "build-1.0/prepared.txt" in archive.getnames() + + +def test_export_prefetch_bundle_excludes_nested_build_outputs( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + source_root = tmp_context.work_dir / "demo-1.0" / "demo-1.0" + (source_root / "subproject" / "build" / "lib").mkdir(parents=True) + (source_root / "subproject" / "build" / "lib" / "generated.c").write_text( + "generated" + ) + (source_root / "vendor" / "rust" / "target" / "debug").mkdir(parents=True) + (source_root / "vendor" / "rust" / "target" / "debug" / "artifact.o").write_text( + "generated" + ) + (source_root / "demo.egg-info").mkdir() + (source_root / "demo.egg-info" / "PKG-INFO").write_text("generated") + (source_root / "__pycache__").mkdir() + (source_root / "__pycache__" / "module.pyc").write_bytes(b"generated") + (source_root / "src").mkdir() + (source_root / "src" / "module.py").write_text("source") + + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + + with tarfile.open( + bundle.package_artifact("demo", "1.0", prebuilt=False) + ) as archive: + names = set(archive.getnames()) + assert "demo-1.0/src/module.py" in names + assert not any( + "/build/" in name + or "/target/" in name + or ".egg-info" in name + or "__pycache__" in name + or name.endswith(".pyc") + for name in names + ) + + +def test_export_prefetch_bundle_requires_prepared_workspace( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + source_root = tmp_context.work_dir / "demo-1.0" / "demo-1.0" + source_root.rename(tmp_context.work_dir / "missing-demo-source") + + with pytest.raises(FileNotFoundError, match="prepared source workspace is missing"): + prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + + +def test_export_prefetch_bundle_normalizes_zip_source_name( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + (tmp_context.sdists_builds / "demo-1.0.tar.gz").rename( + tmp_context.sdists_builds / "demo-1.0.zip" + ) + + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + artifact = bundle.package_artifact("demo", "1.0", prebuilt=False) + + assert artifact.name == "demo-1.0.tar.gz" + with tarfile.open(artifact): + pass + + +def test_load_prefetch_bundle_rejects_tampered_artifact( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + sdist = bundle.package_artifact("demo", "1.0", prebuilt=False) + sdist.write_bytes(b"corrupt source") + + with pytest.raises(ValueError, match="wrong size"): + prefetch.load_prefetch_bundle(bundle.root) + + +def test_load_prefetch_bundle_rejects_tampered_wheel( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + wheel = bundle.root / bundle.manifest.wheels[0].artifact.path + wheel.write_bytes(b"corrupt wheel") + + with pytest.raises(ValueError, match="wrong size"): + prefetch.load_prefetch_bundle(bundle.root) + + +def test_copy_artifact_rejects_symlink_destination(tmp_path: pathlib.Path) -> None: + root = tmp_path / "bundle" + root.mkdir() + source = tmp_path / "source.whl" + source.write_bytes(b"source") + outside = tmp_path / "outside.whl" + outside.write_bytes(b"outside") + destination = root / "wheels" / "source.whl" + destination.parent.mkdir() + destination.symlink_to(outside) + + with pytest.raises(ValueError, match="symlink"): + prefetch._copy_artifact(root, source, pathlib.Path("wheels/source.whl")) + + assert outside.read_bytes() == b"outside" + + +def test_load_prefetch_bundle_rejects_variant_mismatch( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + + with pytest.raises(ValueError, match="does not match"): + prefetch.load_prefetch_bundle(bundle.root, expected_variant="cuda") + + +def test_load_prefetch_bundle_rejects_settings_mismatch( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + tmp_context.settings.max_jobs = 2 + + with pytest.raises(ValueError, match="settings and patches"): + bundle.validate_configuration(tmp_context) + + +def test_configuration_fingerprint_changes_with_settings( + tmp_context: context.WorkContext, +) -> None: + packages = [("demo", "1.0")] + original = prefetch.configuration_fingerprint(tmp_context, packages) + + tmp_context.settings.max_jobs = 1 + + assert prefetch.configuration_fingerprint(tmp_context, packages) != original + + +def test_load_prefetch_bundle_rejects_fromager_version_mismatch( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + manifest_path = bundle.root / prefetch.MANIFEST_FILENAME + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["fromager_version"] = "999.0" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="fromager_version"): + prefetch.load_prefetch_bundle(bundle.root) + + +def test_offline_context_uses_local_wheelhouses( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + wheelhouse = tmp_path / "wheelhouse" + + tmp_context.enable_offline_build([wheelhouse]) + + assert tmp_context.offline + assert tmp_context.pip_wheel_server_args == [ + "--no-index", + "--offline", + "--find-links", + str(tmp_context.wheels_downloads), + "--find-links", + str(wheelhouse), + ] + + +def test_offline_context_uses_prefetched_constraints( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + constraints = tmp_path / "merged-constraints.txt" + constraints.write_text("setuptools==80.0\n", encoding="utf-8") + + tmp_context.enable_offline_build([], constraints_file=constraints) + + assert tmp_context.pip_constraint_args == ["--constraint", str(constraints)] + + +def test_offline_build_environment_install_uses_network_isolation( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + tmp_context.network_isolation = True + tmp_context.enable_offline_build([tmp_path / "wheelhouse"]) + build_env = Mock(spec=build_environment.BuildEnvironment) + build_env._ctx = tmp_context + build_env.path = tmp_path / "build-env" + + build_environment.BuildEnvironment.install( + build_env, [Requirement("setuptools==80.0")] + ) + + build_env.run.assert_called_once() + assert build_env.run.call_args.kwargs["network_isolation"] is True + + +def test_offline_build_environment_blocks_nested_package_indexes( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + wheelhouse = tmp_path / "wheelhouse" + constraints = tmp_path / "constraints.txt" + tmp_context.enable_offline_build([wheelhouse], constraints_file=constraints) + build_env = object.__new__(build_environment.BuildEnvironment) + build_env._ctx = tmp_context + build_env.path = tmp_path / "build-env" + + environ = build_env.get_venv_environ({"PATH": "/usr/bin"}) + + assert environ["UV_OFFLINE"] == "true" + assert environ["UV_NO_CACHE"] == "true" + assert environ["PIP_NO_INDEX"] == "1" + assert environ["PIP_FIND_LINKS"] == (f"{tmp_context.wheels_downloads} {wheelhouse}") + assert environ["PIP_CONSTRAINT"] == str(constraints) + + +def test_build_uses_prefetched_sdist_without_preparing_source( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + req = Requirement("demo==1.0") + tmp_context.wheel_server_url = "http://127.0.0.1:1234/simple/" + source_root = tmp_context.work_dir / "demo-1.0" / "demo-1.0" + source_root.mkdir(parents=True, exist_ok=True) + built_wheel = tmp_context.wheels_build / "demo-1.0-py3-none-any.whl" + built_wheel.write_bytes(b"wheel") + built_sdist = tmp_context.sdists_builds / "demo-1.0.tar.gz" + stale_wheel = tmp_context.wheels_downloads / built_wheel.name + stale_wheel.write_bytes(b"stale wheel") + build_env = Mock(spec=build_environment.BuildEnvironment) + build_env.path = tmp_path / "build-env" + + with ( + patch.object( + build, "_is_wheel_built", return_value=None + ) as remote_wheel_lookup, + patch.object(build.sources, "download_source") as download_source, + patch.object(build.sources, "prepare_source") as prepare_source, + patch.object( + build.sources, "unpack_source", return_value=(source_root, True) + ) as unpack_source, + patch.object( + build.sources, "build_sdist", return_value=built_sdist + ) as build_sdist, + patch.object( + build.build_environment, + "prepare_build_environment", + ) as discover_build_environment, + patch.object( + build.build_environment, + "prepare_build_environment_from_prefetch", + return_value=build_env, + ) as prepare_build_environment, + patch.object( + build.wheels, "build_wheel", return_value=built_wheel + ) as build_wheel, + patch.object(build.hooks, "run_post_build_hooks") as post_build_hooks, + patch.object(build.server, "update_wheel_mirror"), + ): + build._build( + wkctx=tmp_context, + resolved_version=Version("1.0"), + req=req, + source_download_url="https://pkg.test/demo-1.0.tar.gz", + force=False, + cache_wheel_server_url=None, + prefetch_bundle=bundle, + ) + + unpack_source.assert_called_once_with( + ctx=tmp_context, + req=req, + version=Version("1.0"), + source_filename=bundle.package_artifact("demo", "1.0", prebuilt=False), + reuse_existing=False, + ) + download_source.assert_not_called() + prepare_source.assert_not_called() + build_sdist.assert_called_once_with( + ctx=tmp_context, + req=req, + version=Version("1.0"), + sdist_root_dir=source_root, + build_env=build_env, + ) + assert post_build_hooks.call_args.kwargs["sdist_filename"] == built_sdist + discover_build_environment.assert_not_called() + assert prepare_build_environment.call_args.kwargs["build_requirements"] == { + Requirement("packaging==24.0"), + Requirement("setuptools==80.0"), + } + remote_wheel_lookup.assert_not_called() + build_wheel.assert_called_once() + + +def test_clean_prefetched_source_removes_workspace( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + workspace = tmp_path / "workspace" + source_root = workspace / "demo-1.0" + sibling = workspace / "prepared-dependency" + source_root.mkdir(parents=True) + sibling.mkdir() + build_env = Mock(spec=build_environment.BuildEnvironment) + build_env.path = workspace / "build-3.12.0" + build_env.path.mkdir() + prepared = build._PreparedBuildSource( + sdist_filename=tmp_path / "demo-1.0.tar.gz", + source_root_dir=source_root, + build_env=build_env, + workspace_dir=workspace, + ) + + build._clean_prepared_source(tmp_context, prepared) + + assert not workspace.exists() + + +def test_prefetched_prebuilt_ignores_stale_output_wheel( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + wheel_name = "binary_demo-2.0-py3-none-any.whl" + stale_wheel = tmp_context.wheels_downloads / wheel_name + stale_wheel.write_bytes(b"stale wheel") + + with ( + patch.object( + tmp_context, + "package_build_info", + return_value=Mock(pre_built=True), + ), + patch.object(build.hooks, "run_prebuilt_wheel_hooks") as prebuilt_hooks, + patch.object(build.server, "update_wheel_mirror"), + ): + result = build._build( + wkctx=tmp_context, + resolved_version=Version("2.0"), + req=Requirement("binary-demo==2.0"), + source_download_url=f"https://pkg.test/{wheel_name}", + force=False, + cache_wheel_server_url=None, + prefetch_bundle=bundle, + ) + + staged_wheel = tmp_context.wheels_build / wheel_name + assert staged_wheel.read_bytes() == b"prebuilt wheel" + assert result.skipped is False + assert prebuilt_hooks.call_args.kwargs["wheel_filename"] == staged_wheel + + +def test_build_sequence_accepts_prefetch_directory( + tmp_context: context.WorkContext, tmp_path: pathlib.Path +) -> None: + _write_bootstrap_outputs(tmp_context) + bundle = prefetch.export_prefetch_bundle(tmp_context, tmp_path / "prefetch") + built = Mock( + prebuilt=False, + skipped=False, + wheel_filename=pathlib.Path("demo-1.0-py3-none-any.whl"), + ) + + with ( + patch.object(build, "_build", return_value=built) as build_one, + patch.object(build.server, "start_wheel_server") as start_wheel_server, + patch.object(build.metrics, "summarize"), + patch.object(build, "_summary"), + ): + result = CliRunner().invoke( + build.build_sequence, + ["--prefetch-dir", str(bundle.root)], + obj=tmp_context, + ) + + assert result.exit_code == 0 + assert tmp_context.offline + assert tmp_context.network_isolation is True + assert build_one.call_count == 4 + assert build_one.call_args.kwargs["prefetch_bundle"].root == bundle.root + start_wheel_server.assert_not_called() diff --git a/tests/test_sources.py b/tests/test_sources.py index c5f9e3d47..b0f170ca5 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -519,6 +519,20 @@ def test_unpack_source_zip( assert (result / "setup.py").is_file() +def test_normalize_source_timestamps_uses_zip_safe_epoch( + tmp_path: pathlib.Path, +) -> None: + source_root = tmp_path / "source" + source_root.mkdir() + source_file = source_root / "module.py" + source_file.write_text("# module\n", encoding="utf-8") + + sources.normalize_source_timestamps(source_root) + + assert source_root.stat().st_mtime == sources.ZIP_SAFE_EPOCH + assert source_file.stat().st_mtime == sources.ZIP_SAFE_EPOCH + + def test_unpack_source_unknown_extension( tmp_context: context.WorkContext, tmp_path: pathlib.Path, @@ -534,6 +548,23 @@ def test_unpack_source_unknown_extension( ) +def test_unpack_source_rejects_zip_path_traversal( + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, +) -> None: + archive = tmp_path / "mypkg-1.0.zip" + with zipfile.ZipFile(archive, "w") as zip_file: + zip_file.writestr("../outside.txt", "unsafe") + + with pytest.raises(ValueError, match="unsafe path"): + sources.unpack_source( + ctx=tmp_context, + req=Requirement("mypkg==1.0"), + version=Version("1.0"), + source_filename=archive, + ) + + def test_unpack_source_renames_mismatched_dir( tmp_context: context.WorkContext, tmp_path: pathlib.Path, @@ -558,6 +589,39 @@ def test_unpack_source_renames_mismatched_dir( assert result.name == expected_name +def test_unpack_source_preserves_sibling_directories( + tmp_context: context.WorkContext, + tmp_path: pathlib.Path, +) -> None: + req = Requirement("mypkg==1.0") + version = Version("1.0") + archive_root = tmp_path / "archive" + source_dir = archive_root / "mypkg-1.0" + sibling_dir = archive_root / "prepared-dependency" + source_dir.mkdir(parents=True) + sibling_dir.mkdir() + (source_dir / "setup.py").write_text("# setup") + (sibling_dir / "README").write_text("prepared") + + tar_path = tmp_path / "mypkg-1.0.tar.gz" + with tarfile.open(tar_path, "w:gz") as archive: + archive.add(source_dir, arcname=source_dir.name) + archive.add(sibling_dir, arcname=sibling_dir.name) + + result, is_new = sources.unpack_source( + ctx=tmp_context, + req=req, + version=version, + source_filename=tar_path, + ) + + assert is_new is True + assert result == tmp_context.work_dir / "mypkg-1.0" / "mypkg-1.0" + assert (result.parent / "prepared-dependency" / "README").read_text() == ( + "prepared" + ) + + def test_unpack_source_reuse_when_no_cleanup( tmp_path: pathlib.Path, ) -> None: @@ -590,6 +654,40 @@ def test_unpack_source_reuse_when_no_cleanup( assert (result / "setup.py").read_text() == "# old" +def test_unpack_source_replaces_existing_when_reuse_disabled( + tmp_path: pathlib.Path, +) -> None: + ctx = context.WorkContext( + active_settings=None, + patches_dir=tmp_path / "overrides/patches", + sdists_repo=tmp_path / "sdists-repo", + wheels_repo=tmp_path / "wheels-repo", + work_dir=tmp_path / "work-dir", + cleanup=False, + ) + ctx.setup() + existing = ctx.work_dir / "mypkg-1.0" / "mypkg-1.0" + existing.mkdir(parents=True) + (existing / "setup.py").write_text("# old") + archive_root = tmp_path / "archive" / "mypkg-1.0" + archive_root.mkdir(parents=True) + (archive_root / "setup.py").write_text("# new") + source = tmp_path / "mypkg-1.0.tar.gz" + with tarfile.open(source, "w:gz") as archive: + archive.add(archive_root, arcname=archive_root.name) + + result, is_new = sources.unpack_source( + ctx=ctx, + req=Requirement("mypkg==1.0"), + version=Version("1.0"), + source_filename=source, + reuse_existing=False, + ) + + assert is_new is True + assert (result / "setup.py").read_text() == "# new" + + @patch("fromager.overrides.find_and_invoke") def test_download_source_regular_package( mock_invoke: Mock, @@ -786,3 +884,30 @@ def test_default_build_sdist_normalizes_filename( expected_filename = f"{expected_filename_part}-1.0.0.tar.gz" assert sdist_file.name == expected_filename assert sdist_file.parent == tmp_context.sdists_builds + + +def test_default_build_sdist_uses_configured_build_dir_as_root( + testdata_context: context.WorkContext, + tmp_path: pathlib.Path, +) -> None: + source_root = tmp_path / "test_pkg-1.0" + build_dir = source_root / "python" + build_dir.mkdir(parents=True) + (source_root / "PKG-INFO").write_text("metadata") + (build_dir / "pyproject.toml").write_text("[build-system]\n") + + sdist = sources.default_build_sdist( + ctx=testdata_context, + extra_environ={}, + req=Requirement("test-pkg==1.0"), + version=Version("1.0"), + sdist_root_dir=source_root, + build_env=Mock(), + build_dir=build_dir, + ) + + with tarfile.open(sdist) as archive: + names = set(archive.getnames()) + assert "python/PKG-INFO" in names + assert "python/pyproject.toml" in names + assert not any(name.startswith("test_pkg-1.0/") for name in names) diff --git a/tests/test_vendor_rust.py b/tests/test_vendor_rust.py new file mode 100644 index 000000000..2b06700c0 --- /dev/null +++ b/tests/test_vendor_rust.py @@ -0,0 +1,48 @@ +import pathlib +from unittest import mock + +import tomlkit +from packaging.requirements import Requirement + +from fromager import vendor_rust + + +def test_vendor_generic_rust_package_preserves_git_source_replacement( + tmp_path: pathlib.Path, +) -> None: + """The Cargo config redirects registry and Git dependencies to vendor.""" + manifest = tmp_path / "Cargo.toml" + manifest.touch() + cargo_output = f"""[source.crates-io] +replace-with = "vendored-sources" + +[source."git+https://pkg.test/repository?rev=1234"] +git = "https://pkg.test/repository" +rev = "1234" +replace-with = "vendored-sources" + +[source.vendored-sources] +directory = "{tmp_path / "vendor"}" +""" + + def run_cargo(*args: object, **kwargs: object) -> str: + vendor_dir = tmp_path / "vendor" / "dependency" + vendor_dir.mkdir(parents=True) + return cargo_output + + with mock.patch( + "fromager.vendor_rust.external_commands.run", side_effect=run_cargo + ): + vendor_rust.vendor_generic_rust_package( + Requirement("demo"), [manifest], tmp_path, shrink_vendored=False + ) + + config = (tmp_path / ".cargo" / "config.toml").read_text() + assert '[source."git+https://pkg.test/repository?rev=1234"]' in config + parsed_config = tomlkit.parse(config) + sources = parsed_config["source"] + git_source = sources["git+https://pkg.test/repository?rev=1234"] + assert git_source["replace-with"] == "vendored-sources" + assert sources["crates-io"]["replace-with"] == "vendored-sources" + assert sources["vendored-sources"]["directory"] == "vendor" + assert str(tmp_path) not in config