From 7fac02f245a10a16c71e04b395d00a89c67688a2 Mon Sep 17 00:00:00 2001 From: Randolf Jung Date: Thu, 24 Sep 2026 07:53:54 -0700 Subject: [PATCH] fix(bootstrap): unify executable and zipapp startup Describe executables and ZIPs with one private application specification, and share environment preparation, storage ownership and packaging across ordinary binaries, legacy executable ZIPs and py_zipapp entry points. Use native POSIX exec to preserve the application PID, signals and terminal behavior, with a detached watcher for temporary-file cleanup. On Windows, wait for the application child and its console cleanup before removal. Publish complete persistent images without replacing concurrent entries. Preserve public providers, custom templates and stage-two behavior. Keep historical raw-template adapters available and support Python 3.9 ZIP loaders. Document cleanup lifetime and legacy compatibility limitations. Validation: 727 root targets pass; the launcher matrix passes 313 cases with two skips. Native checks pass on macOS, Windows and Linux arm64/x64, including final loader/ZIP checks on all four pools. Changed-file hooks, documentation, distribution and independent implementation audit pass. --- .bazelrc.deleted_packages | 2 +- docs/environment-variables.md | 79 +- news/bootstrap-runtime.fixed.md | 8 + python/private/BUILD.bazel | 50 + .../_rules_python_bootstrap/__init__.py | 1 + .../_rules_python_bootstrap/diagnostics.py | 9 + .../private/_rules_python_bootstrap/driver.py | 27 + .../private/_rules_python_bootstrap/entry.py | 302 +++++ .../_rules_python_bootstrap/environment.py | 281 +++++ .../private/_rules_python_bootstrap/model.py | 101 ++ .../_rules_python_bootstrap/process.py | 137 +++ .../_rules_python_bootstrap/storage.py | 171 +++ python/private/application.bzl | 245 ++++ .../private/application_python_template.txt | 52 + python/private/application_shell_template.sh | 221 ++++ python/private/application_zip_template.txt | 35 + python/private/bootstrap_cleanup.py | 192 +++ python/private/py_application_info.bzl | 11 + python/private/py_executable.bzl | 102 +- python/private/zipapp/BUILD.bazel | 3 + python/private/zipapp/py_zipapp_rule.bzl | 58 +- tests/bootstrap_impls/BUILD.bazel | 282 +++++ .../application_templates_test.py | 178 +++ tests/bootstrap_impls/application_test.py | 531 +++++++++ tests/bootstrap_impls/application_tests.bzl | 103 ++ .../bootstrap_impls/bootstrap_cleanup_test.py | 1054 +++++++++++++++++ .../bootstrap_cleanup_watch_test.py | 433 +++++++ tests/bootstrap_impls/cleanup_fail.py | 15 + tests/bootstrap_impls/cleanup_probe.py | 100 ++ .../bootstrap_impls/cleanup_python_wrapper.sh | 5 + .../bootstrap_impls/cleanup_sitecustomize.py | 18 + .../bootstrap_impls/cleanup_terminal_peer.py | 11 + .../bootstrap_impls/cleanup_wrapper_data.txt | 1 + .../windows_console_fixture.py | 135 +++ tests/py_zipapp/BUILD.bazel | 17 + tests/py_zipapp/system_python_zipapp_test.py | 13 +- tests/py_zipapp/venv_zipapp_test.py | 25 +- tests/tools/zipapp/zip_main_maker_test.py | 155 ++- tools/zipapp/exe_zip_maker.py | 8 + tools/zipapp/zip_main_maker.py | 34 +- 40 files changed, 5045 insertions(+), 160 deletions(-) create mode 100644 news/bootstrap-runtime.fixed.md create mode 100644 python/private/_rules_python_bootstrap/__init__.py create mode 100644 python/private/_rules_python_bootstrap/diagnostics.py create mode 100644 python/private/_rules_python_bootstrap/driver.py create mode 100644 python/private/_rules_python_bootstrap/entry.py create mode 100644 python/private/_rules_python_bootstrap/environment.py create mode 100644 python/private/_rules_python_bootstrap/model.py create mode 100644 python/private/_rules_python_bootstrap/process.py create mode 100644 python/private/_rules_python_bootstrap/storage.py create mode 100644 python/private/application.bzl create mode 100644 python/private/application_python_template.txt create mode 100644 python/private/application_shell_template.sh create mode 100644 python/private/application_zip_template.txt create mode 100644 python/private/bootstrap_cleanup.py create mode 100644 python/private/py_application_info.bzl create mode 100644 tests/bootstrap_impls/application_templates_test.py create mode 100644 tests/bootstrap_impls/application_test.py create mode 100644 tests/bootstrap_impls/application_tests.bzl create mode 100644 tests/bootstrap_impls/bootstrap_cleanup_test.py create mode 100644 tests/bootstrap_impls/bootstrap_cleanup_watch_test.py create mode 100644 tests/bootstrap_impls/cleanup_fail.py create mode 100644 tests/bootstrap_impls/cleanup_probe.py create mode 100755 tests/bootstrap_impls/cleanup_python_wrapper.sh create mode 100644 tests/bootstrap_impls/cleanup_sitecustomize.py create mode 100644 tests/bootstrap_impls/cleanup_terminal_peer.py create mode 100644 tests/bootstrap_impls/cleanup_wrapper_data.txt create mode 100644 tests/bootstrap_impls/windows_console_fixture.py diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index 5654df1266..3eec28298e 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -28,8 +28,8 @@ common --deleted_packages=gazelle/manifest/hasher common --deleted_packages=gazelle/manifest/test common --deleted_packages=gazelle/modules_mapping common --deleted_packages=gazelle/python -common --deleted_packages=gazelle/pythonconfig common --deleted_packages=gazelle/python/private +common --deleted_packages=gazelle/pythonconfig common --deleted_packages=tests/integration/bzlmod_lockfile common --deleted_packages=tests/integration/compile_pip_requirements common --deleted_packages=tests/integration/compile_pip_requirements_test_from_external_repo diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 3eb62221c0..ef06a82868 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -3,7 +3,7 @@ ::::{envvar} RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS This variable allows for additional arguments to be provided to the Python interpreter -at bootstrap time when the `bash` bootstrap is used. If +at bootstrap time. If `RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS` were provided as `-Xaaa`, then the command would be: @@ -20,6 +20,12 @@ in the command executed being: python /path/to/debugger.py --port 12345 --file /path/to/file.py ``` +The Bash entry point parses the first line with `read -a` and places these +arguments before the target's `interpreter_args`. Python entry points use +`shlex.split` and place them after the target arguments. This preserves each +entry point's existing precedence. The variable is removed before the +application runs, so nested launchers do not apply it again. + :::{seealso} The {bzl:obj}`interpreter_args` attribute. @@ -31,13 +37,16 @@ The guide on {any}`How to integrate a debugger` :::{versionchanged} 1.7.0 Support added for {bzl:flag}`--bootstrap_impl=system_python`. ::: +:::{versionchanged} VERSION_NEXT_PATCH +Target and additional interpreter arguments also apply to `python app.zip`. +::: :::: :::{envvar} RULES_PYTHON_BOOTSTRAP_VERBOSE When `1`, debug information about bootstrapping of a program is printed to -stderr. +stderr. Temporary runtime directories are retained to help diagnose failures. ::: :::{envvar} RULES_PYTHON_BZLMOD_DEBUG @@ -57,23 +66,61 @@ be removed in a subsequent major `rules_python` version. Defaults to `0` if unse Directory to use as the root for creating files necessary for bootstrapping so that a binary can run. -Only applicable when {bzl:flag}`--venvs_use_declare_symlink=no` is used. - -When set, a binary will attempt to find a unique, reusable, location within this -directory for the files it needs to create to aid startup. The files may not be -deleted upon program exit; it is the responsibility of the caller to ensure -cleanup. - -Manually specifying the directory is useful to lower the overhead of -extracting/creating files on every program execution. By using a location -outside /tmp, longer lived programs don't have to worry about files in /tmp -being cleaned up by the OS. - -If not set, then a temporary directory will be created and deleted upon program -exit. +Applies to runtime-created virtual environments and to `py_zipapp_binary` and +`py_zipapp_test`. Legacy executable ZIPs always use temporary extraction; their +virtual environments cannot persist because they refer to that extraction. + +When set, a binary reuses files beneath this directory. The caller owns their +lifetime and must arrange cleanup. ZIP applications prepare a unique staging +directory and publish it only after setup succeeds. Concurrent launches reuse +the completed result. Startup refuses an incomplete existing cache rather than +removing files another process may be using. Use a fresh extract root if an +existing entry is damaged. + +Each new cache entry is a directory symlink to a completed image in a hidden +backing directory beside it. Publishing the symlink cannot replace another +entry, including one created concurrently. The backing belongs to the caller +once published. Removing just the symlink does not reclaim its image; clean +the extract root when its applications are no longer running. Older cache +entries stored directly as directories remain readable. + +ZIP cache identities include application files, permissions, bootstrap code, +interpreter options and resolved external-runtime facts. Updating a binary can +leave older cache entries behind. Shell and Python entry points share the same +image identity. Published directories follow the caller's umask. + +When unset, bootstraps create temporary runtime directories. Bash entry points +use `TMPDIR` or `/tmp`; Python entry points follow `tempfile`'s directory +selection. On POSIX, an independent process removes these directories +asynchronously after the original interpreter PID exits, including across +exec. The application keeps its native PID, signal delivery and terminal job. +Windows waits for the application child before removing its runtime. Console +Ctrl-C is delivered by Windows; the bootstrap waits for the application's own +cleanup and exit status without forwarding another interrupt. + +The temporary lifetime ends with the original interpreter, even if a forked +child outlives it. Such applications need a persistent extract root. Linux +namespace PID 1 and child subreapers can adopt the cleanup process; waiting for +every child can then block until application exit. Persistent extraction avoids +that process. Namespace or cgroup shutdown can kill it before removal finishes, +and SIGKILL during setup before registration cannot guarantee cleanup. On +systems without a native exit watch or suitable Linux procfs, PID reuse can +delay removal. + +The lifetime and publication behavior above applies to the default application +launchers. Raw templates and older custom rules exposing only `PyExecutableInfo` +retain their existing behavior. Their Windows ZIP adapter re-extracts a persistent +cache on every launch; directory links can make a repeated launch fail. Use +temporary extraction for that compatibility path. :::{versionadded} 1.2.0 ::: + +:::{versionchanged} VERSION_NEXT_PATCH +Ordinary and ZIP entry points preserve native POSIX execution and share +failure-safe temporary cleanup. ZIP caches are published after preparation and +include bootstrap inputs in their identity. +::: :::: :::{envvar} RULES_PYTHON_GAZELLE_VERBOSE diff --git a/news/bootstrap-runtime.fixed.md b/news/bootstrap-runtime.fixed.md new file mode 100644 index 0000000000..2667f82ed7 --- /dev/null +++ b/news/bootstrap-runtime.fixed.md @@ -0,0 +1,8 @@ +(bootstrap) Fixed cancellation and temporary-runtime cleanup across ordinary +binaries, legacy ZIPs, and both entry points of `py_zipapp_binary` and +`py_zipapp_test`. POSIX launchers preserve the application's PID, signal delivery, +and terminal job; cleanup follows interpreter exit. Ordinary runtime-created +virtual environments now prepare in Python. ZIP caches publish only completed +trees, and Python ZIP entry points honor target and additional interpreter +arguments. Paths containing spaces, partial extraction, concurrent startup, and +custom startup templates retain their intended behavior. diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index b4ff84f14a..1898c7c75d 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -105,6 +105,49 @@ filegroup( visibility = NOT_ACTUALLY_PUBLIC, ) +filegroup( + name = "bootstrap_cleanup", + srcs = ["bootstrap_cleanup.py"], + visibility = NOT_ACTUALLY_PUBLIC, +) + +filegroup( + name = "application_sources", + srcs = glob(["_rules_python_bootstrap/*.py"]), + visibility = NOT_ACTUALLY_PUBLIC, +) + +py_library( + name = "application_bootstrap", + srcs = glob( + ["_rules_python_bootstrap/*.py"], + exclude = ["_rules_python_bootstrap/driver.py"], + ), +) + +exports_files( + [ + "_rules_python_bootstrap/driver.py", + "application_python_template.txt", + "application_zip_template.txt", + "application_shell_template.sh", + ], + visibility = NOT_ACTUALLY_PUBLIC, +) + +bzl_library( + name = "application", + srcs = ["application.bzl"], + deps = [ + ":builders", + ":common", + ":py_application_info", + ":py_internal", + "@bazel_skylib//lib:paths", + "@bazel_skylib//lib:shell", + ], +) + filegroup( name = "stage2_bootstrap_template", srcs = ["stage2_bootstrap_template.py"], @@ -542,6 +585,7 @@ bzl_library( name = "py_executable", srcs = ["py_executable.bzl"], deps = [ + ":application", ":attr_builders", ":attributes", ":builders", @@ -562,6 +606,7 @@ bzl_library( ":venv_runfiles", "@bazel_skylib//lib:dicts", "@bazel_skylib//lib:paths", + "@bazel_skylib//lib:shell", "@bazel_skylib//lib:structs", "@bazel_skylib//rules:common_settings", "@rules_cc//cc/common", @@ -969,6 +1014,11 @@ bzl_library( srcs = ["platform_info.bzl"], ) +bzl_library( + name = "py_application_info", + srcs = ["py_application_info.bzl"], +) + bzl_library( name = "py_cc_toolchain_info", srcs = ["py_cc_toolchain_info.bzl"], diff --git a/python/private/_rules_python_bootstrap/__init__.py b/python/private/_rules_python_bootstrap/__init__.py new file mode 100644 index 0000000000..961d42f327 --- /dev/null +++ b/python/private/_rules_python_bootstrap/__init__.py @@ -0,0 +1 @@ +"""Private application preparation; imported only from a declared image path.""" diff --git a/python/private/_rules_python_bootstrap/diagnostics.py b/python/private/_rules_python_bootstrap/diagnostics.py new file mode 100644 index 0000000000..2af872d6b5 --- /dev/null +++ b/python/private/_rules_python_bootstrap/diagnostics.py @@ -0,0 +1,9 @@ +"""Opt-in preparation diagnostics without application arguments or environment.""" + +import os +import sys + + +def verbose(event, *paths): + if os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE"): + print("rules_python bootstrap:", event, *paths, file=sys.stderr, flush=True) diff --git a/python/private/_rules_python_bootstrap/driver.py b/python/private/_rules_python_bootstrap/driver.py new file mode 100644 index 0000000000..93514a2a50 --- /dev/null +++ b/python/private/_rules_python_bootstrap/driver.py @@ -0,0 +1,27 @@ +"""Load the declared private package without using application import paths.""" + +import sys + +if not getattr(sys.flags, "safe_path", False) and not sys.flags.isolated and sys.path: + del sys.path[0] + +import importlib.machinery +import importlib.util +import os + +spec = importlib.machinery.PathFinder.find_spec( + "_rules_python_bootstrap", [os.path.dirname(os.path.dirname(__file__))] +) +package = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = package +spec.loader.exec_module(package) + +if __name__ == "__main__": + from _rules_python_bootstrap import entry + + if sys.argv[1] == "directory": + sys.exit(entry.shell_directory_main(sys.argv[2:])) + elif sys.argv[1] == "prepare-archive": + entry.prepare_archive(sys.argv[2], sys.argv[3], cached=sys.argv[4] == "1") + else: + raise ValueError("Unknown bootstrap entry: " + sys.argv[1]) diff --git a/python/private/_rules_python_bootstrap/entry.py b/python/private/_rules_python_bootstrap/entry.py new file mode 100644 index 0000000000..b2660a1423 --- /dev/null +++ b/python/private/_rules_python_bootstrap/entry.py @@ -0,0 +1,302 @@ +"""Entry adapters joining application preparation, storage, and launch.""" + +import hashlib +import json +import os +import shlex +import zipfile +from dataclasses import replace + +from .environment import prepare_venv, resolve_runtime, venv_layout_identity +from .model import Application, PreparedApplication, invocation +from .process import Cancellation, execute +from .storage import Workspace, complete_image, extract_archive, publish_image + + +def _python_options(application): + extra = os.environ.pop("RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS", "") + return list(application.interpreter_args) + shlex.split(extra) + + +def _read_application(path): + with open(path, "rb") as source: + return Application.read(source.read()) + + +def _cache_path(metadata, runtime=None, runfiles=None): + root = os.environ.get("RULES_PYTHON_EXTRACT_ROOT") + if not root or not metadata["cache"]: + return None + identity = metadata["identity"] + if runtime is not None: + identity += "-" + runtime.identity(runfiles) + name = metadata["name"] + if os.name == "nt": + name = os.path.basename(name) + identity = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:32] + return os.path.abspath(os.path.join(root, name, identity)) + + +def _prepared_image(application, image): + runfiles = os.path.join(image, "runfiles") + if application.venv is not None: + executable = os.path.join(runfiles, application.venv.executable) + else: + executable = application.interpreter.locate(runfiles) + if not os.path.isfile(executable) or not os.access(executable, os.X_OK): + raise RuntimeError("Prepared interpreter not executable: " + executable) + return PreparedApplication(runfiles, executable) + + +def _prepare_image(application, image, destination, cancellation, runtime): + runfiles = os.path.join(image, "runfiles") + # An application defined entirely in external repositories may have no + # files in the main workspace. It still needs Bazel's working directory. + os.makedirs(os.path.join(runfiles, application.workspace), exist_ok=True) + if application.venv is not None: + prepare_venv( + application, + runfiles, + os.path.join(runfiles, application.venv.root), + cancellation, + runtime=runtime, + image_owned=True, + final_runfiles=os.path.join(destination, "runfiles") + if destination + else None, + ) + cancellation.check() + if destination: + publish_image(image, destination) + image = destination + cancellation.check() + return _prepared_image(application, image) + + +def directory_main( + runfiles, + spec_path, + arguments, + *, + options=None, + selected=False, + temporary_directory=None, + entry=None, + cleanup=None, +): + application = _read_application(os.path.join(runfiles, spec_path)) + if entry is not None: + application = replace(application, entry=entry) + if cleanup is not None: + application = replace(application, cleanup=cleanup) + options = ( + _python_options(application) + if options is None + else options + list(application.interpreter_args) + ) + with Cancellation() as cancellation, Workspace( + directory=temporary_directory, + retain=bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")), + ) as workspace: + venv = application.venv + if venv is None or not venv.recreate: + executable = ( + os.path.join(runfiles, venv.executable) + if venv is not None + else application.interpreter.locate(runfiles) + ) + else: + runtime = ( + resolve_runtime( + application.interpreter, runfiles, cancellation, selected=selected + ) + if application.interpreter.resolve + else None + ) + root = os.environ.get("RULES_PYTHON_EXTRACT_ROOT") + destination = None + if root: + with open(os.path.join(runfiles, spec_path), "rb") as source: + key = hashlib.sha256(source.read() + os.fsencode(runfiles)) + key.update(venv_layout_identity(application, runfiles)) + if runtime is not None: + key.update(runtime.identity().encode("ascii")) + destination = os.path.abspath( + os.path.join(root, venv.root, key.hexdigest()) + ) + if destination and complete_image(destination): + executable = os.path.join( + destination, "venv", os.path.relpath(venv.executable, venv.root) + ) + else: + if destination: + os.makedirs(os.path.dirname(destination), exist_ok=True) + workspace.directory = os.path.dirname(destination) + image = os.path.join(workspace.allocate(), "image") + cancellation.check() + executable = prepare_venv( + application, + runfiles, + os.path.join(image, "venv"), + cancellation, + runtime=runtime, + ) + if destination: + relative = os.path.relpath(executable, image) + publish_image(image, destination) + executable = os.path.join(destination, relative) + workspace.remove() + prepared = PreparedApplication(runfiles, executable) + command = invocation(application, prepared, options, arguments, os.environ) + return execute( + command, + workspace, + os.path.join(runfiles, application.cleanup), + cancellation, + ) + + +def archive_main( + archive, + metadata, + arguments, + *, + options=None, + selected=False, + temporary_directory=None, +): + with zipfile.ZipFile(archive) as source: + application = Application.read(source.read("runfiles/" + metadata["spec"])) + options = ( + _python_options(application) + if options is None + else options + list(application.interpreter_args) + ) + with Cancellation() as cancellation, Workspace( + directory=temporary_directory, + retain=bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")), + ) as workspace: + runtime = None + if application.interpreter.kind != "runfiles": + runtime = resolve_runtime( + application.interpreter, "", cancellation, selected=selected + ) + destination = _cache_path(metadata, runtime) + needs_resolution = application.interpreter.resolve and runtime is None + if destination and not needs_resolution and complete_image(destination): + prepared = _prepared_image(application, destination) + else: + if destination: + os.makedirs(os.path.dirname(destination), exist_ok=True) + workspace.directory = os.path.dirname(destination) + image = os.path.join(workspace.allocate(), "image") + cancellation.check() + extract_archive(archive, image, cancellation) + if application.interpreter.resolve and runtime is None: + runtime = resolve_runtime( + application.interpreter, + os.path.join(image, "runfiles"), + cancellation, + ) + destination = _cache_path( + metadata, runtime, os.path.join(image, "runfiles") + ) + if destination and complete_image(destination): + prepared = _prepared_image(application, destination) + else: + prepared = _prepare_image( + application, image, destination, cancellation, runtime + ) + if destination: + workspace.remove() + # An archive has no caller-provided runfiles manifest of its own. + environment = dict(os.environ) + environment.pop("RUNFILES_MANIFEST_FILE", None) + command = invocation( + application, + prepared, + options, + arguments, + environment, + temporary_zip=os.path.dirname(prepared.runfiles) if not destination else "", + ) + return execute( + command, + workspace, + os.path.join(prepared.runfiles, application.cleanup), + cancellation, + ) + + +def prepare_archive(workspace, image, *, cached=False): + """Prepare under shell ownership and emit a six-field data-only result.""" + with open(os.path.join(image, "_rules_python_archive.json")) as source: + metadata = json.load(source) + application = _read_application(os.path.join(image, "runfiles", metadata["spec"])) + with Cancellation() as cancellation: + runtime = ( + resolve_runtime( + application.interpreter, + os.path.join(image, "runfiles"), + cancellation, + selected=True, + ) + if application.interpreter.resolve + else None + ) + destination = _cache_path(metadata, runtime, os.path.join(image, "runfiles")) + if cached: + if ( + not destination + or os.path.abspath(image) != destination + or not complete_image(image) + ): + raise RuntimeError("Invalid cached application passed by launcher") + prepared = _prepared_image(application, image) + elif destination and complete_image(destination): + prepared = _prepared_image(application, destination) + else: + prepared = _prepare_image( + application, image, destination, cancellation, runtime + ) + command = invocation(application, prepared, (), (), os.environ) + fields = [ + "1", + command.cwd or "", + prepared.runfiles, + prepared.executable, + os.path.join(prepared.runfiles, application.entry), + "" if destination else workspace, + ] + cancellation.check() + with open(os.path.join(workspace, "invocation"), "wb") as result: + for field in fields: + result.write(os.fsencode(field) + b"\0") + cancellation.check() + + +def shell_directory_main(arguments): + runfiles, spec, entry, cleanup, count = arguments[:5] + count = int(count) + return directory_main( + runfiles, + spec, + arguments[5 + count :], + options=arguments[5 : 5 + count], + selected=True, + temporary_directory=os.environ.get("TMPDIR") or "/tmp", + entry=entry, + cleanup=cleanup, + ) + + +def shell_archive_main(archive, metadata, arguments): + count = int(arguments[0]) + return archive_main( + archive, + metadata, + arguments[1 + count :], + options=arguments[1 : 1 + count], + selected=True, + temporary_directory=os.environ.get("TMPDIR") or "/tmp", + ) diff --git a/python/private/_rules_python_bootstrap/environment.py b/python/private/_rules_python_bootstrap/environment.py new file mode 100644 index 0000000000..37170bc2c1 --- /dev/null +++ b/python/private/_rules_python_bootstrap/environment.py @@ -0,0 +1,281 @@ +"""Resolve runtime facts and prepare the image's declared Python environment.""" + +import hashlib +import json +import os +import site +import sys +from dataclasses import dataclass + +from .diagnostics import verbose +from .process import run_child +from .storage import write_atomic + + +@dataclass(frozen=True) +class Runtime: + executable: str + prefix: str + site_packages: str + version: tuple + abi: str + + def identity(self, runfiles=None): + def normalize(path): + if runfiles and _inside(path, runfiles): + return "/" + os.path.relpath(path, runfiles) + return path + + value = [ + normalize(self.executable), + normalize(self.prefix), + self.site_packages, + self.version, + self.abi, + ] + return hashlib.sha256(json.dumps(value).encode("utf-8")).hexdigest() + + +def current_runtime(): + return Runtime( + sys.executable, + sys.base_prefix, + os.path.normpath(site.getsitepackages(["."])[-1]), + tuple(sys.version_info[:3]), + getattr(sys, "abiflags", ""), + ) + + +def resolve_runtime(interpreter, runfiles, cancellation, *, selected=False): + executable = interpreter.locate(runfiles) + verbose("selected interpreter", executable) + if selected: + return current_runtime() + source = ( + "import json,os,site,sys; " + "print(json.dumps([sys.executable,sys.base_prefix," + "os.path.normpath(site.getsitepackages(['.'])[-1])," + "list(sys.version_info[:3]),getattr(sys,'abiflags','')]))" + ) + environment = dict(os.environ) + if runfiles: + environment["RUNFILES_DIR"] = runfiles + environment.pop("RUNFILES_MANIFEST_FILE", None) + environment.pop("__PYVENV_LAUNCHER__", None) + values = json.loads( + run_child( + [executable, "-I", "-S", "-c", source], + cancellation, + capture=True, + env=environment, + ) + ) + return Runtime(values[0], values[1], values[2], tuple(values[3]), values[4]) + + +def venv_layout_identity(application, runfiles): + """Hash preparation inputs without traversing linked dependency trees.""" + venv = application.venv + digest = hashlib.sha256() + with open(os.path.join(runfiles, venv.links), "rb") as source: + links = source.read() + digest.update(links) + directories = {"", os.path.dirname(os.path.relpath(venv.executable, venv.root))} + for line in links.splitlines(): + path, _target = json.loads(line) + directory = os.path.dirname(os.path.normpath(path)) + while directory: + directories.add(directory) + directory = os.path.dirname(directory) + for directory in sorted(directories): + path = os.path.join(runfiles, venv.root, directory) + names = sorted(os.listdir(path)) if os.path.isdir(path) else [] + digest.update(json.dumps([directory, names]).encode("utf-8")) + # Changing preparation code must invalidate old materializations too. + for name in ( + "diagnostics.py", + "environment.py", + "entry.py", + "model.py", + "process.py", + "storage.py", + ): + with open(os.path.join(os.path.dirname(__file__), name), "rb") as source: + digest.update(source.read()) + return digest.digest() + + +def _link(path, target, *, replace=False, relative=False): + os.makedirs(os.path.dirname(path), exist_ok=True) + directory = os.path.isdir(target) + if relative: + target = os.path.relpath(target, os.path.dirname(path)) + if os.path.lexists(path): + if not replace: + return + if os.path.isdir(path) and not os.path.islink(path): + raise RuntimeError("Cannot replace directory with runtime link: " + path) + os.unlink(path) + os.symlink(target, path, target_is_directory=directory) + + +def _inside(path, root): + try: + return os.path.commonpath( + [os.path.abspath(path), os.path.abspath(root)] + ) == os.path.abspath(root) + except ValueError: + return False + + +def _overlay(source, destination, overrides, cancellation): + """Link whole directories, descending only where explicit links require it.""" + pending = [(source, destination, overrides)] + while pending: + src, dst, links = pending.pop() + cancellation.check() + if not links: + if os.path.exists(src): + _link(dst, src) + continue + os.makedirs(dst, exist_ok=True) + names = set(os.listdir(src)) if os.path.isdir(src) else set() + branches = {} + for path, target in links.items(): + first, separator, rest = path.partition(os.sep) + names.add(first) + if separator: + branches.setdefault(first, {})[rest] = target + for name in names: + path = os.path.join(dst, name) + if name in links: + _link(path, links[name]) + else: + pending.append((os.path.join(src, name), path, branches.get(name, {}))) + + +def prepare_venv( + application, + runfiles, + destination, + cancellation, + *, + runtime=None, + image_owned=False, + final_runfiles=None, +): + """Prepare only declared runtime-dependent files, preserving the image layout.""" + verbose("preparing environment", destination) + venv = application.venv + if venv is None: + return application.interpreter.locate(runfiles) + source = os.path.join(runfiles, venv.root) + actual = ( + runtime.executable + if runtime is not None + else application.interpreter.locate(runfiles) + ) + actual = os.path.abspath(actual) + relative_executable = os.path.relpath(venv.executable, venv.root) + executable = os.path.join(destination, relative_executable) + os.makedirs(destination, exist_ok=True) + site_packages = runtime.site_packages if runtime is not None else venv.site_packages + + if os.path.abspath(source) != os.path.abspath(destination): + overrides = {} + with open(os.path.join(runfiles, venv.links)) as stream: + for line in stream: + path, target = json.loads(line) + overrides[os.path.normpath(path)] = os.path.normpath( + os.path.join(runfiles, target) + ) + # The interpreter is recreated below, never copied as a build-time marker. + overrides.pop(os.path.normpath(relative_executable), None) + overrides.pop("pyvenv.cfg", None) + names = set(os.listdir(source)) + names.update(path.split(os.sep, 1)[0] for path in overrides) + for name in names: + cancellation.check() + if name in ( + "pyvenv.cfg", + os.path.dirname(relative_executable), + "lib", + "Lib", + ): + continue + prefix = name + os.sep + branch = { + key[len(prefix) :]: val + for key, val in overrides.items() + if key.startswith(prefix) + } + if name in overrides: + _link(os.path.join(destination, name), overrides[name]) + else: + _overlay( + os.path.join(source, name), + os.path.join(destination, name), + branch, + cancellation, + ) + # Link the site's highest common directory when no overlay is necessary. + prefix = os.path.normpath(venv.site_packages) + os.sep + site_links = { + key[len(prefix) :]: val + for key, val in overrides.items() + if key.startswith(prefix) + } + _overlay( + os.path.join(source, venv.site_packages), + os.path.join(destination, site_packages), + site_links, + cancellation, + ) + bin_name = os.path.dirname(relative_executable) + bin_source = os.path.join(source, bin_name) + os.makedirs(os.path.join(destination, bin_name), exist_ok=True) + if os.path.isdir(bin_source): + for name in os.listdir(bin_source): + if name != os.path.basename(executable): + _link( + os.path.join(destination, bin_name, name), + os.path.join(bin_source, name), + ) + for path, target in overrides.items(): + if path.startswith(bin_name + os.sep): + _link(os.path.join(destination, path), target, replace=True) + elif os.path.normpath(site_packages) != os.path.normpath(venv.site_packages): + _link( + os.path.join(destination, site_packages), + os.path.join(destination, venv.site_packages), + relative=True, + ) + + cancellation.check() + relative = image_owned and _inside(actual, runfiles) + _link(executable, actual, replace=True, relative=relative) + if os.name == "nt": + home = runtime.prefix if runtime is not None else os.path.dirname(actual) + for name in os.listdir(home): + if name.endswith((".dll", ".pdb")): + _link( + os.path.join(os.path.dirname(executable), name), + os.path.join(home, name), + replace=True, + relative=relative, + ) + if final_runfiles and relative: + home = os.path.join(final_runfiles, os.path.relpath(home, runfiles)) + write_atomic( + os.path.join(destination, "pyvenv.cfg"), "home = {}\n".format(home) + ) + elif not os.path.lexists(os.path.join(destination, "pyvenv.cfg")): + if image_owned: + write_atomic(os.path.join(destination, "pyvenv.cfg"), "") + else: + _link( + os.path.join(destination, "pyvenv.cfg"), + os.path.join(source, "pyvenv.cfg"), + ) + cancellation.check() + return executable diff --git a/python/private/_rules_python_bootstrap/model.py b/python/private/_rules_python_bootstrap/model.py new file mode 100644 index 0000000000..8779bdbccd --- /dev/null +++ b/python/private/_rules_python_bootstrap/model.py @@ -0,0 +1,101 @@ +"""The build-produced application contract and resolved launch values.""" + +import json +import os +import shutil +from dataclasses import dataclass +from typing import Optional + + +@dataclass(frozen=True) +class Interpreter: + kind: str + path: str + resolve: bool + + def locate(self, runfiles): + if self.kind == "runfiles": + result = os.path.join(runfiles, self.path) + elif self.kind == "absolute": + result = self.path + elif self.kind == "path": + result = shutil.which(self.path) + else: + raise ValueError("Unknown interpreter kind: " + self.kind) + if not result or not os.path.isfile(result) or not os.access(result, os.X_OK): + raise RuntimeError("Python interpreter not executable: " + self.path) + return os.path.abspath(result) + + +@dataclass(frozen=True) +class Venv: + root: str + executable: str + site_packages: str + recreate: bool + links: str + + +@dataclass(frozen=True) +class Application: + entry: str + workspace: str + interpreter: Interpreter + interpreter_args: tuple + venv: Optional[Venv] + cleanup: str + + @classmethod + def read(cls, contents): + value = json.loads(contents) + if value.pop("version") != 1: + raise ValueError("Unsupported rules_python application version") + value["interpreter"] = Interpreter(**value["interpreter"]) + if value["venv"] is not None: + value["venv"] = Venv(**value["venv"]) + value["interpreter_args"] = tuple(value["interpreter_args"]) + return cls(**value) + + +@dataclass(frozen=True) +class PreparedApplication: + runfiles: str + executable: str + + +@dataclass(frozen=True) +class Invocation: + executable: str + argv: tuple + environment: dict + cwd: Optional[str] + + +def invocation( + application, prepared, interpreter_args, arguments, environment, temporary_zip="" +): + """Construct the final invocation without modifying process state.""" + env = dict(environment) + env.pop("RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS", None) + env.pop("__PYVENV_LAUNCHER__", None) + env["RUNFILES_DIR"] = prepared.runfiles + env.setdefault("PYTHONSAFEPATH", "1") + if temporary_zip: + env.pop("RUNFILES_MANIFEST_FILE", None) + if env.get("RULES_PYTHON_TESTING_TELL_RUNFILES_ROOT"): + env["RULES_PYTHON_TESTING_RUNFILES_ROOT"] = prepared.runfiles + cwd = None + if env.get("RUN_UNDER_RUNFILES") == "1": + cwd = os.path.join(prepared.runfiles, application.workspace) + options = list(interpreter_args) + if temporary_zip: + options.insert(0, "-XRULES_PYTHON_ZIP_DIR=" + temporary_zip) + entry = os.path.join(prepared.runfiles, application.entry) + if not os.path.isfile(entry): + raise RuntimeError("Application entry point not found: " + entry) + return Invocation( + prepared.executable, + tuple([prepared.executable] + options + [entry] + list(arguments)), + env, + cwd, + ) diff --git a/python/private/_rules_python_bootstrap/process.py b/python/private/_rules_python_bootstrap/process.py new file mode 100644 index 0000000000..5fa04ef02f --- /dev/null +++ b/python/private/_rules_python_bootstrap/process.py @@ -0,0 +1,137 @@ +"""Preparation cancellation, owned setup children, and platform launch.""" + +import contextlib +import os +import signal +import subprocess +import sys + +from .diagnostics import verbose + + +class Cancelled(BaseException): + def __init__(self, signum): + self.signum = signum + + +class Cancellation: + """Record cancellation; raise only at explicit resource-safe boundaries.""" + + def __init__(self): + self.pending = None + self.handlers = {} + self.mask = None + + def __enter__(self): + signals = [signal.SIGINT] + if os.name == "posix": + signals += [signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT] + self.mask = signal.pthread_sigmask(signal.SIG_BLOCK, []) + for signum in signals: + previous = signal.getsignal(signum) + if previous != signal.SIG_IGN: + self.handlers[signum] = previous + signal.signal(signum, self._record) + return self + + def _record(self, signum, _frame): + if self.pending is None: + self.pending = signum + + def check(self): + if self.pending is not None: + raise Cancelled(self.pending) + + def restore(self): + for signum, previous in self.handlers.items(): + signal.signal(signum, previous) + if os.name == "posix" and self.mask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, self.mask) + + def prepare_exec(self): + if os.name == "posix": + assert self.mask is not None, "POSIX cancellation context was not entered" + # Stop recording cancellation before restoring the caller's mask. + # Pending kernel signals then see the caller's dispositions; requests + # already recorded by Python are checked after every handler is restored. + signal.pthread_sigmask(signal.SIG_BLOCK, self.handlers) + self.check() + for signum, previous in self.handlers.items(): + signal.signal(signum, previous) + self.check() + signal.pthread_sigmask(signal.SIG_SETMASK, self.mask) + else: + raise RuntimeError("Native exec requires a POSIX cancellation context") + + def __exit__(self, _kind, error, _traceback): + self.restore() + if isinstance(error, Cancelled): + if os.name == "posix": + signal.signal(error.signum, signal.SIG_DFL) + os.kill(os.getpid(), error.signum) + raise SystemExit(128 + error.signum) + + +def run_child(argv, cancellation, *, capture=False, **kwargs): + """Keep setup children owned through cancellation and reap before rollback.""" + cancellation.check() + child = subprocess.Popen( + argv, stdout=subprocess.PIPE if capture else None, **kwargs + ) + try: + while True: + cancellation.check() + try: + output, _ = child.communicate(timeout=0.1) + break + except subprocess.TimeoutExpired: + continue + cancellation.check() + if child.returncode: + raise subprocess.CalledProcessError(child.returncode, argv, output) + return output + finally: + if child.poll() is None: + child.kill() + child.wait() + + +def execute(command, workspace, cleanup_helper, cancellation): + """Transfer temporary ownership before native exec; Windows waits locally.""" + cancellation.check() + verbose("launching interpreter", command.executable) + if os.name == "nt": + child = subprocess.Popen(command.argv, env=command.environment, cwd=command.cwd) + try: + cancellation.check() + # Windows delivers console Ctrl-C to both processes. Keep recording + # it here without terminating the application's own cleanup or + # forwarding a second interrupt. The application chooses its status. + status = child.wait() + # Older CPython sys.exit converts through a signed 32-bit C long. + return status if status < 0x80000000 else status - 0x100000000 + finally: + if child.poll() is None: + child.kill() + child.wait() + + if workspace.path and not workspace.retain: + run_child( + [ + command.executable, + "-I", + "-S", + cleanup_helper, + str(os.getpid()), + workspace.path, + ], + cancellation, + ) + if command.cwd is not None: + os.chdir(command.cwd) + for stream in (sys.stdout, sys.stderr): + if stream is not None: + with contextlib.suppress(OSError, ValueError): + stream.flush() + cancellation.prepare_exec() + os.execve(command.executable, command.argv, command.environment) diff --git a/python/private/_rules_python_bootstrap/storage.py b/python/private/_rules_python_bootstrap/storage.py new file mode 100644 index 0000000000..eb57691672 --- /dev/null +++ b/python/private/_rules_python_bootstrap/storage.py @@ -0,0 +1,171 @@ +"""Owned preparation workspaces and complete immutable application images.""" + +import errno +import os +import shutil +import stat +import tempfile +import zipfile + +from .diagnostics import verbose + +COMPLETION_FILE = ".rules_python_complete" + + +class Workspace: + """Own at most one temporary root; borrowed and published images are separate.""" + + def __init__(self, *, directory=None, retain=False): + self.directory = directory + self.retain = retain + self.path = None + + def __enter__(self): + return self + + def allocate(self): + if self.path is None: + self.path = os.path.abspath( + tempfile.mkdtemp(prefix="rules_python.", dir=self.directory) + ) + verbose("workspace", self.path) + if self.retain: + verbose("retaining workspace", self.path) + return self.path + + def remove(self): + if self.path is not None: + if not self.retain: + remove_tree(self.path) + self.path = None + + def __exit__(self, *_error): + self.remove() + + +def remove_tree(path): + """Remove an owned tree, including read-only archive inputs on Windows.""" + + def retry_readonly(function, failed_path, error): + if isinstance(error[1], FileNotFoundError): + return + if ( + os.name == "nt" + and isinstance(error[1], PermissionError) + and not os.path.islink(failed_path) + ): + os.chmod(failed_path, os.stat(failed_path).st_mode | stat.S_IWRITE) + function(failed_path) + return + raise error[1] + + shutil.rmtree(path, onerror=retry_readonly) + + +def complete_image(path): + if not os.path.lexists(path): + return False + if not os.path.isfile(os.path.join(path, COMPLETION_FILE)): + raise RuntimeError( + "Incomplete Python runtime at " + + os.fspath(path) + + "; use a clean extract root" + ) + verbose("using prepared image", path) + return True + + +def publish_image(staging, destination): + """Publish a prepared image, or discard staging in favor of a complete winner.""" + # Restore the normal directory permissions without changing process-wide umask. + permissions = os.path.join(staging, ".permissions") + os.mkdir(permissions, 0o777) + mode = stat.S_IMODE(os.stat(permissions).st_mode) + os.rmdir(permissions) + os.chmod(staging, mode) + with open(os.path.join(staging, COMPLETION_FILE), "w") as stream: + stream.write("rules_python application 1\n") + if complete_image(destination): + remove_tree(staging) + return + + # POSIX rename can overwrite an empty directory created after an existence + # check. Keep the completed image separately and publish an exclusive link. + # This also works on older libc versions without an exclusive rename API. + parent = os.path.dirname(destination) + backing = tempfile.mkdtemp(prefix=".rules_python_image.", dir=parent) + retain_backing = False + try: + os.chmod(backing, mode) + image = os.path.join(backing, "image") + os.rename(staging, image) + target = os.path.relpath(image, parent) + try: + os.symlink(target, destination, target_is_directory=True) + except OSError as error: + # A network filesystem may report failure after creating the link. + # Never delete an image that may already be visible to readers. + try: + retain_backing = os.readlink(destination) == target + except OSError as inspection: + if inspection.errno not in (errno.ENOENT, errno.EINVAL): + retain_backing = True + raise error + if not retain_backing and ( + error.errno != errno.EEXIST or not complete_image(destination) + ): + raise + else: + retain_backing = True + if retain_backing: + verbose("published image", destination) + finally: + if not retain_backing: + remove_tree(backing) + + +def extract_archive(archive, destination, cancellation): + """Materialize a build-produced image, including executable modes and links.""" + verbose("extracting archive", archive, "into", destination) + os.makedirs(destination) + links = [] + with zipfile.ZipFile(archive) as source: + for info in source.infolist(): + cancellation.check() + path = os.path.abspath(os.path.join(destination, info.filename)) + if os.path.commonpath([destination, path]) != destination: + raise ValueError( + "Archive member escapes application image: " + info.filename + ) + mode = info.external_attr >> 16 + if stat.S_ISLNK(mode): + links.append((info.filename, path, source.read(info).decode("utf-8"))) + continue + source.extract(info, destination) + if mode: + os.chmod(path, stat.S_IMODE(mode)) + for name, path, target in links: + cancellation.check() + os.makedirs(os.path.dirname(path), exist_ok=True) + if os.name == "nt": + target_name = os.path.normpath( + os.path.join(os.path.dirname(name), target) + ) + try: + directory = source.getinfo(target_name.replace("\\", "/")).is_dir() + except KeyError: + directory = True + else: + directory = False + os.symlink(target, path, target_is_directory=directory) + + +def write_atomic(path, contents): + fd, temporary = tempfile.mkstemp(prefix=".config.", dir=os.path.dirname(path)) + try: + with os.fdopen(fd, "w") as stream: + stream.write(contents) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) diff --git a/python/private/application.bzl b/python/private/application.bzl new file mode 100644 index 0000000000..070de9d52d --- /dev/null +++ b/python/private/application.bzl @@ -0,0 +1,245 @@ +"""Build an application once, then expose it as a directory or archive.""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//lib:shell.bzl", "shell") +load(":builders.bzl", "builders") +load(":common.bzl", "ExplicitSymlink", "actions_run", "is_windows_platform", "maybe_create_repo_mapping", "runfiles_root_path") +load(":py_application_info.bzl", "PyApplicationInfo") +load(":py_internal.bzl", "py_internal") + +APPLICATION_ATTRS = { + "_application_default_python": attr.label(default = Label("//python/private:python_bootstrap_template.txt"), allow_single_file = True), + "_application_default_shell": attr.label(default = Label("//python/private:stage1_bootstrap_template"), allow_single_file = True), + "_application_default_zip": attr.label(default = Label("//python/private/zipapp:zip_main_template"), allow_single_file = True), + "_application_driver": attr.label(default = Label("//python/private:_rules_python_bootstrap/driver.py"), allow_single_file = True), + "_application_python": attr.label(default = Label("//python/private:application_python_template.txt"), allow_single_file = True), + "_application_shell": attr.label(default = Label("//python/private:application_shell_template.sh"), allow_single_file = True), + "_application_sources": attr.label(default = Label("//python/private:application_sources")), + "_application_zip": attr.label(default = Label("//python/private:application_zip_template.txt"), allow_single_file = True), + "_application_zip_main_maker": attr.label(default = Label("//tools/zipapp:zip_main_maker"), cfg = "exec"), + "_application_zipper": attr.label(default = Label("//tools/zipapp:zipper"), cfg = "exec"), +} + +def _link_record(link): + return json.encode([link.venv_path, link.link_to_path]) + +def create_application(ctx, *, runtime, stage2, venv, runfiles, interpreter_args): + """Declare the common image and launch specification from executable inputs. + + Args: + ctx: Rule context. + runtime: Selected PyRuntimeInfo. + stage2: Selected application entry File. + venv: Executable-owned virtual environment layout. + runfiles: Application and runtime runfiles without the outer executable. + interpreter_args: Arguments bound to the target interpreter. + + Returns: + The private PyApplicationInfo consumed by launchers and packaging. + """ + spec = ctx.actions.declare_file(ctx.label.name + ".application.json") + link_file = ctx.actions.declare_file(ctx.label.name + ".application_links.jsonl") + links = depset(transitive = [venv.interpreter_symlinks, venv.lib_symlinks]) + content = ctx.actions.args() + content.set_param_file_format("multiline") + content.add_all(links, map_each = _link_record) + ctx.actions.write(link_file, content) + actual = venv.interpreter_actual_path + settings = { + "cleanup": runfiles_root_path(ctx, ctx.file._bootstrap_cleanup.short_path), + "entry": runfiles_root_path(ctx, stage2.short_path), + "interpreter": { + "kind": "runfiles" if runtime.interpreter else ("absolute" if paths.is_absolute(actual) else "path"), + "path": actual, + "resolve": not runtime.supports_build_time_venv, + }, + "interpreter_args": interpreter_args, + "venv": { + "executable": runfiles_root_path(ctx, venv.interpreter.short_path), + "links": runfiles_root_path(ctx, link_file.short_path), + "recreate": venv.recreate_venv_at_runtime, + "root": venv.venv_root, + "site_packages": venv.venv_site_packages, + } if venv.interpreter else None, + "version": 1, + "workspace": ctx.workspace_name, + } + ctx.actions.write(spec, json.encode(settings)) + image_runfiles = runfiles.merge_all([ + ctx.runfiles([spec, link_file, stage2, ctx.file._bootstrap_cleanup] + ctx.files._application_sources), + ctx.runfiles(venv.files_without_interpreter), + venv.lib_runfiles, + venv.interpreter_runfiles, + ]) + interpreter_links = [] + if venv.interpreter and runtime.interpreter: + interpreter_links.append(ExplicitSymlink( + runfiles_path = settings["venv"]["executable"], + venv_path = paths.relativize(settings["venv"]["executable"], venv.venv_root), + link_to_path = actual, + files = depset([runtime.interpreter]), + )) + return PyApplicationInfo( + spec = spec, + settings = settings, + runfiles = image_runfiles, + symlinks = depset(interpreter_links, transitive = [links]), + ) + +def is_default_bootstrap(ctx, template): + return template in (ctx.file._application_default_python, ctx.file._application_default_shell) + +def create_application_launcher(ctx, *, application, output, shell_entry, archive = False, cache = False, shebang = "#!/usr/bin/env python3"): + """Generate a thin entry adapter from the same application specification. + + Args: + ctx: Rule context. + application: Private application image provider. + output: Launcher File to write. + shell_entry: Whether the adapter starts in Bash instead of Python. + archive: Whether the shell adapter has an appended archive. + cache: Whether archive preparation permits a persistent extract root. + shebang: Interpreter directive for a Python adapter. + """ + settings = application.settings + venv = settings["venv"] + interpreter = settings["interpreter"] + subs = { + "%application_spec%": repr(runfiles_root_path(ctx, application.spec.short_path)), + "%archive%": "1" if archive else "0", + "%bootstrap_parent%": repr(paths.dirname(paths.dirname(runfiles_root_path(ctx, ctx.file._application_driver.short_path)))), + "%cache%": "1" if cache else "0", + "%cache_name%": shell.quote(paths.join(ctx.label.repo_name or "_main", ctx.label.package, ctx.label.name)), + "%cleanup_shell%": shell.quote(settings["cleanup"]), + "%driver_shell%": shell.quote(runfiles_root_path(ctx, ctx.file._application_driver.short_path)), + "%entry%": repr(settings["entry"]), + "%entry_shell%": shell.quote(settings["entry"]), + "%interpreter_args_shell%": "\n".join([shell.quote(arg) for arg in settings["interpreter_args"]]), + "%interpreter_kind%": shell.quote(interpreter["kind"]), + "%interpreter_shell%": shell.quote(interpreter["path"]), + "%recreate%": "1" if venv and venv["recreate"] else "0", + "%resolve%": "1" if interpreter["resolve"] else "0", + "%shebang%": shebang, + "%spec_shell%": shell.quote(runfiles_root_path(ctx, application.spec.short_path)), + "%venv_executable_shell%": shell.quote(venv["executable"] if venv else ""), + "%workspace_shell%": shell.quote(settings["workspace"]), + } + ctx.actions.expand_template( + template = ctx.file._application_shell if shell_entry else ctx.file._application_python, + output = output, + substitutions = subs, + is_executable = True, + ) + +def _is_symlink(file): + return str(int(file.is_symlink)) if hasattr(file, "is_symlink") else "-1" + +def _empty_files(callback): + return ["rf-empty|" + path for path in callback().to_list()] + +def _runfile(file): + return "rf-file|" + _is_symlink(file) + "|" + file.short_path + "|" + file.path + +def _symlink(entry): + return "rf-symlink|" + _is_symlink(entry.target_file) + "|" + entry.path + "|" + entry.target_file.path + +def _root_symlink(entry): + return "rf-root-symlink|" + _is_symlink(entry.target_file) + "|" + entry.path + "|" + entry.target_file.path + +def _explicit_symlink(entry): + return "symlink|" + entry.runfiles_path + "|" + entry.link_to_path + +def _manifest(ctx, manifest, application, inputs): + runfiles = application.runfiles + manifest.add_all([lambda: runfiles.empty_filenames], map_each = _empty_files, allow_closure = True) + manifest.add_all(runfiles.files, map_each = _runfile) + manifest.add_all(runfiles.symlinks, map_each = _symlink) + manifest.add_all(runfiles.root_symlinks, map_each = _root_symlink) + manifest.add_all(application.symlinks, map_each = _explicit_symlink) + inputs.add(runfiles.files) + inputs.add([entry.target_file for entry in runfiles.symlinks.to_list()]) + inputs.add([entry.target_file for entry in runfiles.root_symlinks.to_list()]) + for entry in application.symlinks.to_list(): + inputs.add(entry.files) + mapping = maybe_create_repo_mapping(ctx = ctx, runfiles = runfiles) + if mapping: + manifest.add(mapping.path, format = "rf-root-symlink|0|_repo_mapping|%s") + inputs.add(mapping) + +def create_application_archive(ctx, *, application, output, template, cache, compression = ""): + """Package the executable's declared application image without rebuilding it. + + Args: + ctx: Rule context. + application: Private application image provider from the binary. + output: Archive File to write. + template: Runtime-selected ZIP main template, including custom templates. + cache: Whether the archive permits persistent preparation. + compression: Optional zipper compression level. + """ + main = ctx.actions.declare_file(output.basename + ".main.py", sibling = output) + metadata_file = ctx.actions.declare_file(output.basename + ".metadata.json", sibling = output) + modern = template == ctx.file._application_default_zip + template = ctx.file._application_zip if modern else template + settings = application.settings + metadata = { + "cache": cache, + "name": paths.join(ctx.label.repo_name or "_main", ctx.label.package, ctx.label.name), + "spec": runfiles_root_path(ctx, application.spec.short_path), + "version": 1, + } + substitutions = { + "%EXTRACT_DIR%": metadata["name"], + "%bootstrap_parent%": repr(paths.dirname(paths.dirname(runfiles_root_path(ctx, ctx.file._application_driver.short_path)))), + "%python_binary%": settings["venv"]["executable"] if settings["venv"] else "", + "%python_binary_actual%": settings["interpreter"]["path"], + "%stage2_bootstrap%": settings["entry"], + "%workspace_name%": settings["workspace"], + } + args = ctx.actions.args() + args.add(template, format = "--template=%s") + args.add(main, format = "--output=%s") + args.add(metadata_file, format = "--metadata-output=%s") + args.add(json.encode(metadata), format = "--metadata=%s") + for key, value in substitutions.items(): + args.add(key + "=" + value, format = "--substitution=%s") + hash_manifest = ctx.actions.args() + hash_manifest.use_param_file("--hash_files_manifest=%s", use_always = True) + hash_manifest.set_param_file_format("multiline") + inputs = builders.DepsetBuilder() + inputs.add(template) + _manifest(ctx, hash_manifest, application, inputs) + actions_run( + ctx, + executable = ctx.attr._application_zip_main_maker, + arguments = [args, hash_manifest], + inputs = inputs.build(), + outputs = [main, metadata_file], + mnemonic = "PyApplicationArchiveMain", + progress_message = "Preparing application archive: %{label}", + ) + manifest = ctx.actions.args() + manifest.use_param_file("%s", use_always = True) + manifest.set_param_file_format("multiline") + manifest.add("regular|0|__main__.py|" + main.path) + manifest.add("regular|0|_rules_python_archive.json|" + metadata_file.path) + zip_inputs = builders.DepsetBuilder() + zip_inputs.add([main, metadata_file]) + _manifest(ctx, manifest, application, zip_inputs) + zip_args = ctx.actions.args() + zip_args.add(output) + zip_args.add(ctx.workspace_name, format = "--workspace-name=%s") + zip_args.add(str(int(py_internal.get_legacy_external_runfiles(ctx))), format = "--legacy-external-runfiles=%s") + zip_args.add("--runfiles-dir=runfiles") + zip_args.add("\\" if is_windows_platform(ctx) else "/", format = "--target-platform-pathsep=%s") + if compression: + zip_args.add(compression, format = "--compression=%s") + actions_run( + ctx, + executable = ctx.attr._application_zipper, + arguments = [manifest, zip_args], + inputs = zip_inputs.build(), + outputs = [output], + mnemonic = "PyApplicationArchive", + progress_message = "Packaging application image: %{label}", + ) diff --git a/python/private/application_python_template.txt b/python/private/application_python_template.txt new file mode 100644 index 0000000000..2cfe47070a --- /dev/null +++ b/python/private/application_python_template.txt @@ -0,0 +1,52 @@ +%shebang% +"""Generated directory entry; application behavior lives in the private package.""" + +import sys + +if not getattr(sys.flags, "safe_path", False) and not sys.flags.isolated and sys.path: + del sys.path[0] + +import importlib.machinery +import importlib.util +import os + +SPEC = %application_spec% +ENTRY = %entry% +BOOTSTRAP_PARENT = %bootstrap_parent% + + +def find_runfiles(): + directory = os.environ.get("RUNFILES_DIR", "") + manifest = os.environ.get("RUNFILES_MANIFEST_FILE", "") + if not directory and manifest.endswith((".runfiles_manifest", ".runfiles/MANIFEST")): + directory = manifest[:-9] + if directory and os.path.isfile(os.path.join(directory, SPEC)): + return os.path.abspath(directory) + os.environ.pop("RUNFILES_DIR", None) + os.environ.pop("RUNFILES_MANIFEST_FILE", None) + filename = os.path.abspath(sys.argv[0]) + while True: + for suffix in (".runfiles", ".exe.runfiles"): + directory = filename + suffix + if os.path.isfile(os.path.join(directory, SPEC)): + return directory + marker = ".runfiles" + os.sep + if marker in filename: + directory = filename.rsplit(marker, 1)[0] + ".runfiles" + if os.path.isfile(os.path.join(directory, SPEC)): + return directory + if not os.path.islink(filename): + raise RuntimeError("Cannot find application runfiles for " + sys.argv[0]) + filename = os.path.abspath(os.path.join(os.path.dirname(filename), os.readlink(filename))) + + +if __name__ == "__main__": + root = find_runfiles() + spec = importlib.machinery.PathFinder.find_spec( + "_rules_python_bootstrap", [os.path.join(root, BOOTSTRAP_PARENT)]) + package = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = package + spec.loader.exec_module(package) + from _rules_python_bootstrap.entry import directory_main + + sys.exit(directory_main(root, SPEC, sys.argv[1:])) diff --git a/python/private/application_shell_template.sh b/python/private/application_shell_template.sh new file mode 100644 index 0000000000..82f9762cef --- /dev/null +++ b/python/private/application_shell_template.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +set -e + +if [[ -n "${RULES_PYTHON_BOOTSTRAP_VERBOSE:-}" ]]; then set -x; fi + +IS_ZIPFILE=%archive% +CACHE=%cache% +CACHE_NAME=%cache_name% +APP_HASH="%APP_HASH%" +APPLICATION_SPEC=%spec_shell% +BOOTSTRAP_DRIVER=%driver_shell% +STAGE2_BOOTSTRAP=%entry_shell% +BOOTSTRAP_CLEANUP=%cleanup_shell% +WORKSPACE_NAME=%workspace_shell% +INTERPRETER_KIND=%interpreter_kind% +PYTHON_BINARY_ACTUAL=%interpreter_shell% +PYTHON_BINARY=%venv_executable_shell% +RECREATE_VENV_AT_RUNTIME=%recreate% +RESOLVE_PYTHON_BINARY_AT_RUNTIME=%resolve% +declare -a INTERPRETER_ARGS_FROM_TARGET=( +%interpreter_args_shell% +) +declare -a additional_interpreter_args=() +if [[ -n "${RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS:-}" ]]; then + read -a additional_interpreter_args <<< "$RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS" +fi +unset RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS +unset __PYVENV_LAUNCHER__ + +# These values describe only allocations and children owned by this invocation. +workspace="" +child="" +cancelled=0 +starting_child=0 +cleanup() { + local status=$? + trap '' TERM INT HUP QUIT + if [[ -n "$child" ]]; then + # Preparation handles TERM cooperatively and reaps any children it owns. + kill -TERM %+ 2>/dev/null || true + wait "$child" 2>/dev/null || true + fi + if [[ -n "$workspace" && -z "${RULES_PYTHON_BOOTSTRAP_VERBOSE:-}" ]]; then + rm -rf "$workspace" + fi + exit "$status" +} +cancel() { + cancelled=$1 + if [[ "$starting_child" == 0 ]]; then exit "$cancelled"; fi +} +own_workspace() { + trap 'cancel 143' TERM + trap 'cancel 130' INT + trap 'cancel 129' HUP + trap 'cancel 131' QUIT + trap cleanup EXIT + local parent=$1 + if [[ "$parent" != /* ]]; then parent="$PWD/$parent"; fi + workspace=$(trap '' TERM INT HUP QUIT; mktemp -d "$parent/rules_python.XXXXXXXXXX") +} +run_preparer() { + local status=0 + starting_child=1 + "$@" & + child=$! + starting_child=0 + if [[ "$cancelled" != 0 ]]; then exit "$cancelled"; fi + wait "$child" || status=$? + child="" + return "$status" +} +find_runfiles_root() { + local candidate="${RUNFILES_DIR:-}" filename=$1 target + if [[ -z "$candidate" ]]; then + case "${RUNFILES_MANIFEST_FILE:-}" in + *.runfiles_manifest) candidate="${RUNFILES_MANIFEST_FILE%_manifest}" ;; + *.runfiles/MANIFEST) candidate="${RUNFILES_MANIFEST_FILE%/MANIFEST}" ;; + esac + fi + if [[ -n "$candidate" && -f "$candidate/$APPLICATION_SPEC" ]]; then + RUNFILES_DIR=$candidate + return + fi + unset RUNFILES_MANIFEST_FILE + if [[ "$filename" != /* ]]; then filename="$PWD/$filename"; fi + while true; do + if [[ -f "$filename.runfiles/$APPLICATION_SPEC" ]]; then + RUNFILES_DIR="$filename.runfiles" + return + fi + if [[ "$filename" == *.runfiles/* ]]; then + candidate="${filename%.runfiles/*}.runfiles" + if [[ -f "$candidate/$APPLICATION_SPEC" ]]; then + RUNFILES_DIR=$candidate + return + fi + fi + if [[ ! -L "$filename" ]]; then break; fi + target=$(readlink "$filename") + if [[ "$target" == /* ]]; then filename=$target; else filename="${filename%/*}/$target"; fi + done + echo >&2 "ERROR: Cannot find application runfiles for $1" + exit 1 +} +find_interpreter() { + case "$INTERPRETER_KIND" in + runfiles) python_actual="$RUNFILES_DIR/$PYTHON_BINARY_ACTUAL" ;; + absolute) python_actual="$PYTHON_BINARY_ACTUAL" ;; + path) python_actual=$(command -v "$PYTHON_BINARY_ACTUAL") ;; + esac + if [[ "$python_actual" != /* ]]; then python_actual="$PWD/$python_actual"; fi +} + +temporary_zip="" +working_directory="" +if [[ "$IS_ZIPFILE" == 0 ]]; then + find_runfiles_root "$0" + if [[ "$RUNFILES_DIR" != /* ]]; then RUNFILES_DIR="$PWD/$RUNFILES_DIR"; fi + export RUNFILES_DIR + find_interpreter + if [[ "$RECREATE_VENV_AT_RUNTIME" == 1 ]]; then + # No resource exists yet. The Python entry owns all subsequent preparation. + exec "$python_actual" -I -S "$RUNFILES_DIR/$BOOTSTRAP_DRIVER" directory \ + "$RUNFILES_DIR" "$APPLICATION_SPEC" "$STAGE2_BOOTSTRAP" "$BOOTSTRAP_CLEANUP" \ + "${#additional_interpreter_args[@]}" "${additional_interpreter_args[@]}" "$@" + fi + python_exe=$python_actual + if [[ -n "$PYTHON_BINARY" ]]; then python_exe="$RUNFILES_DIR/$PYTHON_BINARY"; fi + entry="$RUNFILES_DIR/$STAGE2_BOOTSTRAP" +else + if [[ "$INTERPRETER_KIND" != runfiles ]]; then + find_interpreter + # An external interpreter is available before any allocation. + exec "$python_actual" -I -S -c \ + 'import runpy,sys; runpy.run_path(sys.argv[1])["shell_main"](sys.argv[1],sys.argv[2:])' \ + "$0" "${#additional_interpreter_args[@]}" "${additional_interpreter_args[@]}" "$@" + fi + destination="" + cached=0 + parent="${TMPDIR:-/tmp}" + if [[ "$CACHE" == 1 && -n "${RULES_PYTHON_EXTRACT_ROOT:-}" ]]; then + destination="$RULES_PYTHON_EXTRACT_ROOT/$CACHE_NAME/$APP_HASH" + if [[ "$destination" != /* ]]; then destination="$PWD/$destination"; fi + parent="${destination%/*}" + mkdir -p "$parent" + if [[ "$RESOLVE_PYTHON_BINARY_AT_RUNTIME" == 0 && ( -e "$destination" || -L "$destination" ) ]]; then + if [[ ! -f "$destination/.rules_python_complete" ]]; then + echo >&2 "ERROR: Incomplete Python runtime at $destination; use a clean extract root" + exit 1 + fi + cached=1 + fi + fi + if [[ "$cached" == 1 ]]; then + # A completed bundled image already contains its final environment. No + # temporary resource or preparer is needed to enter it. + RUNFILES_DIR="$destination/runfiles" + find_interpreter + python_exe=$python_actual + if [[ -n "$PYTHON_BINARY" ]]; then python_exe="$RUNFILES_DIR/$PYTHON_BINARY"; fi + entry="$RUNFILES_DIR/$STAGE2_BOOTSTRAP" + if [[ ! -f "$python_exe" || ! -x "$python_exe" || ! -f "$entry" || ! -r "$entry" ]]; then + echo >&2 "ERROR: Invalid prepared Python application at $destination" + exit 1 + fi + else + own_workspace "$parent" + image="$workspace/image" + status=0 + run_preparer unzip -q -d "$image" "$0" 2>/dev/null || status=$? + if [[ "$status" -gt 1 ]]; then + echo >&2 "ERROR: Unable to extract Python application (unzip status $status)" + exit "$status" + fi + RUNFILES_DIR="$image/runfiles" + export RUNFILES_DIR + unset RUNFILES_MANIFEST_FILE + find_interpreter + run_preparer "$python_actual" -I -S "$RUNFILES_DIR/$BOOTSTRAP_DRIVER" \ + prepare-archive "$workspace" "$image" "$cached" + declare -a result=() + field="" + while IFS= read -r -d '' field; do result+=("$field"); done < "$workspace/invocation" + if [[ "${#result[@]}" != 6 || "${result[0]}" != 1 || -n "$field" ]]; then + echo >&2 "ERROR: Invalid Python preparation result" + exit 1 + fi + working_directory=${result[1]} + RUNFILES_DIR=${result[2]} + python_exe=${result[3]} + entry=${result[4]} + temporary_zip=${result[5]} + if [[ -n "$temporary_zip" && "$temporary_zip" != "$workspace" ]]; then + echo >&2 "ERROR: Python preparation changed workspace ownership" + exit 1 + fi + if [[ -z "$temporary_zip" ]]; then + rm -rf "$workspace" + workspace="" + fi + fi + unset RUNFILES_MANIFEST_FILE +fi + +export RUNFILES_DIR +if [[ -z "${PYTHONSAFEPATH+x}" ]]; then export PYTHONSAFEPATH=1; fi +if [[ -n "${RULES_PYTHON_TESTING_TELL_RUNFILES_ROOT:-}" ]]; then + export RULES_PYTHON_TESTING_RUNFILES_ROOT="$RUNFILES_DIR" +fi +declare -a options=() +if [[ -n "$temporary_zip" ]]; then options+=("-XRULES_PYTHON_ZIP_DIR=$temporary_zip/image"); fi +options+=("${additional_interpreter_args[@]}" "${INTERPRETER_ARGS_FROM_TARGET[@]}") +if [[ -n "$workspace" && -z "${RULES_PYTHON_BOOTSTRAP_VERBOSE:-}" ]]; then + run_preparer "$python_exe" -I -S "$RUNFILES_DIR/$BOOTSTRAP_CLEANUP" "$$" "$workspace" +fi +if [[ "${RUN_UNDER_RUNFILES:-}" == 1 ]]; then working_directory="$RUNFILES_DIR/$WORKSPACE_NAME"; fi +if [[ -n "$working_directory" ]]; then cd "$working_directory"; fi +exec "$python_exe" "${options[@]}" "$entry" "$@" +# A self-executable archive can follow this prelude. +exit 1 diff --git a/python/private/application_zip_template.txt b/python/private/application_zip_template.txt new file mode 100644 index 0000000000..7f5662c337 --- /dev/null +++ b/python/private/application_zip_template.txt @@ -0,0 +1,35 @@ +"""Generated archive entry; only the private bootstrap is imported from ZIP.""" + +import sys + +if not getattr(sys.flags, "safe_path", False) and not sys.flags.isolated and sys.path: + del sys.path[0] + +import importlib.machinery +import importlib.util +import json +import os + +METADATA = json.loads(%archive_metadata%) +BOOTSTRAP_PARENT = %bootstrap_parent% + + +def load_bootstrap(archive): + spec = importlib.machinery.PathFinder.find_spec( + "_rules_python_bootstrap", [archive + "/runfiles/" + BOOTSTRAP_PARENT]) + package = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = package + # Python 3.9 zipimporter implements get_code, but not exec_module. + exec(spec.loader.get_code(spec.name), package.__dict__) + from _rules_python_bootstrap import entry + + return entry + + +def shell_main(archive, arguments): + sys.exit(load_bootstrap(archive).shell_archive_main(archive, METADATA, arguments)) + + +if __name__ == "__main__": + archive = os.path.dirname(__file__) + sys.exit(load_bootstrap(archive).archive_main(archive, METADATA, sys.argv[1:])) diff --git a/python/private/bootstrap_cleanup.py b/python/private/bootstrap_cleanup.py new file mode 100644 index 0000000000..884c3be810 --- /dev/null +++ b/python/private/bootstrap_cleanup.py @@ -0,0 +1,192 @@ +"""Remove a temporary runtime after the exec'ed launcher's process exits. + +This runs with -I -S, outside the application's process group. The direct child +exits successfully only after an orphaned watcher is armed, so stage one can +reap it before exec. Deletion is asynchronous. + +Namespace PID 1 and subreapers can adopt the watcher; namespace shutdown can +kill it before removal finishes. Persistent runtime-venv extract roots avoid +this helper; legacy executable ZIPs always extract to temporary directories. +""" + +import contextlib +import errno +import os +import resource +import select +import shutil +import socket +import sys +import time + + +def _close_inherited_fds(): + # Existing descriptors can be above a limit lowered by the caller. Prefer + # enumerating them to scanning every number up to the resource limit. + for directory in ("/proc/self/fd", "/dev/fd"): + try: + descriptors = os.listdir(directory) + except OSError: + continue + for name in descriptors: + if name.isdigit() and int(name) > 2: + with contextlib.suppress(OSError): + os.close(int(name)) + return + _, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + # Without a descriptor directory, POSIX cannot recover limits that the + # caller lowered below already-open descriptors. Use the current hard limit. + if hard_limit == resource.RLIM_INFINITY: + hard_limit = os.sysconf("SC_OPEN_MAX") + os.closerange(3, hard_limit) + + +def _poll_pid(pid): + # Last resort on systems without a native watch or Linux procfs. PID reuse + # can delay removal, but must never remove the runtime while that PID lives. + while True: + try: + os.kill(pid, 0) + except ProcessLookupError: + return + except PermissionError: + pass + time.sleep(0.1) + + +def _arm_poll_watch(pid, resources): + try: + # An ancestor namespace's procfs can map this PID to another process. + if os.readlink("/proc/self") != str(os.getpid()) or os.readlink( + "/proc/self/ns/pid" + ) != os.readlink("/proc/1/ns/pid"): + return lambda: _poll_pid(pid) + fd = os.open("/proc/{}/stat".format(pid), os.O_RDONLY) + except OSError: + return lambda: _poll_pid(pid) + resources.callback(os.close, fd) + + def wait(): + # An open Linux proc descriptor stays tied to the original process, + # even if its PID is reused. Wait for reaping: a zombie group leader + # can still have live worker threads that need the runtime. + while True: + try: + os.lseek(fd, 0, os.SEEK_SET) + data = os.read(fd, 4096) + except ProcessLookupError: + return + if not data: + return _poll_pid(pid) + time.sleep(0.1) + + return wait + + +def _arm_watch(pid, resources): + try: + if hasattr(os, "pidfd_open"): + fd = os.pidfd_open(pid) + resources.callback(os.close, fd) + + def wait(): + select.select([fd], [], []) + + elif hasattr(select, "kqueue"): + queue = select.kqueue() + resources.callback(queue.close) + event = select.kevent( + pid, + filter=select.KQ_FILTER_PROC, + flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT, + fflags=select.KQ_NOTE_EXIT, + ) + queue.control([event], 0, 0) + + def wait(): + queue.control(None, 1) + + else: + return _arm_poll_watch(pid, resources) + except OSError as error: + if error.errno == errno.ESRCH: + return lambda: None + if error.errno not in (errno.ENOSYS, errno.ENOTSUP, errno.EINVAL, errno.EPERM): + raise + return _arm_poll_watch(pid, resources) + + return wait + + +def _watch(pid, runtimes, control): + # Do not hold the caller's output open, including during handshake failure. + os.dup2(0, 1) + os.dup2(0, 2) + with control, contextlib.ExitStack() as resources: + wait = _arm_watch(pid, resources) + permission = b"" + try: + control.sendall(b"A") # The process watch is armed. + permission = control.recv(1) + except OSError: + pass + # Only the direct child can confirm that its actual parent exited. + # EOF, invalid data, or a dead helper cannot authorize early removal. + if permission != b"E": + try: + wait() + except OSError: + _poll_pid(pid) + + # All cleanup dependencies are imported before any runtime can be removed. + for runtime in reversed(runtimes): + shutil.rmtree(runtime, ignore_errors=True) + + +def main(): + parent_pid = int(sys.argv[1]) + runtimes = [os.path.abspath(path) for path in sys.argv[2:]] + + # Inherited pipes and locks must not outlive the application. Keep stderr + # for setup diagnostics until the watcher forks. + _close_inherited_fds() + os.setsid() + null_fd = os.open(os.devnull, os.O_RDWR) + os.dup2(null_fd, 0) + os.dup2(0, 1) + # Reserve standard descriptors before creating the watch: the caller may + # have closed stdin or stderr, and a watch must never occupy those slots. + try: + os.fstat(2) + except OSError as error: + if error.errno != errno.EBADF: + raise + os.dup2(0, 2) + if null_fd > 2: + os.close(null_fd) + os.chdir("/") + + if os.getppid() != parent_pid: + for runtime in reversed(runtimes): + shutil.rmtree(runtime, ignore_errors=True) + return 1 + + parent, watcher = socket.socketpair() + if os.fork() == 0: + parent.close() + _watch(parent_pid, runtimes, watcher) + return 0 + + watcher.close() + with parent: + if parent.recv(1) != b"A": + return 1 + # Check real parent identity after watch registration. If the original + # launcher exited during setup, the watch may refer to a reused PID. + alive = os.getppid() == parent_pid + parent.sendall(b"W" if alive else b"E") + return 0 if alive else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/private/py_application_info.bzl b/python/private/py_application_info.bzl new file mode 100644 index 0000000000..c7e9440976 --- /dev/null +++ b/python/private/py_application_info.bzl @@ -0,0 +1,11 @@ +"""Private application image shared by directory and archive executables.""" + +PyApplicationInfo = provider( + doc = "Private logical application files and preparation specification.", + fields = { + "runfiles": "Complete logical application image, without its launcher.", + "settings": "The specification's build-time values, for entry adapters.", + "spec": "File containing the private versioned launch specification.", + "symlinks": "depset[ExplicitSymlink] completing the logical image.", + }, +) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 62742d334c..387ad4f285 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -15,10 +15,12 @@ load("@bazel_skylib//lib:dicts.bzl", "dicts") load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//lib:shell.bzl", "shell") load("@bazel_skylib//lib:structs.bzl", "structs") load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("@rules_cc//cc/common:cc_common.bzl", "cc_common") load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") +load(":application.bzl", "APPLICATION_ATTRS", "create_application", "create_application_archive", "create_application_launcher", "is_default_bootstrap") load(":attr_builders.bzl", "attrb") load( ":attributes.bzl", @@ -82,6 +84,7 @@ _INIT_PY = "__init__.py" EXECUTABLE_ATTRS = dicts.add( COMMON_ATTRS, AGNOSTIC_EXECUTABLE_ATTRS, + APPLICATION_ATTRS, PY_SRCS_ATTRS, IMPORTS_ATTRS, WINDOWS_CONSTRAINTS_ATTRS, @@ -208,6 +211,10 @@ accepting arbitrary Python versions. "_allowlist_function_transition": lambda: attrb.Label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), + "_bootstrap_cleanup": lambda: attrb.Label( + allow_single_file = True, + default = "//python/private:bootstrap_cleanup", + ), "_bootstrap_impl_flag": lambda: attrb.Label( default = labels.BOOTSTRAP_IMPL, providers = [BuildSettingInfo], @@ -333,6 +340,7 @@ def _create_executable( # avoid collisions between targets like foo/tool, bar/tool, and foo_tool. venv_output_prefix = ctx.label.name venv = None + zip_main = None # The check for stage2_bootstrap_template is to support legacy # BuiltinPyRuntimeInfo providers, which is likely to come from @@ -364,19 +372,23 @@ def _create_executable( build_data_file = runfiles_details.build_data_file, ) extra_runfiles = ctx.runfiles( - [stage2_bootstrap] + ( + [stage2_bootstrap, ctx.file._bootstrap_cleanup] + ( venv.files_without_interpreter if venv else [] ), ).merge(venv.lib_runfiles) - zip_main = _create_zip_main( + application = create_application( ctx, - stage2_bootstrap = stage2_bootstrap, - runtime_details = runtime_details, + runtime = runtime_details.effective_runtime, + stage2 = stage2_bootstrap, venv = venv, + runfiles = runfiles_details.runfiles_without_exe, + interpreter_args = ctx.attr.interpreter_args, ) + extra_runfiles = extra_runfiles.merge(application.runfiles) else: stage2_bootstrap = None - extra_runfiles = ctx.runfiles() + application = None + extra_runfiles = ctx.runfiles([ctx.file._bootstrap_cleanup]) zip_main = ctx.actions.declare_file(base_executable_name + ".temp", sibling = executable) _create_stage1_bootstrap( ctx, @@ -388,12 +400,21 @@ def _create_executable( ) zip_file = ctx.actions.declare_file(base_executable_name + ".zip", sibling = executable) - _create_zip_file( - ctx, - output = zip_file, - zip_main = zip_main, - runfiles = runfiles_details.runfiles_without_exe.merge(extra_runfiles), - ) + if application: + create_application_archive( + ctx, + application = application, + output = zip_file, + template = runtime_details.effective_runtime.zip_main_template, + cache = False, + ) + else: + _create_zip_file( + ctx, + output = zip_file, + zip_main = zip_main, + runfiles = runfiles_details.runfiles_without_exe.merge(extra_runfiles), + ) extra_default_outputs = [] @@ -464,6 +485,7 @@ WARNING: Target: {} ctx, output = executable, zip_file = zip_file, + application = application, stage2_bootstrap = stage2_bootstrap, runtime_details = runtime_details, venv = venv, @@ -472,6 +494,7 @@ WARNING: Target: {} _create_stage1_bootstrap( ctx, output = bootstrap_output, + application = application, stage2_bootstrap = stage2_bootstrap, runtime_details = runtime_details, is_for_zip = False, @@ -506,6 +529,7 @@ WARNING: Target: {} # depset[File] of additional files that should be included as default # outputs. extra_default_outputs = depset(extra_default_outputs), + application = application, # dict[str, depset[File]]; additional output groups that should be # returned. output_groups = {"python_zip_file": depset([zip_file])}, @@ -527,29 +551,6 @@ WARNING: Target: {} venv_interpreter_symlinks = venv.interpreter_symlinks if venv else None, ) -def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv): - if venv.interpreter: - python_binary = runfiles_root_path(ctx, venv.interpreter.short_path) - else: - python_binary = "" - python_binary_actual = venv.interpreter_actual_path - - # The location of this file doesn't really matter. It's added to - # the zip file as the top-level __main__.py file and not included - # elsewhere. - output = ctx.actions.declare_file(ctx.label.name + "_zip__main__.py") - ctx.actions.expand_template( - template = runtime_details.effective_runtime.zip_main_template, - output = output, - substitutions = { - "%python_binary%": python_binary, - "%python_binary_actual%": python_binary_actual, - "%stage2_bootstrap%": runfiles_root_path(ctx, stage2_bootstrap.short_path), - "%workspace_name%": ctx.workspace_name, - }, - ) - return output - # Create a venv the executable can use. # For venv details and the venv startup process, see: # * https://docs.python.org/3/library/venv.html @@ -923,7 +924,8 @@ def _create_stage1_bootstrap( imports = None, is_for_zip, runtime_details, - venv = None): + venv = None, + application = None): """Create a legacy bootstrap script that is written in Python.""" runtime = runtime_details.effective_runtime @@ -947,7 +949,9 @@ def _create_stage1_bootstrap( resolve_python_binary_at_runtime = "1" subs = { + "%bootstrap_cleanup%": runfiles_root_path(ctx, ctx.file._bootstrap_cleanup.short_path), "%interpreter_args%": "\n".join(ctx.attr.interpreter_args), + "%interpreter_args_shell%": "\n".join([shell.quote(arg) for arg in ctx.attr.interpreter_args]), "%is_zipfile%": "1" if is_for_zip else "0", "%python_binary%": python_binary_path, "%python_binary_actual%": python_binary_actual, @@ -994,13 +998,23 @@ def _create_stage1_bootstrap( subs["%imports%"] = ":".join(imports.to_list()) subs["%main%"] = runfiles_root_path(ctx, main_py.short_path) - ctx.actions.expand_template( - template = template, - output = output, - substitutions = subs, - computed_substitutions = computed_subs, - is_executable = True, - ) + if application and is_default_bootstrap(ctx, template): + create_application_launcher( + ctx, + application = application, + output = output, + shell_entry = template == ctx.file._application_default_shell, + archive = is_for_zip, + shebang = runtime.stub_shebang, + ) + else: + ctx.actions.expand_template( + template = template, + output = output, + substitutions = subs, + computed_substitutions = computed_subs, + is_executable = True, + ) def _map_runtime_venv_symlink(entry): return entry.venv_path + "|" + entry.link_to_path @@ -1099,6 +1113,7 @@ def _create_executable_zip_file( *, output, zip_file, + application, stage2_bootstrap, runtime_details, venv): @@ -1110,6 +1125,7 @@ def _create_executable_zip_file( _create_stage1_bootstrap( ctx, output = prelude, + application = application, stage2_bootstrap = stage2_bootstrap, runtime_details = runtime_details, is_for_zip = True, @@ -1281,7 +1297,7 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = ) )) - providers = [] + providers = [exec_result.application] if exec_result.application else [] _add_provider_default_info( providers, diff --git a/python/private/zipapp/BUILD.bazel b/python/private/zipapp/BUILD.bazel index 395e1d242a..314877c071 100644 --- a/python/private/zipapp/BUILD.bazel +++ b/python/private/zipapp/BUILD.bazel @@ -33,16 +33,19 @@ bzl_library( name = "py_zipapp_rule", srcs = ["py_zipapp_rule.bzl"], deps = [ + "//python/private:application", "//python/private:attributes", "//python/private:builders", "//python/private:common", "//python/private:common_labels", + "//python/private:py_application_info", "//python/private:py_executable_info", "//python/private:py_internal", "//python/private:py_runtime_info", "//python/private:toolchain_types", "//python/private:transition_labels", "@bazel_skylib//lib:paths", + "@bazel_skylib//lib:shell", "@rules_python_internal//:rules_python_config", ], ) diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index b664c1f628..721c891237 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -1,7 +1,9 @@ """Implementation of the zipapp rules.""" load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//lib:shell.bzl", "shell") load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") +load("//python/private:application.bzl", "APPLICATION_ATTRS", "create_application_archive", "create_application_launcher") load("//python/private:attributes.bzl", "apply_config_settings_attr") load("//python/private:builders.bzl", "builders") load( @@ -15,6 +17,7 @@ load( "runfiles_root_path", ) load("//python/private:common_labels.bzl", "labels") +load("//python/private:py_application_info.bzl", "PyApplicationInfo") load("//python/private:py_executable_info.bzl", "PyExecutableInfo") load("//python/private:py_internal.bzl", "py_internal") load("//python/private:py_runtime_info.bzl", "PyRuntimeInfo") @@ -40,9 +43,10 @@ def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap, run python_binary_actual_path = py_runtime.interpreter_path zip_main_py = ctx.actions.declare_file(ctx.label.name + ".zip_main.py") + template = py_runtime.zip_main_template args = ctx.actions.args() - args.add(py_runtime.zip_main_template, format = "--template=%s") + args.add(template, format = "--template=%s") args.add(zip_main_py, format = "--output=%s") args.add( @@ -57,13 +61,15 @@ def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap, run args.add("%python_binary_actual%=" + python_binary_actual_path, format = "--substitution=%s") args.add("%stage2_bootstrap%=" + runfiles_root_path(ctx, stage2_bootstrap.short_path), format = "--substitution=%s") args.add("%workspace_name%=" + ctx.workspace_name, format = "--substitution=%s") + args.add("%bootstrap_cleanup%=" + runfiles_root_path(ctx, ctx.file._bootstrap_cleanup.short_path), format = "--substitution=%s") + args.add("# %interpreter_args_python%=INTERPRETER_ARGS = " + repr(py_executable.interpreter_args), format = "--substitution=%s") hash_files_manifest = ctx.actions.args() hash_files_manifest.use_param_file("--hash_files_manifest=%s", use_always = True) hash_files_manifest.set_param_file_format("multiline") inputs = builders.DepsetBuilder() - inputs.add(py_runtime.zip_main_template) + inputs.add(template) _build_manifest(ctx, hash_files_manifest, runfiles, explicit_symlinks, inputs) actions_run( @@ -135,16 +141,23 @@ def _create_zip(ctx, py_runtime, py_executable, stage2_bootstrap): if py_runtime.files != None: runfiles.add(py_runtime.files) + if py_runtime.interpreter: + runfiles.add(py_runtime.interpreter) if py_executable.venv_python_exe: runfiles.add(py_executable.venv_python_exe) - if py_executable.venv_interpreter_runfiles: runfiles.add(py_executable.venv_interpreter_runfiles) runfiles.add(py_executable.app_runfiles) runfiles.add(stage2_bootstrap) + # Packaging rebuilds runfiles from the public provider. Keep private support + # explicit here instead of changing the meaning of app_runfiles. + runfiles.add(ctx.file._bootstrap_cleanup) + runfiles = runfiles.build(ctx) + # A custom rule may supply its own executable. Only replace archive files + # with links that the provider explicitly declares. explicit_symlinks = depset(transitive = [ py_executable.venv_interpreter_symlinks, py_executable.venv_app_symlinks, @@ -206,10 +219,12 @@ def _create_shell_bootstrap(ctx, py_runtime, py_executable, stage2_bootstrap): ctx.label.name, ), "%INTERPRETER_ARGS%": "\n".join([ - '"{}"'.format(v) + shell.quote(v) for v in py_executable.interpreter_args ]), "%STAGE2_BOOTSTRAP%": runfiles_root_path(ctx, stage2_bootstrap.short_path), + "%bootstrap_cleanup%": runfiles_root_path(ctx, ctx.file._bootstrap_cleanup.short_path), + "%workspace_name%": ctx.workspace_name, } ctx.actions.expand_template( template = ctx.file._zip_shell_template, @@ -242,7 +257,21 @@ def _py_zipapp_executable_impl(ctx): stage2_bootstrap = py_executable.stage2_bootstrap - zip_file = _create_zip(ctx, py_runtime, py_executable, stage2_bootstrap) + application = ctx.attr.binary[PyApplicationInfo] if PyApplicationInfo in ctx.attr.binary else None + if application: + zip_file = ctx.actions.declare_file(ctx.label.name + ".zip") + create_application_archive( + ctx, + application = application, + output = zip_file, + template = py_runtime.zip_main_template, + cache = True, + compression = ctx.attr.compression, + ) + else: + # Older custom rules expose no runtime preparation recipe. Keep their + # public-provider compatibility path instead of inventing missing facts. + zip_file = _create_zip(ctx, py_runtime, py_executable, stage2_bootstrap) if ctx.attr.executable: if is_windows_platform(ctx): executable = ctx.actions.declare_file(ctx.label.name + ".exe") @@ -268,7 +297,18 @@ def _py_zipapp_executable_impl(ctx): ) default_outputs = [executable, zip_file] else: - preamble = _create_shell_bootstrap(ctx, py_runtime, py_executable, stage2_bootstrap) + if application and py_runtime.zip_main_template == ctx.file._application_default_zip: + preamble = ctx.actions.declare_file(ctx.label.name + ".preamble.sh") + create_application_launcher( + ctx, + application = application, + output = preamble, + shell_entry = True, + archive = True, + cache = True, + ) + else: + preamble = _create_shell_bootstrap(ctx, py_runtime, py_executable, stage2_bootstrap) executable = _create_self_executable_zip(ctx, preamble, zip_file) default_outputs = [executable] else: @@ -305,7 +345,7 @@ _zipapp_transition = transition( ] + BUILTIN_BUILD_PYTHON_ZIP, ) -_ATTRS = { +_ATTRS = APPLICATION_ATTRS | { "binary": attr.label( doc = """ A `py_binary` or `py_test` (or equivalent) target to package. @@ -360,6 +400,10 @@ Whether the output should be an executable zip file. "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), + "_bootstrap_cleanup": attr.label( + allow_single_file = True, + default = "//python/private:bootstrap_cleanup", + ), "_exe_zip_maker": attr.label( cfg = "exec", default = "//tools/zipapp:exe_zip_maker", diff --git a/tests/bootstrap_impls/BUILD.bazel b/tests/bootstrap_impls/BUILD.bazel index 2c4eae8d21..56c74f58f4 100644 --- a/tests/bootstrap_impls/BUILD.bazel +++ b/tests/bootstrap_impls/BUILD.bazel @@ -12,12 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. load("@rules_pkg//pkg:tar.bzl", "pkg_tar") +load("@rules_python_internal//:rules_python_config.bzl", "config") load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//python:py_runtime.bzl", "py_runtime") +load("//python:py_runtime_pair.bzl", "py_runtime_pair") load("//python:py_test.bzl", "py_test") +load("//python/zipapp:py_zipapp_binary.bzl", "py_zipapp_binary") load("//tests/support:py_reconfig.bzl", "py_reconfig_binary", "py_reconfig_test") load("//tests/support:sh_py_run_test.bzl", "sh_py_run_test") load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT") load("//tests/support/pytest_test:pytest_test.bzl", "pytest_test") +load(":application_tests.bzl", "application_test_suite", "public_executable", "template_fixture", "template_output") load(":venv_relative_path_tests.bzl", "relative_path_test_suite") py_reconfig_binary( @@ -239,3 +244,280 @@ py_test( ) relative_path_test_suite(name = "relative_path_tests") + +application_test_suite(name = "application_build_tests") + +template_fixture( + name = "custom_directory_template", + contents = "custom\r\nworkspace=%workspace_name%\r\nmain=%stage2_bootstrap%", +) + +template_fixture( + name = "custom_zip_template", + contents = "custom zip\r\nworkspace=%workspace_name%\r\nmain=%stage2_bootstrap%", +) + +template_fixture( + name = "custom_application_interpreter", + contents = "This interpreter is packaged for template inspection, never executed.", +) + +py_runtime( + name = "custom_application_runtime", + # Also supplies the non-test current_interpreter_executable toolchain alias. + bootstrap_template = ":custom_directory_template", + # Windows venvs declare an interpreter symlink even for custom templates. + # Use a declared file so remote packaging does not need a worker-local path. + interpreter = select({ + "@platforms//os:windows": ":custom_application_interpreter", + "//conditions:default": None, + }), + interpreter_path = select({ + "@platforms//os:windows": "", + "//conditions:default": "/custom/python", + }), + supports_build_time_venv = False, + zip_main_template = ":custom_zip_template", +) + +py_runtime_pair( + name = "custom_application_pair", + py3_runtime = ":custom_application_runtime", +) + +toolchain( + name = "custom_application_toolchain", + toolchain = ":custom_application_pair", + toolchain_type = "//python:toolchain_type", +) + +py_reconfig_binary( + name = "custom_application", + testonly = True, + srcs = ["bin.py"], + bootstrap_impl = "system_python", + build_python_zip = False, + extra_toolchains = [ + "//tests/bootstrap_impls:custom_application_toolchain", + "//tests/support/cc_toolchains:all", + ], + main = "bin.py", +) + +py_zipapp_binary( + name = "custom_application_zip", + testonly = True, + binary = ":custom_application", + executable = False, +) + +template_output( + name = "custom_application_bootstrap", + testonly = True, + binary = ":custom_application", +) + +filegroup( + name = "custom_application_archive", + testonly = True, + srcs = [":custom_application_zip"], + output_group = "python_zip_file", +) + +template_fixture( + name = "supplied_venv", + testonly = True, + contents = "custom executable supplied without interpreter runfiles", +) + +public_executable( + name = "public_external_application", + testonly = True, + binary = ":custom_application", + supplied_venv = ":supplied_venv", +) + +py_zipapp_binary( + name = "public_external_zip", + testonly = True, + binary = ":public_external_application", + executable = False, +) + +filegroup( + name = "public_external_archive", + testonly = True, + srcs = [":public_external_zip"], + output_group = "python_zip_file", +) + +pytest_test( + name = "application_templates_test", + srcs = ["application_templates_test.py"], + data = [ + ":custom_application_archive", + ":custom_application_bootstrap", + ":public_external_archive", + "//python/private:application_zip_template.txt", + "//python/private:stage1_bootstrap_template", + ], + env = { + "CUSTOM_BOOTSTRAP": "$(rlocationpath :custom_application_bootstrap)", + "CUSTOM_ZIP": "$(rlocationpath :custom_application_archive)", + "PUBLIC_ZIP": "$(rlocationpath :public_external_archive)", + "RAW_SHELL_TEMPLATE": "$(rlocationpath //python/private:stage1_bootstrap_template)", + "ZIP_TEMPLATE": "$(rlocationpath //python/private:application_zip_template.txt)", + }, + deps = ["//python/runfiles"], +) + +py_runtime( + name = "cleanup_wrapper_runtime", + # Also supplies the non-test current_interpreter_executable toolchain alias. + files = ["cleanup_wrapper_data.txt"], + interpreter = "cleanup_python_wrapper.sh", + supports_build_time_venv = False, +) + +py_runtime_pair( + name = "cleanup_wrapper_pair", + py3_runtime = ":cleanup_wrapper_runtime", +) + +toolchain( + name = "cleanup_wrapper_toolchain", + toolchain = ":cleanup_wrapper_pair", + toolchain_type = "//python:toolchain_type", +) + +py_reconfig_binary( + name = "cleanup_wrapper", + testonly = True, + srcs = ["cleanup_probe.py"], + bootstrap_impl = "script", + build_python_zip = False, + extra_toolchains = [ + "//tests/bootstrap_impls:cleanup_wrapper_toolchain", + "//tests/support/cc_toolchains:all", + ], + main = "cleanup_probe.py", + target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, +) + +pytest_test( + name = "application_test", + srcs = ["application_test.py"], + data = [ + "windows_console_fixture.py", + "//python/private:python_bootstrap_template.txt", + ], + env = { + "PY_TEMPLATE_RLOCATION": "$(rlocationpath //python/private:python_bootstrap_template.txt)", + "WINDOWS_CONSOLE_FIXTURE": "$(rlocationpath windows_console_fixture.py)", + }, + deps = [ + "//python/private:application_bootstrap", + "//python/runfiles", + ], +) + +[ + py_reconfig_binary( + name = "cleanup_" + mode, + testonly = True, + srcs = ["cleanup_probe.py"], + bootstrap_impl = "system_python" if mode.startswith("py") else "script", + build_python_zip = mode.endswith("zip"), + interpreter_args = [ + "-Xbootstrap_target=argument with spaces and 'quotes' and \"double quotes\"", + "-Xbootstrap_precedence=target", + ], + main = "cleanup_probe.py", + target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, + venvs_use_declare_symlink = "no", + ) + for mode in [ + "venv", + "zip", + "pyvenv", + "pyzip", + ] +] + +filegroup( + name = "cleanup_legacy_python", + testonly = True, + srcs = [":cleanup_zip"], + output_group = "python_zip_file", +) + +[ + py_zipapp_binary( + name = "cleanup_" + name, + testonly = True, + binary = ":cleanup_" + binary, + compression = compression, + target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, + ) + for name, binary, compression in [ + ("zipapp", "venv", "0"), + ("zipapp_compressed", "venv", "4"), + ("zipapp_system", "pyvenv", "0"), + ("wrapper_zip", "wrapper", "0"), + ] +] + +pytest_test( + name = "bootstrap_cleanup_test", + timeout = "long", + srcs = ["bootstrap_cleanup_test.py"], + data = [ + "cleanup_fail.py", + "cleanup_probe.py", + "cleanup_sitecustomize.py", + "cleanup_terminal_peer.py", + "cleanup_wrapper_data.txt", + ":cleanup_legacy_python", + ":cleanup_pyvenv", + ":cleanup_pyzip", + ":cleanup_venv", + ":cleanup_wrapper", + ":cleanup_wrapper_zip", + ":cleanup_zip", + ":cleanup_zipapp", + ":cleanup_zipapp_compressed", + ":cleanup_zipapp_system", + ], + env = { + "FAIL_RLOCATION": "$(rlocationpath cleanup_fail.py)", + "LEGACY_PYTHON_RLOCATION": "$(rlocationpath :cleanup_legacy_python)", + "PEER_RLOCATION": "$(rlocationpath cleanup_terminal_peer.py)", + "PROBE_RLOCATION": "$(rlocationpath cleanup_probe.py)", + "PYVENV_RLOCATION": "$(rlocationpath :cleanup_pyvenv)", + # Bazel 7's system_python bootstrap uses its interpreter directly. + "PYVENV_TEMPORARY": "1" if config.bazel_8_or_later else "0", + "PYZIP_RLOCATION": "$(rlocationpath :cleanup_pyzip)", + "STARTUP_RLOCATION": "$(rlocationpath cleanup_sitecustomize.py)", + "VENV_RLOCATION": "$(rlocationpath :cleanup_venv)", + "WRAPPER_DATA_RLOCATION": "$(rlocationpath cleanup_wrapper_data.txt)", + "WRAPPER_RLOCATION": "$(rlocationpath :cleanup_wrapper)", + "WRAPPER_ZIP_PYTHON_RLOCATION": "$(rlocationpath :cleanup_wrapper_zip)", + "WRAPPER_ZIP_RLOCATION": "$(rlocationpath :cleanup_wrapper_zip)", + "ZIPAPP_COMPRESSED_RLOCATION": "$(rlocationpath :cleanup_zipapp_compressed)", + "ZIPAPP_PYTHON_RLOCATION": "$(rlocationpath :cleanup_zipapp)", + "ZIPAPP_RLOCATION": "$(rlocationpath :cleanup_zipapp)", + "ZIPAPP_SYSTEM_RLOCATION": "$(rlocationpath :cleanup_zipapp_system)", + "ZIP_RLOCATION": "$(rlocationpath :cleanup_zip)", + }, + target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, + deps = ["//python/runfiles"], +) + +pytest_test( + name = "bootstrap_cleanup_watch_test", + srcs = ["bootstrap_cleanup_watch_test.py"], + data = ["//python/private:bootstrap_cleanup"], + env = {"HELPER_RLOCATION": "$(rlocationpath //python/private:bootstrap_cleanup)"}, + target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, + deps = ["//python/runfiles"], +) diff --git a/tests/bootstrap_impls/application_templates_test.py b/tests/bootstrap_impls/application_templates_test.py new file mode 100644 index 0000000000..6754654783 --- /dev/null +++ b/tests/bootstrap_impls/application_templates_test.py @@ -0,0 +1,178 @@ +"""Entry templates preserve their interpreter and expansion contracts.""" + +import json +import os +import subprocess +import sys +import textwrap +import zipfile +from pathlib import Path + +import pytest + +from python.runfiles import runfiles + + +def test_zip_template_accepts_python39_loader_protocol(tmp_path): + template = runfiles.CreateOrRaise().Rlocation(os.environ["ZIP_TEMPLATE"]) + assert template is not None + contents = ( + Path(template) + .read_text() + .replace("%archive_metadata%", repr('{"version": 1}')) + .replace("%bootstrap_parent%", repr("private")) + ) + archive_path = tmp_path / "application.zip" + package = "runfiles/private/_rules_python_bootstrap/" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("__main__.py", contents) + archive.writestr(package + "__init__.py", "from . import entry\n") + archive.writestr( + package + "entry.py", + "import json\n" + "def archive_main(archive, metadata, arguments):\n" + " print(json.dumps([__package__, archive, metadata, arguments]))\n" + " return 23\n", + ) + # Retain the real archive spec and code, but expose only the loader protocol + # available before zipimporter gained exec_module in Python 3.10. + driver = textwrap.dedent(""" + import importlib.machinery + import runpy + import sys + import types + + find_spec = importlib.machinery.PathFinder.find_spec + def find_legacy_spec(name, path=None, target=None): + spec = find_spec(name, path, target) + if name == "_rules_python_bootstrap": + spec.loader = types.SimpleNamespace(get_code=spec.loader.get_code) + return spec + importlib.machinery.PathFinder.find_spec = find_legacy_spec + sys.argv = sys.argv[1:] + runpy.run_path(sys.argv[0], run_name="__main__") + """) + result = subprocess.run( + [sys.executable, "-c", driver, str(archive_path), "a b", ""], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 23, result.stderr + assert json.loads(result.stdout) == [ + "_rules_python_bootstrap", + str(archive_path), + {"version": 1}, + ["a b", ""], + ] + + +def test_custom_directory_template_preserves_bytes(): + path = runfiles.CreateOrRaise().Rlocation(os.environ["CUSTOM_BOOTSTRAP"]) + assert path is not None + contents = Path(path).read_bytes() + workspace = os.environ["TEST_WORKSPACE"] + assert contents.startswith( + f"custom\r\nworkspace={workspace}\r\nmain={workspace}/".encode() + ) + assert contents.endswith(b"_custom_application_stage2_bootstrap.py") + assert b"_rules_python_bootstrap" not in contents + + +def test_public_provider_keeps_independently_supplied_executable(): + path = runfiles.CreateOrRaise().Rlocation(os.environ["PUBLIC_ZIP"]) + assert path is not None + with zipfile.ZipFile(path) as archive: + contents = archive.read( + f"runfiles/{os.environ['TEST_WORKSPACE']}/" + "tests/bootstrap_impls/supplied_venv.txt" + ) + assert contents == b"custom executable supplied without interpreter runfiles" + + +def test_custom_zip_template_preserves_public_substitutions(): + path = runfiles.CreateOrRaise().Rlocation(os.environ["CUSTOM_ZIP"]) + assert path is not None + with zipfile.ZipFile(path) as archive: + contents = archive.read("__main__.py") + workspace = os.environ["TEST_WORKSPACE"] + assert contents.startswith( + f"custom zip\r\nworkspace={workspace}\r\nmain={workspace}/".encode() + ) + assert contents.endswith(b"_custom_application_stage2_bootstrap.py") + assert b"_rules_python_bootstrap" not in contents + + +@pytest.mark.parametrize("status", [0, 23]) +@pytest.mark.skipif(os.name == "nt", reason="The raw template uses Bash") +def test_raw_shell_template_accepts_historical_substitutions(tmp_path, status): + template = runfiles.CreateOrRaise().Rlocation(os.environ["RAW_SHELL_TEMPLATE"]) + assert template is not None + root = tmp_path / "runfiles" + venv = root / "app.venv" + (venv / "bin").mkdir(parents=True) + (venv / "lib/site-packages").mkdir(parents=True) + (venv / "pyvenv.cfg").touch() + (root / "entry.py").write_text( + "import json, os, sys\n" + "print(json.dumps([sys.executable, sys.argv[1:], sys._xoptions, " + "os.environ.get('RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS')]))\n" + "sys.exit(int(sys.argv[1]))\n" + ) + substitutions = { + "%stage2_bootstrap%": "entry.py", + "%python_binary%": "app.venv/bin/python3", + "%python_binary_actual%": Path(sys.executable).resolve().as_posix(), + "%is_zipfile%": "0", + "%recreate_venv_at_runtime%": "1", + "%resolve_python_binary_at_runtime%": "0", + "%venv_rel_site_packages%": "lib/site-packages", + "%interpreter_args%": "'-Xscope=target' '-Xtarget_only=present'", + } + contents = Path(template).read_text() + for key, value in substitutions.items(): + contents = contents.replace(key, value) + launcher = tmp_path / "launcher" + launcher.write_text(contents) + temporary = tmp_path / "temporary" + temporary.mkdir() + env = dict(os.environ) + for key in ( + "RULES_PYTHON_EXTRACT_ROOT", + "RULES_PYTHON_BOOTSTRAP_VERBOSE", + "PYTHONHOME", + "PYTHONPATH", + "PYTHONEXECUTABLE", + "__PYVENV_LAUNCHER__", + ): + env.pop(key, None) + env.update( + RUNFILES_DIR=str(root), + TMPDIR=str(temporary), + RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS=( + "-Xscope=environment -Xenv_only=present" + ), + ) + # The historical macOS mktemp ignores TMPDIR without a template. Keep this + # fixture's allocations inside the test directory without changing the stub. + wrapper = ( + 'mktemp() { command mktemp -d "$TMPDIR/raw.XXXXXXXXXX"; }; ' + 'export -f mktemp; exec bash "$@"' + ) + result = subprocess.run( + ["bash", "-c", wrapper, "raw-expander", str(launcher), str(status), "a b", ""], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == status, result.stderr + executable, args, options, additional = json.loads(result.stdout) + assert args == [str(status), "a b", ""] + assert options["scope"] == "target" + assert options["target_only"] == "present" + assert options["env_only"] == "present" + assert additional is None + assert Path(executable).is_relative_to(temporary) + assert not Path(executable).exists() + assert list(temporary.iterdir()) == [] diff --git a/tests/bootstrap_impls/application_test.py b/tests/bootstrap_impls/application_test.py new file mode 100644 index 0000000000..e30e8f1cc5 --- /dev/null +++ b/tests/bootstrap_impls/application_test.py @@ -0,0 +1,531 @@ +"""Behavior at the application model's storage, preparation and launch boundaries.""" + +import errno +import json +import ntpath +import os +import signal +import stat +import subprocess +import sys +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from python.private._rules_python_bootstrap import ( + entry, + environment, + model, + process, + storage, +) +from python.runfiles import runfiles + + +@pytest.fixture(name="application") +def fixture_application(tmp_path): + root = tmp_path / "runfiles" + source = root / "app.venv" + (source / "bin").mkdir(parents=True) + (source / "lib/site-packages").mkdir(parents=True) + (source / "pyvenv.cfg").touch() + (root / "links.jsonl").write_text("") + (root / "entry.py").write_text("pass\n") + value = model.Application( + entry="entry.py", + workspace="_main", + interpreter=model.Interpreter("absolute", sys.executable, False), + interpreter_args=(), + cleanup="cleanup.py", + venv=model.Venv( + "app.venv", "app.venv/bin/python", "lib/site-packages", True, "links.jsonl" + ), + ) + return value, root + + +def test_preparation_failure_removes_only_owned_workspace(tmp_path): + (tmp_path / "caller").touch() + with pytest.raises(ValueError, match="setup failed"): + with storage.Workspace(directory=tmp_path) as workspace: + owned = Path(workspace.allocate()) + (owned / "partial").touch() + assert workspace.allocate() == str(owned) + raise ValueError("setup failed") + assert not owned.exists() + assert [path.name for path in tmp_path.iterdir()] == ["caller"] + + +def test_verbose_retention_releases_ownership(tmp_path): + with storage.Workspace(directory=tmp_path, retain=True) as workspace: + owned = Path(workspace.allocate()) + workspace.remove() + assert workspace.path is None + assert owned.is_dir() + + +def test_rollback_removes_readonly_files_without_changing_link_targets(tmp_path): + caller = tmp_path / "caller" + caller.write_text("keep") + caller.chmod(0o444) + before = caller.stat().st_mode + try: + with storage.Workspace(directory=tmp_path) as workspace: + owned = Path(workspace.allocate()) + readonly = owned / "readonly" + readonly.write_text("remove") + readonly.chmod(0o444) + (owned / "borrowed").symlink_to(caller) + assert not owned.exists() + assert caller.stat().st_mode == before + assert caller.read_text() == "keep" + finally: + caller.chmod(0o644) + + +@pytest.mark.parametrize("existing", ["empty", "partial", "complete"]) +def test_publication_preserves_existing_destination(tmp_path, existing): + staging, destination = tmp_path / "staging", tmp_path / "destination" + staging.mkdir() + (staging / "new").touch() + destination.mkdir() + if existing != "empty": + (destination / "caller").write_text("keep") + if existing == "complete": + (destination / storage.COMPLETION_FILE).touch() + storage.publish_image(staging, destination) + assert not staging.exists() + else: + with pytest.raises(RuntimeError, match="Incomplete Python runtime"): + storage.publish_image(staging, destination) + assert staging.exists() + assert not (destination / "new").exists() + if existing != "empty": + assert (destination / "caller").read_text() == "keep" + + +def test_published_image_survives_workspace_rollback(tmp_path): + destination = tmp_path / "cache" + with storage.Workspace(directory=tmp_path) as workspace: + image = Path(workspace.allocate()) / "image" + image.mkdir() + (image / "payload").write_text("complete") + storage.publish_image(image, destination) + assert storage.complete_image(destination) + assert (destination / "payload").read_text() == "complete" + assert destination.is_symlink() + assert not os.path.isabs(os.readlink(destination)) + backing = destination.resolve().parent + assert set(tmp_path.iterdir()) == {destination, backing} + probe = tmp_path / "permissions" + probe.mkdir() + assert stat.S_IMODE(backing.stat().st_mode) == stat.S_IMODE(probe.stat().st_mode) + + +@pytest.mark.parametrize("collision", ["empty", "symlink", "complete"]) +def test_publication_collision_never_replaces_caller_entry( + tmp_path, monkeypatch, collision +): + destination = tmp_path / "cache" + caller = tmp_path / "caller" + caller.mkdir() + (caller / "keep").write_text("untouched") + original = os.symlink + + def concurrent_entry(target, path, **kwargs): + if collision == "symlink": + original("missing-caller-target", path, target_is_directory=True) + else: + destination.mkdir() + if collision == "complete": + (destination / storage.COMPLETION_FILE).touch() + (destination / "winner").write_text("complete") + return original(target, path, **kwargs) + + monkeypatch.setattr(os, "symlink", concurrent_entry) + with storage.Workspace(directory=tmp_path) as workspace: + image = Path(workspace.allocate()) / "image" + image.mkdir() + (image / "loser").touch() + if collision == "complete": + storage.publish_image(image, destination) + assert (destination / "winner").read_text() == "complete" + else: + with pytest.raises(RuntimeError, match="Incomplete Python runtime"): + storage.publish_image(image, destination) + assert set(tmp_path.iterdir()) == {caller, destination} + assert (caller / "keep").read_text() == "untouched" + assert not (destination / "loser").exists() + if collision == "symlink": + assert os.readlink(destination) == "missing-caller-target" + elif collision == "empty": + assert not list(destination.iterdir()) + + +def test_failed_publication_rolls_back_only_unpublished_backing(tmp_path, monkeypatch): + caller = tmp_path / "keep" + caller.write_text("untouched") + + def denied(*_args, **_kwargs): + raise OSError(errno.EACCES, "publication denied") + + monkeypatch.setattr(os, "symlink", denied) + with pytest.raises(OSError, match="publication denied"): + with storage.Workspace(directory=tmp_path) as workspace: + image = Path(workspace.allocate()) / "image" + image.mkdir() + storage.publish_image(image, tmp_path / "cache") + assert list(tmp_path.iterdir()) == [caller] + assert caller.read_text() == "untouched" + + +@pytest.mark.parametrize("after_publish", ["error", "cancellation"]) +def test_visible_image_survives_publication_interruption( + tmp_path, monkeypatch, after_publish +): + original = os.symlink + destination = tmp_path / "cache" + with process.Cancellation() as cancellation, storage.Workspace( + directory=tmp_path + ) as workspace: + + def interrupted(*args, **kwargs): + original(*args, **kwargs) + if after_publish == "error": + raise OSError(errno.EIO, "publication reply lost") + signal.raise_signal(signal.SIGINT) + + monkeypatch.setattr(os, "symlink", interrupted) + image = Path(workspace.allocate()) / "image" + image.mkdir() + (image / "payload").write_text("complete") + storage.publish_image(image, destination) + if after_publish == "cancellation": + with pytest.raises(process.Cancelled): + cancellation.check() + assert storage.complete_image(destination) + assert (destination / "payload").read_text() == "complete" + assert set(tmp_path.iterdir()) == {destination, destination.resolve().parent} + + +def test_external_only_image_preserves_main_workspace_directory(application, tmp_path): + app, _ = application + app = replace(app, entry="external/main.py", venv=None) + image = tmp_path / "image" + runfiles_root = image / "runfiles" + (runfiles_root / "external").mkdir(parents=True) + (runfiles_root / "external/main.py").write_text("pass\n") + cache = tmp_path / "published" + with process.Cancellation() as cancellation: + prepared = entry._prepare_image(app, str(image), str(cache), cancellation, None) + command = model.invocation(app, prepared, (), (), {"RUN_UNDER_RUNFILES": "1"}) + assert command.cwd is not None + assert command.cwd == str(cache / "runfiles/_main") + assert Path(command.cwd).is_dir() + assert storage.complete_image(cache) + + +def test_windows_child_registration_cancellation_reaps_before_removal( + tmp_path, monkeypatch +): + # Real process ownership, with Windows dispatch selected independently of + # the host. Native Windows console delivery is covered by platform tests. + monkeypatch.setattr(process, "os", SimpleNamespace(name="nt")) + children = [] + original = subprocess.Popen + + def interrupted_spawn(*args, **kwargs): + child = original(*args, **kwargs) + children.append(child) + signal.raise_signal(signal.SIGINT) + return child + + monkeypatch.setattr(subprocess, "Popen", interrupted_spawn) + command = model.Invocation( + sys.executable, + (sys.executable, "-c", "import time; time.sleep(60)"), + dict(os.environ), + None, + ) + before = signal.getsignal(signal.SIGINT) + try: + with pytest.raises(SystemExit) as error: + with process.Cancellation() as cancellation, storage.Workspace( + directory=tmp_path + ) as workspace: + owned = Path(workspace.allocate()) + process.execute(command, workspace, "unused", cancellation) + assert error.value.code == 128 + signal.SIGINT + assert len(children) == 1 and children[0].poll() is not None + assert not owned.exists() + assert signal.getsignal(signal.SIGINT) == before + finally: + for child in children: + if child.poll() is None: + child.kill() + child.wait() + + +@pytest.mark.parametrize( + "mode,status", + [ + ("handled", 23), + ("ignored", 17), + ("inherited_ignore", 29), + ("high_bit", 0xC000013A), + ], +) +def test_native_windows_console_handoff(tmp_path, mode, status): + if sys.platform != "win32": + pytest.skip("native Windows console delivery") + else: + fixture = runfiles.CreateOrRaise().Rlocation( + os.environ["WINDOWS_CONSOLE_FIXTURE"] + ) + assert fixture is not None + startup = subprocess.STARTUPINFO() + startup.dwFlags |= subprocess.STARTF_USESHOWWINDOW + result = subprocess.run( + [sys.executable, fixture, "controller", str(tmp_path), mode], + capture_output=True, + text=True, + timeout=30, + creationflags=subprocess.CREATE_NEW_CONSOLE, + startupinfo=startup, + check=False, + ) + assert result.returncode == status, result.stdout + result.stderr + assert not Path((tmp_path / "workspace").read_text()).exists() + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX signal-mask handoff") +def test_exec_transition_detects_signal_during_handler_restoration(monkeypatch): + original = signal.signal + with process.Cancellation() as cancellation: + injected = False + + def restore(signum, disposition): + nonlocal injected + previous = original(signum, disposition) + if not injected: + injected = True + # Python handlers can run on the main thread for a signal + # delivered to another, unblocked thread during this transition. + cancellation._record(signal.SIGTERM, None) + return previous + + with monkeypatch.context() as patch: + patch.setattr(signal, "signal", restore) + with pytest.raises(process.Cancelled) as error: + cancellation.prepare_exec() + assert error.value.signum == signal.SIGTERM + assert injected + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX inherited masks") +def test_exec_transition_preserves_inherited_mask(): + if sys.platform == "win32": + pytest.skip("POSIX inherited masks") + else: + original = signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGHUP]) + try: + expected = signal.pthread_sigmask(signal.SIG_BLOCK, []) + with process.Cancellation() as cancellation: + cancellation.prepare_exec() + assert signal.pthread_sigmask(signal.SIG_BLOCK, []) == expected + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, original) + + +def test_runfiles_wrapper_can_resolve_external_interpreter(application, tmp_path): + app, root = application + # Declared as runfiles, but the wrapper resolves to an external executable. + # Publication must not turn that absolute reference into a relative link. + app = replace(app, interpreter=model.Interpreter("runfiles", "wrapper", True)) + runtime = environment.current_runtime() + destination = root / app.venv.root + with process.Cancellation() as cancellation: + executable = environment.prepare_venv( + app, + str(root), + str(destination), + cancellation, + runtime=runtime, + image_owned=True, + ) + assert os.path.isabs(os.readlink(executable)) + assert Path(executable).resolve() == Path(sys.executable).resolve() + + +def test_overlay_includes_explicit_links_without_source_directories( + application, tmp_path +): + app, root = application + payload = root / "header.h" + payload.write_text("header") + (root / app.venv.links).write_text( + json.dumps(["include/package/header.h", "header.h"]) + "\n" + ) + destination = tmp_path / "prepared" + with process.Cancellation() as cancellation: + environment.prepare_venv(app, str(root), str(destination), cancellation) + assert (destination / "include/package/header.h").read_text() == "header" + assert (destination / "lib/site-packages").is_symlink() + + +def test_preparation_does_not_enumerate_dependency_tree( + application, tmp_path, monkeypatch +): + app, root = application + dependencies = root / app.venv.root / app.venv.site_packages + nested = dependencies / "package/nested" + nested.mkdir(parents=True) + (nested / "module.py").write_text("VALUE = 42\n") + original_listdir = os.listdir + + def listdir(path): + assert not Path(path).is_relative_to(dependencies), ( + "walked application dependencies" + ) + return original_listdir(path) + + monkeypatch.setattr(os, "listdir", listdir) + destination = tmp_path / "prepared" + with process.Cancellation() as cancellation: + environment.venv_layout_identity(app, str(root)) + environment.prepare_venv(app, str(root), str(destination), cancellation) + assert ( + destination / app.venv.site_packages / "package/nested/module.py" + ).read_text() == "VALUE = 42\n" + + +@pytest.mark.parametrize("change", ["links", "bin", "data", "implementation"]) +def test_directory_cache_tracks_preparation_inputs( + application, monkeypatch, tmp_path, change +): + app, root = application + implementation = tmp_path / "implementation" + implementation.mkdir() + for name in ( + "diagnostics.py", + "environment.py", + "entry.py", + "model.py", + "process.py", + "storage.py", + ): + (implementation / name).write_text("initial") + monkeypatch.setattr(environment, "__file__", str(implementation / "environment.py")) + original = environment.venv_layout_identity(app, str(root)) + assert environment.venv_layout_identity(app, str(root)) == original + if change == "links": + (root / app.venv.links).write_text( + json.dumps(["include/header.h", "header.h"]) + "\n" + ) + elif change == "bin": + (root / app.venv.root / "bin/new-tool").touch() + elif change == "data": + (root / app.venv.root / "share").mkdir() + else: + (implementation / "environment.py").write_text("changed") + assert environment.venv_layout_identity(app, str(root)) != original + + +def test_runtime_identity_ignores_image_staging_path(tmp_path): + root1, root2 = ( + str(tmp_path / name) for name in ("first/runfiles", "second/runfiles") + ) + runtime = environment.Runtime( + root1 + "/python/bin/python", + root1 + "/python", + "lib/site-packages", + (3, 11, 1), + "", + ) + relocated = replace( + runtime, executable=root2 + "/python/bin/python", prefix=root2 + "/python" + ) + assert runtime.identity(root1) == relocated.identity(root2) + assert runtime.identity(root1) != replace(runtime, version=(3, 12, 0)).identity( + root1 + ) + + +def test_image_containment_accepts_external_windows_drive(monkeypatch): + monkeypatch.setattr(environment, "os", SimpleNamespace(path=ntpath)) + assert environment._inside(r"D:\runtime\bin\python.exe", r"D:\runtime") + assert not environment._inside(r"C:\Python\python.exe", r"D:\runtime") + assert not environment._inside(r"D:\runtime-other\python.exe", r"D:\runtime") + + +def test_invocation_keeps_preparation_state_out_of_application_environment(application): + app, root = application + inherited = { + "PYTHONSAFEPATH": "", + "RUN_UNDER_RUNFILES": "1", + "RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS": "-Xagain", + "__PYVENV_LAUNCHER__": "wrong", + "RUNFILES_MANIFEST_FILE": "unrelated", + } + prepared = model.PreparedApplication(str(root), sys.executable) + command = model.invocation( + app, + prepared, + ["-m", "debugpy"], + ["argument with spaces"], + inherited, + temporary_zip=str(root.parent), + ) + assert command.argv == ( + sys.executable, + "-XRULES_PYTHON_ZIP_DIR=" + str(root.parent), + "-m", + "debugpy", + str(root / app.entry), + "argument with spaces", + ) + assert command.cwd == str(root / "_main") + assert command.environment == { + "PYTHONSAFEPATH": "", + "RUN_UNDER_RUNFILES": "1", + "RUNFILES_DIR": str(root), + } + assert inherited["RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS"] == "-Xagain" + + +def test_old_python_template_expansion_keeps_public_arguments(tmp_path): + files = runfiles.CreateOrRaise() + path = files.Rlocation(os.environ["PY_TEMPLATE_RLOCATION"]) + assert path is not None + template = Path(path).read_text() + root = tmp_path / "runfiles" + root.mkdir() + (root / "probe.py").write_text( + "import sys; assert sys._xoptions['legacy'] == 'works'\n" + ) + substitutions = { + "%shebang%": "#!/usr/bin/env python3", + "%main%": "probe.py", + "%python_binary%": Path(sys.executable).as_posix(), + "%python_binary_actual%": Path(sys.executable).as_posix(), + "%interpreter_args%": "-Xlegacy=works", + "%runtime_venv_symlinks%": "", + "%is_zipfile%": "0", + "%recreate_venv_at_runtime%": "0", + "%resolve_python_binary_at_runtime%": "0", + "%workspace_name%": "_main", + } + for key, value in substitutions.items(): + template = template.replace(key, value) + script = tmp_path / "legacy.py" + script.write_text(template) + result = subprocess.run( + [sys.executable, str(script)], + env=dict(os.environ, RUNFILES_DIR=str(root)), + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 0, result.stderr diff --git a/tests/bootstrap_impls/application_tests.bzl b/tests/bootstrap_impls/application_tests.bzl new file mode 100644 index 0000000000..229a134ee0 --- /dev/null +++ b/tests/bootstrap_impls/application_tests.bzl @@ -0,0 +1,103 @@ +"""Build contracts shared by directory and ZIP application transports.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python:py_executable_info.bzl", "PyExecutableInfo") +load("//python:py_runtime_info.bzl", "PyRuntimeInfo") +load("//python/private:py_application_info.bzl", "PyApplicationInfo") # buildifier: disable=bzl-visibility + +def _test_directory_image(name): + analysis_test(name = name, impl = _test_directory_image_impl, target = ":cleanup_venv") + +def _test_directory_image_impl(env, target): + image = target[PyApplicationInfo] + public = target[PyExecutableInfo] + files = image.runfiles.files.to_list() + env.expect.that_collection(files).contains_at_least(public.app_runfiles.files.to_list()) + env.expect.that_collection(files).contains(image.spec) + env.expect.that_collection(files).contains(public.stage2_bootstrap) + env.expect.that_bool(image.settings["entry"].endswith(public.stage2_bootstrap.short_path)).equals(True) + env.expect.that_collection([file.basename for file in files]).contains_at_least([ + "entry.py", + "environment.py", + "process.py", + "storage.py", + "model.py", + "bootstrap_cleanup.py", + ]) + env.expect.that_collection([a.mnemonic for a in target.actions]).contains_none_of([ + "PyBootstrapTemplate", + ]) + +def _test_archive_image(name): + analysis_test(name = name, impl = _test_archive_image_impl, target = ":cleanup_zipapp") + +def _test_archive_image_impl(env, target): + archives = [a for a in target.actions if a.mnemonic == "PyApplicationArchive"] + env.expect.that_int(len(archives)).equals(1) + inputs = [file.basename for file in archives[0].inputs.to_list()] + env.expect.that_collection(inputs).contains_at_least([ + "cleanup_venv.application.json", + "cleanup_venv.application_links.jsonl", + "entry.py", + "environment.py", + ]) + + # Packaging uses the binary's environment and entry, with no second venv. + env.expect.that_collection(inputs).contains_none_of([ + "cleanup_zipapp.application.json", + "cleanup_zipapp.application_links.jsonl", + ]) + +def application_test_suite(name): + test_suite(name = name, tests = [_test_directory_image, _test_archive_image]) + +def _template_fixture_impl(ctx): + output = ctx.actions.declare_file(ctx.label.name + ".txt") + ctx.actions.write(output, ctx.attr.contents) + return [DefaultInfo(files = depset([output]))] + +template_fixture = rule( + implementation = _template_fixture_impl, + attrs = {"contents": attr.string()}, +) + +def _template_output_impl(ctx): + binary = ctx.attr.binary[DefaultInfo] + path = binary.files_to_run.executable.path + if path.endswith(".exe"): + path = path[:-4] + outputs = [file for file in binary.files.to_list() if file.path == path] + if len(outputs) != 1: + fail("Expected one text bootstrap output for {}".format(ctx.attr.binary.label)) + return [DefaultInfo(files = depset(outputs))] + +template_output = rule( + implementation = _template_output_impl, + attrs = {"binary": attr.label(mandatory = True)}, +) + +def _public_executable_impl(ctx): + binary = ctx.attr.binary + public = binary[PyExecutableInfo] + if ctx.file.supplied_venv: + # A public provider may supply its executable independently of the + # optional interpreter runfiles collection. + public = PyExecutableInfo( + app_runfiles = public.app_runfiles, + interpreter_args = public.interpreter_args, + stage2_bootstrap = public.stage2_bootstrap, + venv_app_symlinks = public.venv_app_symlinks, + venv_interpreter_runfiles = None, + venv_interpreter_symlinks = depset(), + venv_python_exe = ctx.file.supplied_venv, + ) + return [public, binary[PyRuntimeInfo]] + +public_executable = rule( + implementation = _public_executable_impl, + attrs = { + "binary": attr.label(providers = [PyExecutableInfo, PyRuntimeInfo]), + "supplied_venv": attr.label(allow_single_file = True), + }, +) diff --git a/tests/bootstrap_impls/bootstrap_cleanup_test.py b/tests/bootstrap_impls/bootstrap_cleanup_test.py new file mode 100644 index 0000000000..e75169439b --- /dev/null +++ b/tests/bootstrap_impls/bootstrap_cleanup_test.py @@ -0,0 +1,1054 @@ +"""Exercise generated launchers without replacing their process or signals.""" + +import contextlib +import json +import os +import pty +import re +import select +import shlex +import shutil +import signal +import struct +import subprocess +import sys +import threading +import time +import zipfile +from pathlib import Path + +import pytest + +from python.runfiles import runfiles + + +def wait_until(predicate, description): + deadline = time.monotonic() + 10 + while not predicate(): + assert time.monotonic() < deadline, description + time.sleep(0.01) + + +class Launcher: + def __init__(self, root, mode): + self.root = root + self.mode = mode + self.shell = mode in ( + "venv", + "zip", + "zipapp", + "zipapp_compressed", + "wrapper", + "wrapper_zip", + ) + self.archive = mode not in ("venv", "pyvenv", "wrapper") + self.temporary_runtime = ( + mode != "pyvenv" or os.environ["PYVENV_TEMPORARY"] == "1" + ) + self.files = runfiles.CreateOrRaise() + self.scratch = root / "temporary runtime with spaces" + self.scratch.mkdir() + self.environment = dict(os.environ, TMPDIR=str(self.scratch)) + for key in ( + "RULES_PYTHON_EXTRACT_ROOT", + "RULES_PYTHON_BOOTSTRAP_VERBOSE", + "RUNFILES_DIR", + "RUNFILES_MANIFEST_FILE", + ): + self.environment.pop(key, None) + self.binary = root / "launcher with spaces" + source = self.locate(mode.upper() + "_RLOCATION") + if self.archive: + self.binary.symlink_to(source) + else: + shutil.copyfile(source, self.binary) + self.binary.chmod(0o755) + Path(str(self.binary) + ".runfiles").symlink_to(self.files.root()) + self.command = [ + os.environ.get("BOOTSTRAP_TEST_BASH", "/bin/bash") + if self.shell + else sys.executable, + str(self.binary), + ] + + def locate(self, name): + result = self.files.Rlocation(os.environ[name]) + assert result is not None, name + return result + + def replace_template_path(self, variable, location): + assert self.mode == "venv" + source, count = re.subn( + rf"^{variable}=.*$", + f"{variable}={shlex.quote(location)}", + self.binary.read_text(), + flags=re.MULTILINE, + ) + assert count == 1 + self.binary.write_text(source) + + def run(self, *arguments, environment=None, command=None): + return subprocess.run( + [*(command or self.command), *arguments], + env=environment or self.environment, + input="input with spaces", + text=True, + capture_output=True, + timeout=20, + check=False, + ) + + def cleaned(self): + wait_until(lambda: not list(self.scratch.iterdir()), "temporary runtime leaked") + + @contextlib.contextmanager + def process(self, *arguments, environment=None, **kwargs): + process = subprocess.Popen( + [*self.command, *arguments], + env=environment or self.environment, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + **kwargs, + ) + try: + yield process + finally: + if process.poll() is None: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.communicate(timeout=10) + self.cleaned() + + +@pytest.fixture( + name="launcher", + params=[ + "venv", + "zip", + "pyvenv", + "pyzip", + "legacy_python", + "zipapp", + "zipapp_python", + "zipapp_compressed", + "zipapp_system", + ], +) +def fixture_launcher(tmp_path, request): + return Launcher(tmp_path, request.param) + + +@pytest.fixture(name="venv") +def fixture_venv(tmp_path): + return Launcher(tmp_path, "venv") + + +@pytest.fixture(name="shell_launcher", params=["zip", "zipapp"]) +def fixture_shell_launcher(tmp_path, request): + return Launcher(tmp_path, request.param) + + +@pytest.mark.parametrize("status", [0, 17, 143]) +def test_spaces_tmpdir_stdin_arguments_and_status(launcher, status): + result = launcher.run("echo", str(status), "argument with spaces") + assert result.returncode == status, result.stderr + actual = json.loads(result.stdout) + assert actual["argv"] == ["argument with spaces"] + assert actual["stdin"] == "input with spaces" + assert ( + Path(actual["executable"]).is_relative_to(launcher.scratch) + == launcher.temporary_runtime + ) + assert actual["ignored"] == [False, False] + assert ( + actual["xoptions"]["bootstrap_target"] + == "argument with spaces and 'quotes' and \"double quotes\"" + ) + launcher.cleaned() + + +def test_persistent_extract_root(venv): + extract_root = venv.root / "persistent runtime with spaces" + result = venv.run( + "echo", + "0", + environment=dict(venv.environment, RULES_PYTHON_EXTRACT_ROOT=str(extract_root)), + ) + assert result.returncode == 0, result.stderr + assert Path(json.loads(result.stdout)["executable"]).is_relative_to(extract_root) + assert list(extract_root.iterdir()) + venv.cleaned() + + +@pytest.mark.parametrize("mode", ["wrapper", "wrapper_zip", "wrapper_zip_python"]) +def test_runtime_handoff_resolves_wrapper_once_and_preserves_basename(tmp_path, mode): + launcher = Launcher(tmp_path, mode) + environment = startup_environment(launcher) + environment["CLEANUP_REAL_INTERPRETER"] = sys.executable + environment["CLEANUP_WRAPPER_DATA"] = os.environ["WRAPPER_DATA_RLOCATION"] + environment["RUNFILES_MANIFEST_FILE"] = str(tmp_path / "unrelated manifest") + result = launcher.run("echo", "0", environment=environment) + assert result.returncode == 0, result.stderr + actual = json.loads(result.stdout) + assert Path(actual["executable"]).name == "cleanup_python_wrapper.sh" + assert actual["handler"] == "user_interrupt" + launcher.cleaned() + + +def test_shell_handoff_preserves_read_array_parsing(venv): + result = venv.run( + "echo", + "0", + environment=dict( + venv.environment, + RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS="-Xread_array=two\\ words\n-Xignored_second_line=1", + ), + ) + assert result.returncode == 0, result.stderr + options = json.loads(result.stdout)["xoptions"] + assert options["read_array"] == "two words" + assert "ignored_second_line" not in options + venv.cleaned() + + +def test_shell_handoff_exports_selected_runfiles_with_manifest(venv): + root = venv.root / "provided.runfiles" + root.symlink_to(os.fspath(venv.files.root()), target_is_directory=True) + manifest = venv.root / "provided.runfiles_manifest" + manifest.write_text( + os.environ["TEST_WORKSPACE"] + "/fixture " + str(venv.binary) + "\n" + ) + result = venv.run( + "echo", + "0", + environment=dict( + venv.environment, + RUNFILES_MANIFEST_FILE=str(manifest), + ), + ) + assert result.returncode == 0, result.stderr + assert Path(json.loads(result.stdout)["runfiles"]) == root + venv.cleaned() + + +def test_shell_handoff_finds_interpreter_on_path(venv): + directory = venv.root / "interpreter on path" + directory.mkdir() + (directory / "python3").symlink_to(sys.executable) + venv.replace_template_path("PYTHON_BINARY_ACTUAL", "python3") + venv.replace_template_path("INTERPRETER_KIND", "path") + result = venv.run( + "echo", + "0", + environment=dict( + venv.environment, + PATH=str(directory) + os.pathsep + venv.environment["PATH"], + ), + ) + assert result.returncode == 0, result.stderr + venv.cleaned() + + +def test_caller_ignored_signals(launcher): + result = launcher.run( + "echo", + "0", + command=[ + "/bin/bash", + "-c", + 'trap "" INT QUIT; exec "$@"', + "ignored", + *launcher.command, + ], + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["ignored"] == [True, True] + launcher.cleaned() + + +def test_application_can_wait_for_all_children(launcher): + result = launcher.run("wait_children") + assert result.returncode == 0, result.stderr + launcher.cleaned() + + +@pytest.mark.parametrize( + "value", [signal.SIGINT, signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT] +) +@pytest.mark.parametrize("group", [False, True]) +def test_signal_waits_for_application_cleanup(launcher, value, group): + with launcher.process("handled", str(launcher.root)) as process: + wait_until( + lambda: (launcher.root / "ready").exists(), "application did not start" + ) + assert int((launcher.root / "ready").read_text()) == process.pid + if group: + os.killpg(process.pid, value) + else: + process.send_signal(value) + wait_until( + lambda: (launcher.root / "stopping").exists(), "cancellation did not arrive" + ) + assert process.poll() is None + assert bool(list(launcher.scratch.iterdir())) == launcher.temporary_runtime + process.send_signal(signal.SIGTERM) + process.send_signal(signal.SIGHUP) + assert process.stdin is not None + process.stdin.write("cleanup\n") + process.stdin.flush() + stdout, stderr = process.communicate(timeout=10) + assert process.returncode == 128 + value, stdout + stderr + assert (launcher.root / "cleaned").exists() + + +def test_one_group_interrupt_does_not_interrupt_finally_twice(launcher): + with launcher.process("interrupt", str(launcher.root)) as process: + wait_until( + lambda: (launcher.root / "ready").exists(), "application did not start" + ) + os.killpg(process.pid, signal.SIGINT) + wait_until( + lambda: (launcher.root / "stopping").exists(), "finally did not start" + ) + assert process.stdin is not None + process.stdin.write("cleanup\n") + process.stdin.flush() + stdout, stderr = process.communicate(timeout=10) + assert process.returncode in (-signal.SIGINT, 130), stdout + stderr + assert (launcher.root / "cleaned").exists() + + +@pytest.mark.parametrize("value", [signal.SIGTERM, signal.SIGKILL]) +def test_default_signal_status_and_cleanup(launcher, value): + with launcher.process("default", str(launcher.root)) as process: + wait_until( + lambda: (launcher.root / "ready").exists(), "application did not start" + ) + os.killpg(process.pid, value) + process.communicate(timeout=10) + assert process.returncode == -value + + +def test_custom_stage2_keeps_default_signal_dispositions(venv): + venv.replace_template_path("STAGE2_BOOTSTRAP", os.environ["PROBE_RLOCATION"]) + result = venv.run("echo", "0") + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["ignored"] == [False, False] + venv.cleaned() + + +def startup_environment(launcher): + startup = launcher.root / "user startup" + startup.mkdir() + shutil.copyfile(launcher.locate("STARTUP_RLOCATION"), startup / "sitecustomize.py") + return dict(launcher.environment, PYTHONPATH=str(startup)) + + +def test_sitecustomize_handler_and_interpreter_arguments(launcher): + environment = startup_environment(launcher) + environment["RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS"] = "-Xbootstrap_probe=1" + result = launcher.run("echo", "0", environment=environment) + assert result.returncode == 0, result.stderr + actual = json.loads(result.stdout) + assert actual["handler"] == "user_interrupt" + assert actual["xoptions"]["bootstrap_probe"] == "1" + assert actual["additional_args"] is None + launcher.cleaned() + + +def test_interpreter_argument_precedence_and_safe_path_optout(launcher): + result = launcher.run( + "echo", + "0", + environment=dict( + launcher.environment, + PYTHONSAFEPATH="", + RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS="-Xbootstrap_precedence=additional", + ), + ) + assert result.returncode == 0, result.stderr + actual = json.loads(result.stdout) + assert actual["xoptions"]["bootstrap_precedence"] == ( + "target" if launcher.shell else "additional" + ) + assert actual["safe_path"] == "" + assert actual["additional_args"] is None + launcher.cleaned() + + +def test_nested_launcher_does_not_reconsume_additional_args(launcher): + result = launcher.run( + "nested", + *launcher.command, + "echo", + "0", + environment=dict( + launcher.environment, + RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS="-Xouter_only=1", + ), + ) + assert result.returncode == 0, result.stderr + actual = json.loads(result.stdout) + assert "outer_only" not in actual["xoptions"] + assert actual["additional_args"] is None + launcher.cleaned() + + +@pytest.mark.parametrize( + "mode", ["pyvenv", "pyzip", "legacy_python", "zipapp_python", "zipapp_system"] +) +def test_python_debugger_injection(tmp_path, mode): + launcher = Launcher(tmp_path, mode) + debugger = tmp_path / "debugger.py" + debugger.write_text( + "import runpy, sys\nassert sys.argv[1] == '--file'\nsys.argv = sys.argv[2:]\nrunpy.run_path(sys.argv[0], run_name='__main__')\n" + ) + result = launcher.run( + "echo", + "0", + environment=dict( + launcher.environment, + RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS=shlex.join( + [str(debugger), "--file"] + ), + ), + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["additional_args"] is None + launcher.cleaned() + + +def test_relative_tmpdir_and_zip_working_directory(launcher): + environment = dict( + launcher.environment, + TMPDIR=os.path.relpath(launcher.scratch), + RUN_UNDER_RUNFILES="1", + ) + result = launcher.run("echo", "0", environment=environment) + assert result.returncode == 0, result.stderr + actual = json.loads(result.stdout) + assert ( + Path(actual["executable"]).is_relative_to(launcher.scratch) + == launcher.temporary_runtime + ) + if launcher.archive: + assert actual["cwd"] == os.path.normpath( + str(Path(actual["runfiles"]) / os.environ["TEST_WORKSPACE"]) + ) + launcher.cleaned() + + +@pytest.mark.parametrize( + "mode", ["zipapp", "zipapp_python", "zipapp_compressed", "zipapp_system"] +) +def test_persistent_zip_reuse_and_concurrent_publication(tmp_path, mode): + launcher = Launcher(tmp_path, mode) + root = tmp_path / "persistent cache with spaces" + environment = dict( + launcher.environment, RULES_PYTHON_EXTRACT_ROOT=os.path.relpath(root) + ) + with contextlib.ExitStack() as resources: + processes = [ + resources.enter_context( + launcher.process("echo", "0", environment=environment) + ) + for _ in range(4) + ] + results = [ + process.communicate("input with spaces", timeout=30) + for process in processes + ] + for process, (_, stderr) in zip(processes, results): + assert process.returncode == 0, stderr + records = [json.loads(stdout) for stdout, _ in results] + assert len({record["executable"] for record in records}) == 1 + assert len(list(root.rglob(".rules_python_complete"))) == 1 + assert not list(root.rglob(".rules_python.*")) + assert all("RULES_PYTHON_ZIP_DIR" not in record["xoptions"] for record in records) + first = records[0] + sentinel = Path(first["runfiles"]) / "caller-kept" + sentinel.write_text("preserve completed runtime") + result = launcher.run("echo", "0", environment=environment) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["executable"] == first["executable"] + assert sentinel.read_text() == "preserve completed runtime" + launcher.cleaned() + + +@pytest.mark.parametrize("mode", ["zip", "pyzip", "legacy_python"]) +def test_legacy_zip_does_not_cache_links_into_temporary_extraction(tmp_path, mode): + launcher = Launcher(tmp_path, mode) + root = tmp_path / "persistent root" + root.mkdir() + (root / "keep").touch() + environment = dict(launcher.environment, RULES_PYTHON_EXTRACT_ROOT=str(root)) + for _ in range(2): + result = launcher.run("echo", "0", environment=environment) + assert result.returncode == 0, result.stderr + assert Path(json.loads(result.stdout)["executable"]).is_relative_to( + launcher.scratch + ) + launcher.cleaned() + assert [path.name for path in root.iterdir()] == ["keep"] + + +def test_warm_shell_zip_executes_without_preparation(tmp_path): + launcher = Launcher(tmp_path, "zipapp") + cache = tmp_path / "cache" + environment = dict(launcher.environment, RULES_PYTHON_EXTRACT_ROOT=str(cache)) + cold = launcher.run("echo", "0", environment=environment) + assert cold.returncode == 0, cold.stderr + prepared = json.loads(cold.stdout) + runfiles_root = Path(prepared["runfiles"]) + sentinel = runfiles_root / "caller-kept" + sentinel.write_text("keep") + + payload = launcher.binary.read_bytes() + payload, count = re.subn( + rb"^BOOTSTRAP_DRIVER=.*$", b"BOOTSTRAP_DRIVER=must-not-run", payload, flags=re.M + ) + assert count == 1 + launcher.binary.unlink() + launcher.binary.write_bytes(payload) + tools = tmp_path / "tools" + tools.mkdir() + for name in ("mktemp", "unzip"): + tool = tools / name + tool.write_text("#!/bin/sh\nexit 97\n") + tool.chmod(0o755) + environment.update( + PATH=str(tools) + os.pathsep + environment["PATH"], + RUN_UNDER_RUNFILES="1", + RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS="-Xwarm_cache=once", + ) + result = launcher.run("echo", "17", "warm argument", environment=environment) + assert result.returncode == 17, result.stderr + actual = json.loads(result.stdout) + assert actual["argv"] == ["warm argument"] + assert actual["runfiles"] == prepared["runfiles"] + assert actual["executable"] == prepared["executable"] + assert Path(actual["cwd"]).samefile(runfiles_root / os.environ["TEST_WORKSPACE"]) + assert actual["xoptions"]["warm_cache"] == "once" + assert "RULES_PYTHON_ZIP_DIR" not in actual["xoptions"] + assert sentinel.read_text() == "keep" + launcher.cleaned() + + # A failed native exec must not take ownership of the completed image. + invalid = runfiles_root / "invalid-executable" + invalid.write_text("#!/nonexistent/interpreter\n") + invalid.chmod(0o755) + payload, count = re.subn( + rb"^PYTHON_BINARY=.*$", b"PYTHON_BINARY=invalid-executable", payload, flags=re.M + ) + assert count == 1 + launcher.binary.write_bytes(payload) + failed = launcher.run("echo", "0", environment=environment) + assert failed.returncode != 0 + assert sentinel.read_text() == "keep" + assert list(cache.rglob(".rules_python_complete")) + launcher.cleaned() + + +def test_standalone_archive(launcher): + if not launcher.archive: + pytest.skip("ordinary binary requires runfiles") + Path(str(launcher.binary) + ".runfiles").unlink() + # Copy the bytes too: do not let the test depend on a Bazel output symlink. + archive = launcher.binary.resolve() + launcher.binary.unlink() + shutil.copyfile(archive, launcher.binary) + result = launcher.run("echo", "0") + assert result.returncode == 0, result.stderr + launcher.cleaned() + + +@pytest.mark.parametrize( + "mode", ["zip", "pyzip", "legacy_python", "zipapp", "zipapp_python"] +) +def test_failed_partial_extraction_removes_owned_tree(tmp_path, mode): + launcher = Launcher(tmp_path, mode) + archive = launcher.binary.resolve() + launcher.binary.unlink() + shutil.copyfile(archive, launcher.binary) + with zipfile.ZipFile(launcher.binary) as source: + entry = next( + info + for info in source.infolist() + if info.filename.endswith("cleanup_probe.py") + ) + offset = entry.header_offset + with launcher.binary.open("r+b") as stream: + stream.seek(offset + 26) + name_size, extra_size = struct.unpack(" "$EXTRACTION_BARRIER/extractor"\n' + "while true; do sleep .05; done\n" + ) + unzip.chmod(0o755) + environment = dict( + launcher.environment, + PATH=str(tools) + os.pathsep + launcher.environment["PATH"], + EXTRACTION_BARRIER=str(launcher.root), + ) + with launcher.process("echo", "0", environment=environment) as parent: + wait_until( + lambda: (launcher.root / "extractor").exists(), "extraction did not start" + ) + parent.terminate() + stdout, stderr = parent.communicate(timeout=10) + assert parent.returncode == 143, stdout + stderr + assert (launcher.root / "stopped").exists() + extractor = int((launcher.root / "extractor").read_text()) + with pytest.raises(ProcessLookupError): + os.kill(extractor, 0) + + +@pytest.mark.parametrize("invalid", ["truncated", "extra", "version", "ownership"]) +def test_shell_rejects_invalid_preparation_result(shell_launcher, invalid): + launcher = shell_launcher + fields = ["1", "", "runfiles", "python", "entry", "unowned"] + if invalid == "extra": + fields.append("extra") + elif invalid == "version": + fields[0] = "2" + record = b"\0".join(field.encode() for field in fields) + b"\0" + if invalid == "truncated": + record = record[:-1] + result = launcher.root / "preparation-result" + result.write_bytes(record) + source = launcher.binary.read_bytes() + command = re.compile( + re.escape( + b'run_preparer "$python_actual" -I -S "$RUNFILES_DIR/$BOOTSTRAP_DRIVER" ' + b"\\\n" + ) + + rb"[ \t]+" + + re.escape(b'prepare-archive "$workspace" "$image" "$cached"') + ) + source, count = command.subn( + b'cp "$PREPARATION_RESULT" "$workspace/invocation"', source + ) + assert count == 1 + launcher.binary.unlink() + launcher.binary.write_bytes(source) + completed = launcher.run( + "echo", + "0", + environment=dict(launcher.environment, PREPARATION_RESULT=str(result)), + ) + assert completed.returncode != 0 + assert ( + "workspace ownership" in completed.stderr + if invalid == "ownership" + else "Invalid Python preparation result" in completed.stderr + ) + assert not completed.stdout + launcher.cleaned() + + +def test_cancellation_before_stage2(launcher): + environment = startup_environment(launcher) + environment["CLEANUP_STARTUP_PROBE"] = str(launcher.root) + with launcher.process("echo", "0", environment=environment) as process: + wait_until( + lambda: (launcher.root / "starting").exists(), "interpreter did not start" + ) + process.terminate() + process.communicate(timeout=10) + assert process.returncode == -signal.SIGTERM + + +def test_cleanup_readiness_failure_prevents_application_start(venv): + venv.replace_template_path("BOOTSTRAP_CLEANUP", os.environ["FAIL_RLOCATION"]) + result = venv.run("echo", "0") + assert result.returncode == 1 + assert "cleanup_fail.py" in result.stderr + assert not result.stdout + venv.cleaned() + + +def test_cancellation_before_helper_readiness(venv): + venv.replace_template_path("BOOTSTRAP_CLEANUP", os.environ["FAIL_RLOCATION"]) + environment = dict(venv.environment, CLEANUP_HELPER_STARTUP_PROBE=str(venv.root)) + with venv.process("echo", "0", environment=environment) as process: + wait_until( + lambda: (venv.root / "helper_starting").exists(), + "helper did not start", + ) + process.terminate() + process.communicate(timeout=10) + assert process.returncode == -signal.SIGTERM + helper_pid = int((venv.root / "helper_starting").read_text()) + with pytest.raises(ProcessLookupError): + os.kill(helper_pid, 0) + + +def test_setup_failure_is_cleaned(venv): + venv.replace_template_path("PYTHON_BINARY_ACTUAL", "missing/interpreter") + result = venv.run("echo", "0") + assert result.returncode != 0 + assert "missing/interpreter" in result.stderr + venv.cleaned() + + +def sentinel_environment(launcher): + environment = dict(launcher.environment) + sentinels = [] + for name in ("zip_dir", "venv"): + directory = launcher.root / ("caller-owned-" + name) + directory.mkdir() + sentinel = directory / "keep" + sentinel.write_text("caller-owned") + sentinels.append(sentinel) + environment[name] = str(directory) + return environment, sentinels + + +@pytest.mark.parametrize("allocation_fails", [False, True]) +def test_inherited_path_variables_cannot_select_cleanup(launcher, allocation_fails): + environment, sentinels = sentinel_environment(launcher) + if allocation_fails: + environment["TMPDIR"] = str(launcher.root / "missing temporary root") + result = launcher.run("echo", "0", environment=environment) + if not launcher.temporary_runtime: + assert result.returncode == 0, result.stderr + assert Path(json.loads(result.stdout)["executable"]).is_file() + elif allocation_fails and not launcher.shell: + # tempfile falls back from an invalid TMPDIR to the platform temp root. + assert result.returncode == 0, result.stderr + wait_until( + lambda: not Path(json.loads(result.stdout)["executable"]).exists(), + "fallback runtime leaked", + ) + else: + assert (result.returncode != 0) == allocation_fails, result.stderr + launcher.cleaned() + assert [path.read_text() for path in sentinels] == ["caller-owned"] * 2 + + +def test_cancellation_during_temporary_directory_allocation(shell_launcher): + launcher = shell_launcher + environment, sentinels = sentinel_environment(launcher) + result = launcher.run( + "echo", + "0", + environment=environment, + command=[ + "/bin/bash", + "-c", + 'mktemp() { /usr/bin/mktemp "$@"; kill -TERM "$$"; }; ' + 'export -f mktemp; exec "$@"', + "cancel-allocation", + *launcher.command, + ], + ) + assert result.returncode == 143, result.stderr + assert not result.stdout + launcher.cleaned() + assert [path.read_text() for path in sentinels] == ["caller-owned"] * 2 + + +def test_exec_failure_is_cleaned(venv): + match = re.search( + r"^BOOTSTRAP_CLEANUP=(.*)$", venv.binary.read_text(), re.MULTILINE + ) + assert match is not None + location = venv.files.Rlocation(shlex.split(match[1])[0]) + assert location is not None + real_helper = Path(location).resolve() + assert real_helper.is_file() + helper = venv.root / "remove_prepared_interpreter.py" + marker = venv.root / "prepared-executable" + helper.write_text( + "import os, runpy, sys\n" + "from pathlib import Path\n" + "pid = os.getpid()\n" + f"status = runpy.run_path({str(real_helper)!r})['main']()\n" + "if status == 0 and os.getpid() == pid:\n" + " executable = Path(sys.executable)\n" + " assert executable.is_relative_to(sys.argv[2])\n" + " assert executable.is_symlink()\n" + f" Path({str(marker)!r}).write_text(str(executable))\n" + " executable.unlink()\n" + "sys.exit(status)\n" + ) + venv.replace_template_path("BOOTSTRAP_CLEANUP", str(helper)) + result = venv.run("echo", "0") + assert result.returncode != 0 + assert "FileNotFoundError" in result.stderr + assert marker.is_file(), result.stderr + assert marker.read_text() in result.stderr + assert not result.stdout + venv.cleaned() + + +def test_failed_stage2_is_cleaned(venv): + venv.replace_template_path("STAGE2_BOOTSTRAP", "missing-stage2.py") + result = venv.run("echo", "0") + assert result.returncode != 0 + venv.cleaned() + + +def test_verbose_mode_retains_runtime(launcher): + result = launcher.run( + "echo", + "0", + environment=dict(launcher.environment, RULES_PYTHON_BOOTSTRAP_VERBOSE="1"), + ) + assert result.returncode == 0, result.stderr + assert Path(json.loads(result.stdout)["executable"]).is_file() + if not launcher.shell: + assert "rules_python bootstrap: launching interpreter" in result.stderr + assert ( + "rules_python bootstrap: retaining workspace" in result.stderr + ) == launcher.temporary_runtime + if launcher.archive: + assert "rules_python bootstrap: extracting archive" in result.stderr + + +def test_helper_does_not_retain_application_descriptors(launcher): + read_fd, write_fd = os.pipe() + try: + with launcher.process( + "close_fds", str(launcher.root), str(write_fd), pass_fds=(write_fd,) + ) as process: + os.close(write_fd) + write_fd = -1 + wait_until( + lambda: (launcher.root / "ready").exists(), "application did not start" + ) + assert process.stdout is not None and process.stderr is not None + for fd in (read_fd, process.stdout.fileno(), process.stderr.fileno()): + assert select.select([fd], [], [], 5)[0], ( + "cleanup helper retained a descriptor" + ) + assert os.read(fd, 1) == b"" + assert process.poll() is None + (launcher.root / "finish").touch() + process.communicate(timeout=10) + assert process.returncode == 0 + finally: + os.close(read_fd) + if write_fd >= 0: + os.close(write_fd) + + +@contextlib.contextmanager +def terminal_session(launcher): + shell_pid, terminal = pty.fork() + if shell_pid == 0: + os.execve( + "/bin/bash", + ["/bin/bash", "--noprofile", "--norc", "--noediting", "-i"], + launcher.environment, + ) + output = bytearray() + finished = threading.Event() + + def collect_output(): + with contextlib.suppress(OSError): + while not finished.is_set(): + if select.select([terminal], [], [], 0.1)[0]: + chunk = os.read(terminal, 65536) + if not chunk: + return + output.extend(chunk) + + reader = threading.Thread(target=collect_output) + reader.start() + try: + yield shell_pid, terminal + finally: + # These fixtures have no application grandchildren. Detached cleanup + # helpers must observe their real parents exiting and finish themselves. + for marker in ("ready", "peer_ready"): + path = launcher.root / marker + if path.exists(): + with contextlib.suppress(ProcessLookupError): + pid = int(path.read_text()) + if os.getsid(pid) == shell_pid: + os.kill(pid, signal.SIGKILL) + with contextlib.suppress(ProcessLookupError): + os.kill(shell_pid, signal.SIGKILL) + finished.set() + reader.join(timeout=2) + os.close(terminal) + os.waitpid(shell_pid, 0) + print(output.decode(errors="replace")) + launcher.cleaned() + + +def send_command(terminal, command): + os.write(terminal, (shlex.join(command) + "\n").encode()) + + +@pytest.mark.parametrize("ignored_continue", [False, True]) +def test_foreground_input_and_job_control(launcher, ignored_continue): + with terminal_session(launcher) as (shell_pid, terminal): + command = [*launcher.command, "default", str(launcher.root)] + if ignored_continue: + command = [ + "/bin/bash", + "-c", + 'trap "" CONT; exec "$@"', + "ignored", + *command, + ] + send_command(terminal, command) + wait_until( + lambda: (launcher.root / "ready").exists(), + "terminal application did not start", + ) + app_pid = int((launcher.root / "ready").read_text()) + assert os.tcgetpgrp(terminal) == os.getpgid(app_pid) + + os.write(terminal, b"\x1a") # Ctrl-Z: stop the foreground job. + wait_until( + lambda: os.tcgetpgrp(terminal) == shell_pid, + "shell did not recover foreground", + ) + assert bool(list(launcher.scratch.iterdir())) == launcher.temporary_runtime + stopped = launcher.root / "background-stop" + # Wait in the owning shell for the new SIGTTIN stop. Sampling `ps` + # here can observe the old Ctrl-Z stop before `bg` has run. + os.write( + terminal, + ( + 'bg; wait %+; printf "%s\\n" "$?" > ' + shlex.quote(str(stopped)) + "\n" + ).encode(), + ) + wait_until( + lambda: stopped.exists() and bool(stopped.read_text()), + "background terminal reader did not stop", + ) + assert int(stopped.read_text()) == 128 + signal.SIGTTIN + os.write(terminal, b"fg\n") + wait_until( + lambda: os.tcgetpgrp(terminal) == os.getpgid(app_pid), + "fg did not restore the job", + ) + os.write(terminal, b"finish\n") + wait_until( + lambda: (launcher.root / "cleaned").exists(), "foreground input was lost" + ) + wait_until( + lambda: os.tcgetpgrp(terminal) == shell_pid, + "shell did not recover after exit", + ) + launcher.cleaned() + + +def test_terminal_ctrl_c_preserves_finally_input(launcher): + with terminal_session(launcher) as (shell_pid, terminal): + send_command(terminal, [*launcher.command, "interrupt", str(launcher.root)]) + wait_until( + lambda: (launcher.root / "ready").exists(), + "terminal application did not start", + ) + os.write(terminal, b"\x03") + wait_until( + lambda: (launcher.root / "stopping").exists(), "Ctrl-C did not reach Python" + ) + os.write(terminal, b"cleanup\n") + wait_until( + lambda: (launcher.root / "cleaned").exists(), + "Ctrl-C interrupted cleanup twice", + ) + wait_until( + lambda: os.tcgetpgrp(terminal) == shell_pid, + "shell did not recover foreground", + ) + + +def test_pipeline_peer_keeps_terminal_access(launcher): + # Keep the typed command below the terminal's canonical input line limit. + launcher.environment["CLEANUP_TERMINAL_PYTHON"] = sys.executable + launcher.environment["CLEANUP_TERMINAL_PEER"] = launcher.locate("PEER_RLOCATION") + with terminal_session(launcher) as (shell_pid, terminal): + application = shlex.join([*launcher.command, "pipeline", str(launcher.root)]) + peer = '"$CLEANUP_TERMINAL_PYTHON" "$CLEANUP_TERMINAL_PEER" ' + shlex.quote( + str(launcher.root) + ) + os.write(terminal, (application + " | " + peer + "\n").encode()) + wait_until( + lambda: (launcher.root / "ready").exists(), + "pipeline application did not start", + ) + wait_until( + lambda: (launcher.root / "peer_ready").exists(), + "pipeline peer did not start", + ) + app_pid = int((launcher.root / "ready").read_text()) + peer_pid = int((launcher.root / "peer_ready").read_text()) + assert os.getpgid(app_pid) == os.getpgid(peer_pid) == os.tcgetpgrp(terminal) + os.write(terminal, b"finish\n") + wait_until( + lambda: (launcher.root / "peer_done").exists(), + "pipeline peer lost terminal input", + ) + wait_until( + lambda: (launcher.root / "cleaned").exists(), + "pipeline application did not finish", + ) + wait_until( + lambda: os.tcgetpgrp(terminal) == shell_pid, + "shell did not recover after pipeline", + ) diff --git a/tests/bootstrap_impls/bootstrap_cleanup_watch_test.py b/tests/bootstrap_impls/bootstrap_cleanup_watch_test.py new file mode 100644 index 0000000000..cb21f3f5ba --- /dev/null +++ b/tests/bootstrap_impls/bootstrap_cleanup_watch_test.py @@ -0,0 +1,433 @@ +"""Exit-watch races and fallbacks, separate from generated launcher tests.""" + +import contextlib +import errno +import os +import runpy +import subprocess +import sys +import time + +import pytest + +from python.runfiles import runfiles + + +@pytest.fixture(name="helper") +def fixture_helper(): + location = runfiles.CreateOrRaise().Rlocation(os.environ["HELPER_RLOCATION"]) + assert location is not None + return location, runpy.run_path(location) + + +def test_parent_already_exited(helper, monkeypatch): + _, support = helper + + def exited(_pid): + raise ProcessLookupError(errno.ESRCH, "parent exited") + + monkeypatch.setattr(os, "pidfd_open", exited, raising=False) + with contextlib.ExitStack() as resources: + support["_arm_watch"](os.getppid(), resources)() + + +@pytest.mark.parametrize("error", [errno.ENOSYS, errno.EPERM, errno.EINVAL]) +def test_native_watch_unavailable_uses_conservative_pid_poll( + helper, monkeypatch, error +): + _, support = helper + + def unavailable(_pid): + raise OSError(error, "unavailable") + + monkeypatch.setattr(os, "pidfd_open", unavailable, raising=False) + monkeypatch.setattr(os, "readlink", lambda _path: str(os.getpid())) + monkeypatch.setattr(os, "open", lambda *args: unavailable(0)) + results = iter([None, PermissionError(), None, ProcessLookupError()]) + + def exists(_pid, _signal): + result = next(results) + if result is not None: + raise result + + monkeypatch.setattr(os, "kill", exists) + with contextlib.ExitStack() as resources: + support["_arm_watch"](os.getppid(), resources)() + + +def test_pidfd_readiness_and_descriptor_cleanup(helper, monkeypatch): + _, support = helper + read_fd, write_fd = os.pipe() + os.close(write_fd) + monkeypatch.setattr(os, "pidfd_open", lambda _pid: read_fd, raising=False) + with contextlib.ExitStack() as resources: + support["_arm_watch"](os.getppid(), resources)() + with pytest.raises(OSError): + os.fstat(read_fd) + + +@pytest.mark.parametrize( + "backend", + [ + pytest.param( + "pidfd", + marks=pytest.mark.skipif( + sys.platform != "linux" or not hasattr(os, "pidfd_open"), + reason="Linux pidfd contract", + ), + ), + pytest.param( + "proc", + marks=pytest.mark.skipif( + sys.platform != "linux", reason="Linux procfs contract" + ), + ), + pytest.param( + "kqueue", + marks=pytest.mark.skipif( + sys.platform != "darwin", reason="macOS kqueue contract" + ), + ), + ], +) +def test_native_exit_watch_waits_and_closes_descriptor(helper, backend): + path, _ = helper + driver = """ +import contextlib, errno, os, runpy, select, subprocess, sys, threading +support = runpy.run_path(sys.argv[1]) +backend = sys.argv[2] +descriptors = [] +with subprocess.Popen([sys.executable, '-c', 'import sys; sys.stdin.read()'], + stdin=subprocess.PIPE) as child: + # Check host support independently so unsupported kernels or namespaces + # are reported explicitly, instead of silently exercising a fallback. + try: + if backend == 'pidfd': + os.close(os.pidfd_open(child.pid)) + elif backend == 'proc': + if (os.readlink('/proc/self') != str(os.getpid()) or + os.readlink('/proc/self/ns/pid') != os.readlink('/proc/1/ns/pid')): + raise OSError(errno.ENOTSUP, 'procfs is from another namespace') + os.close(os.open('/proc/%d/stat' % child.pid, os.O_RDONLY)) + else: + with contextlib.closing(select.kqueue()) as queue: + queue.control([select.kevent(child.pid, filter=select.KQ_FILTER_PROC, + flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT, + fflags=select.KQ_NOTE_EXIT)], 0, 0) + except OSError as error: + if error.errno not in (errno.ENOSYS, errno.ENOTSUP, errno.EPERM, errno.EACCES): + raise + print(str(error), file=sys.stderr) + child.terminate() + child.wait(timeout=5) + sys.exit(77) + + def unexpected_fallback(*args): + raise AssertionError('native watch unexpectedly used a fallback') + + support['_arm_watch'].__globals__['_poll_pid'] = unexpected_fallback + if backend == 'proc': + arm = support['_arm_poll_watch'] + create = os.open + def open_proc(*args): + fd = create(*args) + descriptors.append(fd) + return fd + os.open = open_proc + else: + arm = support['_arm_watch'] + arm.__globals__['_arm_poll_watch'] = unexpected_fallback + if backend == 'pidfd': + create = os.pidfd_open + def open_pidfd(pid): + fd = create(pid) + descriptors.append(fd) + return fd + os.pidfd_open = open_pidfd + else: + create = select.kqueue + def open_queue(): + queue = create() + descriptors.append(queue.fileno()) + return queue + select.kqueue = open_queue + + finished = threading.Event() + errors = [] + with contextlib.ExitStack() as resources: + wait = arm(child.pid, resources) + assert len(descriptors) == 1, 'expected backend did not register a watch' + def observe(): + try: + wait() + except BaseException as error: + errors.append(error) + finally: + finished.set() + thread = threading.Thread(target=observe, daemon=True) + thread.start() + assert not finished.wait(.1), 'watch returned before application exit' + child.terminate() + child.wait(timeout=5) + assert finished.wait(5), 'watch did not observe application exit' + thread.join() + assert not errors, errors + try: + os.fstat(descriptors[0]) + except OSError as error: + assert error.errno == errno.EBADF + else: + raise AssertionError('watch descriptor remained open') +""" + result = subprocess.run( + [sys.executable, "-c", driver, path, backend], + capture_output=True, + text=True, + timeout=20, + ) + if result.returncode == 77: + pytest.skip(f"{backend} unavailable on this host: {result.stderr.strip()}") + assert result.returncode == 0, result.stderr + + +def test_unexpected_setup_error_is_reported(helper, monkeypatch): + _, support = helper + + def failure(_pid): + raise OSError(errno.EIO, "watch registration failed") + + monkeypatch.setattr(os, "pidfd_open", failure, raising=False) + with contextlib.ExitStack() as resources, pytest.raises( + OSError, match="watch registration failed" + ): + support["_arm_watch"](os.getppid(), resources) + + +@pytest.mark.parametrize("state", [b"Z", b"X"]) +def test_proc_watch_waits_for_original_process_reaping( + helper, tmp_path, monkeypatch, state +): + _, support = helper + path = tmp_path / "proc-stat" + path.write_bytes(b"") + fd = os.open(path, os.O_RDONLY) + monkeypatch.setattr(os, "readlink", lambda _path: str(os.getpid())) + monkeypatch.setattr(os, "open", lambda *_args: fd) + records = iter([b"123 (a process) name) S 1", state, "gone"]) + observed = [] + + def read(_fd, _length): + assert _fd == fd + value = next(records) + observed.append(value) + if value == "gone": + raise ProcessLookupError(errno.ESRCH, "original process exited") + if value in (b"Z", b"X"): + return b"123 (a process) name) " + value + b" 1" + return value + + monkeypatch.setattr(os, "read", read) + with contextlib.ExitStack() as resources: + support["_arm_poll_watch"](123, resources)() + assert observed[-1] == "gone", "zombie leader can still have live threads" + with pytest.raises(OSError): + os.fstat(fd) + + +def test_proc_from_another_pid_namespace_uses_pid_poll(helper, monkeypatch): + _, support = helper + monkeypatch.setattr( + os, + "readlink", + lambda path: ( + str(os.getpid()) + if path == "/proc/self" + else ("pid:[100]" if path == "/proc/self/ns/pid" else "pid:[200]") + ), + ) + checked = [] + support["_arm_poll_watch"].__globals__["_poll_pid"] = checked.append + with contextlib.ExitStack() as resources: + support["_arm_poll_watch"](123, resources)() + assert checked == [123] + + +@pytest.mark.parametrize("message", ["", "invalid", "closed_before_ready"]) +def test_failed_control_handshake_does_not_remove_live_runtime( + helper, tmp_path, message +): + path, _ = helper + runtime = tmp_path / "runtime" + runtime.mkdir() + driver = """ +import os, runpy, socket, sys, time +support = runpy.run_path(sys.argv[1]) +parent_pid = os.getpid() +parent, watcher = socket.socketpair() +child = os.fork() +if child == 0: + parent.close() + os.setsid() + fd = os.open(os.devnull, os.O_RDWR) + os.dup2(fd, 0) + os.close(fd) + support['_watch'](parent_pid, [sys.argv[2]], watcher) + os._exit(0) +watcher.close() +if sys.argv[3] != 'closed_before_ready': + assert parent.recv(1) == b'A' + if sys.argv[3]: + parent.sendall(sys.argv[3].encode()) +parent.close() +time.sleep(.2) +assert os.waitpid(child, os.WNOHANG) == (0, 0), 'watcher exited while parent lives' +assert os.path.isdir(sys.argv[2]), 'control failure removed a live runtime' +""" + result = subprocess.run( + [sys.executable, "-c", driver, path, str(runtime), message], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode == 0, result.stderr + deadline = time.monotonic() + 10 + while runtime.exists(): + assert time.monotonic() < deadline, "watcher did not clean after parent exit" + time.sleep(0.01) + + +@pytest.mark.parametrize("phase", ["early", "arming", "ready"]) +def test_real_parent_exits_before_or_after_readiness(helper, tmp_path, phase): + path, _ = helper + runtime = tmp_path / "runtime" + runtime.mkdir() + child = """ +import os, runpy, sys, time +path, parent, runtime, phase = sys.argv[1:] +support = runpy.run_path(path) +if phase == 'arming': + original = support['main'].__globals__['_arm_watch'] + def arm(pid, resources): + open(runtime + '/arming', 'w').close() + time.sleep(.2) + return original(pid, resources) + support['main'].__globals__['_arm_watch'] = arm +sys.argv = [path, parent, runtime] +sys.exit(support['main']()) +""" + driver = """ +import os, subprocess, sys, time +child = subprocess.Popen([ + sys.executable, '-I', '-S', '-c', sys.argv[1], sys.argv[2], str(os.getpid()), *sys.argv[3:] +]) +if sys.argv[4] == 'ready': + assert child.wait(timeout=10) == 0 + assert os.path.isdir(sys.argv[3]) +elif sys.argv[4] == 'arming': + deadline = time.monotonic() + 10 + while not os.path.isfile(sys.argv[3] + '/arming'): + assert time.monotonic() < deadline + time.sleep(.01) +sys.exit(17) +""" + result = subprocess.run( + [sys.executable, "-c", driver, child, path, str(runtime), phase], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode == 17, result.stderr + deadline = time.monotonic() + 10 + while runtime.exists(): + assert time.monotonic() < deadline, "orphaned helper did not remove runtime" + time.sleep(0.01) + + +@pytest.mark.parametrize("closed_fds", ["", "0", "2", "0,2"]) +@pytest.mark.parametrize("fallback", [False, True]) +def test_closed_stdio_and_descriptor_above_soft_limit( + helper, tmp_path, closed_fds, fallback +): + path, _ = helper + runtime = tmp_path / "runtime" + runtime.mkdir() + child = """ +import os, resource, runpy, select, sys +path, parent, runtime, closed_fds, fallback = sys.argv[1:] +resource.setrlimit(resource.RLIMIT_NOFILE, (64, resource.getrlimit(resource.RLIMIT_NOFILE)[1])) +for fd in closed_fds.split(','): + if fd: + os.close(int(fd)) +if fallback == 'True': + for module, name in ((os, 'pidfd_open'), (select, 'kqueue')): + if hasattr(module, name): + delattr(module, name) +sys.argv = [path, parent, runtime] +runpy.run_path(path, run_name='__main__') +""" + driver = """ +import os, select, subprocess, sys +read_fd, write_fd = os.pipe() +os.dup2(write_fd, 200) +os.close(write_fd) +child = subprocess.Popen( + [sys.executable, '-I', '-S', '-c', sys.argv[1], sys.argv[2], str(os.getpid()), *sys.argv[3:]], + pass_fds=(200,), stdout=subprocess.PIPE, +) +os.close(200) +assert child.wait(timeout=10) == 0, 'helper failed to arm the watcher' +assert select.select([read_fd], [], [], 5)[0], 'helper retained fd200 above its soft limit' +assert os.read(read_fd, 1) == b'' +assert os.path.isdir(sys.argv[3]), 'live parent lost its runtime' +""" + result = subprocess.run( + [ + sys.executable, + "-c", + driver, + child, + path, + str(runtime), + closed_fds, + str(fallback), + ], + capture_output=True, + text=True, + timeout=15, + check=False, + ) + assert result.returncode == 0, result.stderr + deadline = time.monotonic() + 10 + while runtime.exists(): + assert time.monotonic() < deadline, "runtime leaked with closed descriptors" + time.sleep(0.01) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux subreaper contract") +def test_subreaper_adopts_watcher_until_application_exit(helper, tmp_path): + path, _ = helper + runtime = tmp_path / "runtime" + runtime.mkdir() + driver = """ +import ctypes, os, subprocess, sys +libc = ctypes.CDLL(None, use_errno=True) +assert libc.prctl(36, 1, 0, 0, 0) == 0 # PR_SET_CHILD_SUBREAPER +subprocess.run([sys.executable, '-I', '-S', sys.argv[1], str(os.getpid()), sys.argv[2]], check=True) +# The direct helper has exited, but a subreaper adopts its live watcher. A +# blocking wait-for-all would deadlock; users need persistent roots here. +assert os.waitpid(-1, os.WNOHANG) == (0, 0) +assert os.path.isdir(sys.argv[2]) +""" + result = subprocess.run( + [sys.executable, "-c", driver, path, str(runtime)], + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 0, result.stderr + deadline = time.monotonic() + 10 + while runtime.exists(): + assert time.monotonic() < deadline, "watcher did not clean after subreaper exit" + time.sleep(0.01) diff --git a/tests/bootstrap_impls/cleanup_fail.py b/tests/bootstrap_impls/cleanup_fail.py new file mode 100644 index 0000000000..41d6a14992 --- /dev/null +++ b/tests/bootstrap_impls/cleanup_fail.py @@ -0,0 +1,15 @@ +"""Exercise failure and cancellation before the helper's readiness message.""" + +import os +import time +from pathlib import Path + +if root := os.environ.get("CLEANUP_HELPER_STARTUP_PROBE"): + parent = os.getppid() + os.setsid() + Path(root, "helper_starting").write_text(str(os.getpid())) + while os.getppid() == parent: + time.sleep(0.01) + Path(root, "helper_finished").touch() + +raise SystemExit(23) diff --git a/tests/bootstrap_impls/cleanup_probe.py b/tests/bootstrap_impls/cleanup_probe.py new file mode 100644 index 0000000000..0e1c3514b6 --- /dev/null +++ b/tests/bootstrap_impls/cleanup_probe.py @@ -0,0 +1,100 @@ +"""Application used to observe temporary-runtime launchers from the outside.""" + +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + + +def main(): + mode = sys.argv[1] + if mode == "nested": + child = subprocess.run( + sys.argv[2:], + input=sys.stdin.read(), + text=True, + capture_output=True, + check=False, + ) + print(child.stdout, end="") + print(child.stderr, end="", file=sys.stderr) + return child.returncode + if mode == "wait_children": + # A cleanup watcher left as our child would make this block forever. + try: + while True: + os.waitpid(-1, 0) + except ChildProcessError: + return 0 + if mode == "echo": + print( + json.dumps( + { + "argv": sys.argv[3:], + "stdin": sys.stdin.read(), + "executable": sys.executable, + "pid": os.getpid(), + "pgid": os.getpgrp(), + "ignored": [ + signal.getsignal(value) == signal.SIG_IGN + for value in (signal.SIGINT, signal.SIGQUIT) + ], + "handler": getattr(signal.getsignal(signal.SIGINT), "__name__", ""), + "xoptions": sys._xoptions, + "cwd": os.getcwd(), + "runfiles": os.environ.get("RUNFILES_DIR"), + "additional_args": os.environ.get( + "RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS" + ), + "safe_path": os.environ.get("PYTHONSAFEPATH"), + } + ) + ) + return int(sys.argv[2]) + + root = Path(sys.argv[2]) + if mode == "close_fds": + for fd in (0, 1, 2, int(sys.argv[3])): + os.close(fd) + (root / "ready").touch() + while not (root / "finish").exists(): + time.sleep(0.01) + return 0 + + if mode == "handled": + + def stop(signum, _frame): + for value in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT): + signal.signal(value, signal.SIG_IGN) + raise SystemExit(128 + signum) + + for value in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT): + signal.signal(value, stop) + + try: + # Announce readiness only once the cleanup handler is active and the + # complete PID can be read. Signals may arrive immediately afterward. + (root / "ready.tmp").write_text(str(os.getpid())) + (root / "ready.tmp").replace(root / "ready") + if mode == "pipeline": + while not (root / "peer_done").exists(): + time.sleep(0.01) + return 0 + if sys.stdin.readline().strip() == "finish": + return 0 + raise AssertionError("expected cancellation or foreground input") + finally: + (root / "stopping").touch() + if mode in ("handled", "interrupt"): + assert sys.stdin.readline().strip() == "cleanup" + assert Path(sys.executable).is_file(), ( + "runtime removed before application cleanup" + ) + (root / "cleaned").touch() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/bootstrap_impls/cleanup_python_wrapper.sh b/tests/bootstrap_impls/cleanup_python_wrapper.sh new file mode 100755 index 0000000000..aa83375fd9 --- /dev/null +++ b/tests/bootstrap_impls/cleanup_python_wrapper.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# A system-runtime wrapper can resolve to an interpreter with a different name. +IFS= read -r data < "$RUNFILES_DIR/$CLEANUP_WRAPPER_DATA" +[[ "$data" == "wrapper runfile" ]] || exit 91 +exec "${CLEANUP_REAL_INTERPRETER:-python3}" "$@" diff --git a/tests/bootstrap_impls/cleanup_sitecustomize.py b/tests/bootstrap_impls/cleanup_sitecustomize.py new file mode 100644 index 0000000000..6bf62c6197 --- /dev/null +++ b/tests/bootstrap_impls/cleanup_sitecustomize.py @@ -0,0 +1,18 @@ +"""User startup hook, also used to pause before stage two starts.""" + +import os +import signal +import time +from pathlib import Path + + +def user_interrupt(_signum, _frame): + raise SystemExit(19) + + +signal.signal(signal.SIGINT, user_interrupt) +if directory := os.environ.get("CLEANUP_STARTUP_PROBE"): + root = Path(directory) + (root / "starting").touch() + while not (root / "continue").exists(): + time.sleep(0.01) diff --git a/tests/bootstrap_impls/cleanup_terminal_peer.py b/tests/bootstrap_impls/cleanup_terminal_peer.py new file mode 100644 index 0000000000..3a0383085a --- /dev/null +++ b/tests/bootstrap_impls/cleanup_terminal_peer.py @@ -0,0 +1,11 @@ +"""A pipeline peer that needs its original foreground terminal group.""" + +import os +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +with open("/dev/tty") as terminal: + (root / "peer_ready").write_text(str(os.getpid())) + assert terminal.readline().strip() == "finish" +(root / "peer_done").touch() diff --git a/tests/bootstrap_impls/cleanup_wrapper_data.txt b/tests/bootstrap_impls/cleanup_wrapper_data.txt new file mode 100644 index 0000000000..ba00c90a5d --- /dev/null +++ b/tests/bootstrap_impls/cleanup_wrapper_data.txt @@ -0,0 +1 @@ +wrapper runfile diff --git a/tests/bootstrap_impls/windows_console_fixture.py b/tests/bootstrap_impls/windows_console_fixture.py new file mode 100644 index 0000000000..3d203910e4 --- /dev/null +++ b/tests/bootstrap_impls/windows_console_fixture.py @@ -0,0 +1,135 @@ +"""An isolated native console for observing Windows application handoff.""" + +import ctypes +import os +import signal +import subprocess +import sys +import threading +import time +from pathlib import Path + + +def wait_for(predicate): + deadline = time.monotonic() + 10 + while not predicate(): + if time.monotonic() >= deadline: + raise AssertionError("console fixture did not reach its barrier") + time.sleep(0.01) + + +def application(root, mode, workspace): + if mode == "high_bit": + ctypes.windll.kernel32.ExitProcess(0xC000013A) + count = 0 + + def interrupted(_signum, _frame): + nonlocal count + count += 1 + (root / "interrupts").write_text(str(count)) + if mode == "handled": + raise SystemExit(23) + + if mode != "inherited_ignore": + signal.signal(signal.SIGINT, interrupted) + try: + (root / "ready").touch() + wait_for(lambda: (root / "release").exists()) + return 17 if mode == "ignored" else 29 + finally: + (root / "stopping").touch() + wait_for(lambda: (root / "release").exists()) + assert workspace.is_dir(), "workspace removed before application finally" + (root / "finished").touch() + + +def controller(root, mode): + from python.private._rules_python_bootstrap import model, process, storage + + # Test runners may inherit an ignored console; establish this fixture's + # disposition explicitly before creating its application child. + assert ctypes.windll.kernel32.SetConsoleCtrlHandler( + None, mode == "inherited_ignore" + ) + if mode == "inherited_ignore": + signal.signal(signal.SIGINT, signal.SIG_IGN) + else: + signal.signal(signal.SIGINT, signal.default_int_handler) + waiting = threading.Event() + children = [] + failures = [] + original_popen = subprocess.Popen + + def spawn(*args, **kwargs): + child = original_popen(*args, **kwargs) + children.append(child) + original_wait = child.wait + + def wait(*args, **kwargs): + # This barrier is after execute's registration-time cancellation + # check. Child readiness alone does not establish that handoff. + waiting.set() + return original_wait(*args, **kwargs) + + child.wait = wait + return child + + subprocess.Popen = spawn + with process.Cancellation() as cancellation, storage.Workspace( + directory=root + ) as workspace: + owned = Path(workspace.allocate()) + (root / "workspace").write_text(str(owned)) + + def interrupt(): + try: + assert waiting.wait(10), "parent never entered child wait" + wait_for(lambda: (root / "ready").exists()) + # Broadcast only within this fixture's CREATE_NEW_CONSOLE. + # CTRL_C_EVENT cannot be scoped to a nonzero process-group ID. + assert ctypes.windll.kernel32.GenerateConsoleCtrlEvent(0, 0) + if mode == "handled": + wait_for(lambda: (root / "stopping").exists()) + elif mode == "ignored": + wait_for(lambda: (root / "interrupts").exists()) + time.sleep(0.2) + assert children[0].poll() is None, "parent killed application cleanup" + assert owned.is_dir(), "parent removed live application workspace" + if mode != "inherited_ignore": + assert (root / "interrupts").read_text() == "1" + else: + assert not (root / "interrupts").exists() + except BaseException as error: + failures.append(error) + finally: + (root / "release").touch() + + thread = threading.Thread(target=interrupt) if mode != "high_bit" else None + if thread is not None: + thread.start() + command = model.Invocation( + sys.executable, + (sys.executable, __file__, "application", str(root), mode, str(owned)), + dict(os.environ), + None, + ) + try: + status = process.execute(command, workspace, "unused", cancellation) + finally: + if thread is not None: + thread.join(12) + assert not thread.is_alive(), "console controller did not finish" + if failures: + raise failures[0] + assert not owned.exists(), "workspace survived application exit" + if mode != "high_bit": + assert (root / "finished").exists() + return status + + +if __name__ == "__main__": + role, directory, mode = sys.argv[1:4] + root = Path(directory) + if role == "application": + sys.exit(application(root, mode, Path(sys.argv[4]))) + sys.exit(controller(root, mode)) diff --git a/tests/py_zipapp/BUILD.bazel b/tests/py_zipapp/BUILD.bazel index e0f878cbe1..01acdc41ba 100644 --- a/tests/py_zipapp/BUILD.bazel +++ b/tests/py_zipapp/BUILD.bazel @@ -4,6 +4,8 @@ load("//python:py_library.bzl", "py_library") load("//python:py_test.bzl", "py_test") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility load("//python/zipapp:py_zipapp_binary.bzl", "py_zipapp_binary") +load("//python/zipapp:py_zipapp_test.bzl", "py_zipapp_test") +load("//tests/bootstrap_impls:application_tests.bzl", "public_executable") py_binary( name = "venv_bin", @@ -27,6 +29,21 @@ py_zipapp_binary( binary = ":venv_bin", ) +py_zipapp_test( + name = "application_zipapp_test", + binary = ":venv_bin", +) + +public_executable( + name = "public_only_binary", + binary = ":venv_bin", +) + +py_zipapp_test( + name = "public_provider_zipapp_test", + binary = ":public_only_binary", +) + py_test( name = "venv_zipapp_test", srcs = ["venv_zipapp_test.py"], diff --git a/tests/py_zipapp/system_python_zipapp_test.py b/tests/py_zipapp/system_python_zipapp_test.py index 7c3e2deeaf..ac127ac757 100644 --- a/tests/py_zipapp/system_python_zipapp_test.py +++ b/tests/py_zipapp/system_python_zipapp_test.py @@ -1,18 +1,29 @@ import os import subprocess +import sys import unittest class SystemPythonZipAppTest(unittest.TestCase): def test_zipapp_runnable(self): zipapp_path = os.environ["TEST_ZIPAPP"] + environment = dict(os.environ) + if os.name == "nt": + # Supply the test runtime as the system interpreter for the ZIP. + environment["PATH"] = ( + os.path.dirname(sys.executable) + + os.pathsep + + environment.get("PATH", "") + ) self.assertTrue(os.path.exists(zipapp_path)) self.assertTrue(os.path.isfile(zipapp_path)) try: output = ( - subprocess.check_output([zipapp_path], stderr=subprocess.STDOUT) + subprocess.check_output( + [zipapp_path], stderr=subprocess.STDOUT, env=environment + ) .decode("utf-8") .strip() ) diff --git a/tests/py_zipapp/venv_zipapp_test.py b/tests/py_zipapp/venv_zipapp_test.py index bd26d533a3..fe9cb8f11b 100644 --- a/tests/py_zipapp/venv_zipapp_test.py +++ b/tests/py_zipapp/venv_zipapp_test.py @@ -1,17 +1,40 @@ import contextlib import os import subprocess +import sys +import tempfile import unittest import zipfile class PyZipAppTest(unittest.TestCase): def test_zipapp_runnable(self): + self.assertZipappRuns() + + def test_persistent_zipapp_runs_after_publication(self): + with tempfile.TemporaryDirectory(dir=os.environ["TEST_TMPDIR"]) as cache: + for _ in range(2): + self.assertZipappRuns(cache=cache) + + def assertZipappRuns(self, cache=None): zipapp_path = os.environ["TEST_ZIPAPP"] + environment = dict(os.environ) + environment.pop("RULES_PYTHON_EXTRACT_ROOT", None) + if cache is not None: + environment["RULES_PYTHON_EXTRACT_ROOT"] = cache + if os.name == "nt": + # The native ZIP launcher needs a Python to open the archive. + environment["PATH"] = ( + os.path.dirname(sys.executable) + + os.pathsep + + environment.get("PATH", "") + ) try: output = ( - subprocess.check_output([zipapp_path], stderr=subprocess.STDOUT) + subprocess.check_output( + [zipapp_path], stderr=subprocess.STDOUT, env=environment + ) .decode("utf-8") .strip() ) diff --git a/tests/tools/zipapp/zip_main_maker_test.py b/tests/tools/zipapp/zip_main_maker_test.py index c27ff88e33..172802df29 100644 --- a/tests/tools/zipapp/zip_main_maker_test.py +++ b/tests/tools/zipapp/zip_main_maker_test.py @@ -1,90 +1,75 @@ -import hashlib -import os - -from tools.zipapp import zip_main_maker - - -def test_creates_zip_main(tmp_path, monkeypatch): - temp_dir = str(tmp_path) - template_path = os.path.join(temp_dir, "template.py") - with open(template_path, "w", encoding="utf-8") as f: - f.write("hash=%APP_HASH%\nfoo=%FOO%\n") - - output_path = os.path.join(temp_dir, "output.py") - - file1_path = os.path.join(temp_dir, "file1.txt") - with open(file1_path, "wb") as f: - f.write(b"content1") - - file2_path = os.path.join(temp_dir, "file2.txt") - with open(file2_path, "wb") as f: - f.write(b"content2") +"""Cache identities follow runtime contents and bootstrap configuration.""" - # Add a symlink to test symlink hashing - symlink_path = os.path.join(temp_dir, "symlink.txt") - os.symlink(file1_path, symlink_path) - - manifest_path = os.path.join(temp_dir, "manifest.txt") - with open(manifest_path, "w", encoding="utf-8") as f: - f.write(f"rf-file|0|file1.txt|{file1_path}\n") - f.write(f"rf-file|0|file2.txt|{file2_path}\n") - f.write(f"rf-symlink|1|symlink.txt|{symlink_path}\n") - f.write("rf-empty|empty_file.txt\n") - - argv = [ - "zip_main_maker.py", - "--template", - template_path, - "--output", - output_path, - "--substitution", - "%FOO%=bar", - "--hash_files_manifest", - manifest_path, - ] - - monkeypatch.setattr("sys.argv", argv) - zip_main_maker.main() - - # Calculate expected hash - h = hashlib.sha256() - line1 = f"rf-file|0|file1.txt|{file1_path}" - line2 = f"rf-file|0|file2.txt|{file2_path}" - line3 = f"rf-symlink|1|symlink.txt|{symlink_path}" - line4 = "rf-empty|empty_file.txt" - - # Sort lines like the program does - lines = sorted([line1, line2, line3, line4]) - for line in lines: - parts = line.split("|") - if len(parts) > 1: - _, rest = line.split("|", 1) - h.update(rest.encode("utf-8")) - else: - h.update(line.encode("utf-8")) - - type_ = parts[0] - if type_ == "rf-empty": - continue - if len(parts) >= 4: - is_symlink_str = parts[1] - path = parts[-1] - if not path: - continue - if is_symlink_str == "-1": - is_symlink = not os.path.exists(path) - else: - is_symlink = is_symlink_str == "1" +import os - if is_symlink: - h.update(os.readlink(path).encode("utf-8")) - else: - with open(path, "rb") as f: - h.update(f.read()) +import pytest - expected_hash = h.hexdigest() +from tools.zipapp import zip_main_maker - with open(output_path, "r", encoding="utf-8") as f: - content = f.read() - assert content == f"hash={expected_hash}\nfoo=bar\n" +@pytest.fixture(name="generate") +def fixture_generate(tmp_path, monkeypatch): + template = tmp_path / "template" + template.write_text("%APP_HASH%\n%OPTIONS%\n") + payload = tmp_path / "payload" + payload.write_text("application") + symlink = tmp_path / "symlink" + symlink.symlink_to(payload) + manifest = tmp_path / "manifest" + manifest.write_text( + f"rf-file|0|app|{payload}\nrf-symlink|1|link|{symlink}\nrf-empty|empty\n" + ) + output = tmp_path / "output" + + def generate(options="-Xoriginal"): + monkeypatch.setattr( + "sys.argv", + [ + "zip_main_maker", + "--template", + str(template), + "--output", + str(output), + "--substitution", + "%OPTIONS%=" + options, + "--hash_files_manifest", + str(manifest), + ], + ) + zip_main_maker.main() + digest, actual = output.read_text().splitlines()[:2] + assert len(digest) == 64 + assert actual == options + return digest + + return generate, template, payload, symlink, manifest + + +def test_hash_is_stable_across_runs_and_manifest_order(generate): + run, _, _, _, manifest = generate + first = run() + assert run() == first + manifest.write_text("\n".join(reversed(manifest.read_text().splitlines())) + "\n") + assert run() == first + + +@pytest.mark.parametrize( + "changed", ["content", "link", "layout", "template", "options", "mode"] +) +def test_hash_invalidates_every_runtime_input(generate, changed): + run, template, payload, symlink, manifest = generate + original = run() + if changed == "content": + payload.write_text("changed application") + elif changed == "link": + symlink.unlink() + symlink.symlink_to(payload.parent / "different") + elif changed == "layout": + manifest.write_text(manifest.read_text() + "rf-empty|new/path\n") + elif changed == "template": + template.write_text(template.read_text() + "new lifecycle code\n") + elif changed == "mode": + if os.name == "nt": + pytest.skip("Windows chmod does not expose executable mode bits") + payload.chmod(payload.stat().st_mode ^ 0o100) + assert run("-Xchanged" if changed == "options" else "-Xoriginal") != original diff --git a/tools/zipapp/exe_zip_maker.py b/tools/zipapp/exe_zip_maker.py index 29391c86da..15bc40c62f 100644 --- a/tools/zipapp/exe_zip_maker.py +++ b/tools/zipapp/exe_zip_maker.py @@ -1,8 +1,10 @@ import hashlib +import json import os import shutil import stat import sys +import zipfile BLOCK_SIZE = 256 * 1024 @@ -18,6 +20,12 @@ def create_exe_zip(preamble_path, zip_path, output_path): preamble_content = f.read() preamble_content = preamble_content.replace(b"%ZIP_HASH%", zip_hash.encode("utf-8")) + if b"%APP_HASH%" in preamble_content: + with zipfile.ZipFile(zip_path) as archive: + metadata = json.loads(archive.read("_rules_python_archive.json")) + preamble_content = preamble_content.replace( + b"%APP_HASH%", metadata["identity"].encode("ascii") + ) with open(output_path, "wb") as out_f: out_f.write(preamble_content) diff --git a/tools/zipapp/zip_main_maker.py b/tools/zipapp/zip_main_maker.py index ae112ffc66..3e755c105c 100644 --- a/tools/zipapp/zip_main_maker.py +++ b/tools/zipapp/zip_main_maker.py @@ -7,6 +7,7 @@ import argparse import hashlib +import json import os BLOCK_SIZE = 256 * 1024 @@ -17,6 +18,8 @@ def create_parser() -> argparse.ArgumentParser: parser.add_argument("--template", required=True) parser.add_argument("--output", required=True) parser.add_argument("--substitution", action="append", default=[]) + parser.add_argument("--metadata") + parser.add_argument("--metadata-output") parser.add_argument( "--hash_files_manifest", required=True, @@ -34,7 +37,7 @@ def compute_inputs_hash(manifest_path: str) -> str: # content. for line in sorted(manifest_lines): type_, _, rest = line.partition("|") - h.update(rest.encode("utf-8")) + h.update(line.encode("utf-8") + b"\0") parts = rest.split("|") if type_ == "rf-empty": @@ -55,6 +58,8 @@ def compute_inputs_hash(manifest_path: str) -> str: if is_symlink: h.update(os.readlink(path).encode("utf-8")) else: + # zipper preserves these mode bits, including executable permissions. + h.update((os.stat(path).st_mode & 0xFFFF).to_bytes(2, "big")) with open(path, "rb") as f: while True: chunk = f.read(BLOCK_SIZE) @@ -66,13 +71,13 @@ def compute_inputs_hash(manifest_path: str) -> str: def expand_template(template_path: str, output_path: str, substitutions: dict) -> None: - with open(template_path, "r", encoding="utf-8") as f: + with open(template_path, "r", encoding="utf-8", newline="") as f: content = f.read() for key, val in substitutions.items(): content = content.replace(key, val) - with open(output_path, "w", encoding="utf-8") as f: + with open(output_path, "w", encoding="utf-8", newline="") as f: f.write(content) @@ -80,13 +85,30 @@ def main(): parser = create_parser() args = parser.parse_args() - app_hash = compute_inputs_hash(args.hash_files_manifest) - - substitutions = {"%APP_HASH%": app_hash} + substitutions = {} for s in args.substitution: key, val = s.split("=", 1) substitutions[key] = val + # A completed cache must also change when startup code or interpreter + # options change, even if the application runfiles are identical. + digest = hashlib.sha256( + compute_inputs_hash(args.hash_files_manifest).encode("ascii") + ) + with open(args.template, "rb") as source: + digest.update(source.read()) + digest.update(json.dumps(substitutions, sort_keys=True).encode("utf-8")) + metadata = json.loads(args.metadata) if args.metadata else None + if metadata is not None: + digest.update(json.dumps(metadata, sort_keys=True).encode("utf-8")) + substitutions["%APP_HASH%"] = digest.hexdigest() + if metadata is not None: + metadata["identity"] = digest.hexdigest() + contents = json.dumps(metadata, sort_keys=True) + substitutions["%archive_metadata%"] = repr(contents) + with open(args.metadata_output, "w", encoding="utf-8") as output: + output.write(contents) + expand_template(args.template, args.output, substitutions)