fix(build): resume interrupted test-data checkouts

AGENTS.md

@@ -125,6 +125,8 @@ cmake --build cmake-build-relwithdebinfo --target translate  # CLI: file to HTML
   into `test/data/`. It is off by default because the data is several
   gigabytes and `odr_test` builds without it. The `update_test_data` target
   moves existing checkouts onto the pins. Two repositories are private.
+  Data stays in the source tree for shared builds and reference-output updates.
+  It is not a submodule: SwiftPM would fetch it in every consumer checkout.
 - pugixml is built with `PUGIXML_COMPACT`. The odf, ooxml, svg and xml engines
   keep the parsed DOM as their backing store, so the node size is the document
   size. The define changes the ABI, and a mismatch between translation units

cmake/test_data.cmake

@@ -1,93 +1,44 @@
-# Fetches the test-data repositories pinned in `test/data.cmake` into
-# `test/data/`. They were git submodules until this file existed.
+# Fetch pinned test data. Configure leaves existing checkouts untouched,
+# because during a reference-output regeneration the local revision is ahead
+# of the pin on purpose; ODR_TEST_DATA_UPDATE moves clean checkouts to their
+# pins.
 #
-# Why they stopped being submodules: SwiftPM runs `git submodule update --init
-# --recursive` on a package repository at *every consumer's* checkout, so the
-# root `Package.swift` that ships the Apple framework would have made everyone
-# clone 4.5 GB of test data — and then fail outright on the two private
-# remotes. Fetching here keeps that cost on the people who run the tests.
-#
-# The checkouts live in the source tree, not the build tree, on purpose: the
-# path is baked into `test_info.cpp`, the three usual build directories would
-# otherwise hold three copies, and `reference-output/*` has to stay a working
-# copy the regeneration workflow can commit from.
-#
-# Two modes:
-#
-#   include()   configure time. Clones what is missing and reports what has
-#               drifted. Never moves an existing checkout — during a
-#               reference-output regeneration the local revision is ahead of
-#               the pin on purpose, and silently resetting it would throw the
-#               work away.
-#   cmake -P    the `update_test_data` target. Moves clean checkouts onto their
-#               pinned revisions.
-#
-# This shells out to plain `git`, so authentication is git's business and not
-# CMake's: the two private repositories need whatever your git is already set
-# up with. A credential helper (`gh auth setup-git` is the one-liner, and what
-# CI does), or ssh, in which case:
+# This runs plain `git`, so the two private repositories need whatever git is
+# set up with: a credential helper (`gh auth setup-git`, as CI does), or ssh:
 #
 #   git config --global url."git@github.com:".insteadOf "https://github.com/"
 
-# Only in script mode: calling this from an include() would reset the policy
-# defaults of the including directory.
+# Only in script mode: from an include() this would reset the policy defaults
+# of the including directory.
 if (CMAKE_SCRIPT_MODE_FILE)
     cmake_minimum_required(VERSION 3.15)
 endif ()
 
 find_package(Git REQUIRED)
 
-# Both overridable, so a downstream can keep the data elsewhere or pin a
-# different set.
 if (NOT DEFINED ODR_TEST_DATA_ROOT)
     set(ODR_TEST_DATA_ROOT "${CMAKE_CURRENT_LIST_DIR}/../test/data")
 endif ()
 if (NOT DEFINED ODR_TEST_DATA_PINS)
     set(ODR_TEST_DATA_PINS "${CMAKE_CURRENT_LIST_DIR}/../test/data.cmake")
 endif ()
 
-# Clone at a single revision. `--depth 1` of a bare sha needs the server to
-# allow it; GitHub does, but a mirror may not, so fall back to a full fetch
-# rather than failing.
-function(odr_test_data_clone directory url revision)
-    message(STATUS "test data: cloning ${url} at ${revision}")
-
-    file(MAKE_DIRECTORY "${directory}")
-    execute_process(
-            COMMAND "${GIT_EXECUTABLE}" init --quiet
-            COMMAND_ERROR_IS_FATAL ANY
-            WORKING_DIRECTORY "${directory}")
-    execute_process(
-            COMMAND "${GIT_EXECUTABLE}" remote add origin "${url}"
-            COMMAND_ERROR_IS_FATAL ANY
-            WORKING_DIRECTORY "${directory}")
-
+function(odr_test_data_git directory)
     execute_process(
-            COMMAND "${GIT_EXECUTABLE}" fetch --depth 1 origin "${revision}"
-            RESULT_VARIABLE shallow
+            COMMAND "${GIT_EXECUTABLE}" ${ARGN}
+            RESULT_VARIABLE result
+            OUTPUT_VARIABLE output
+            OUTPUT_STRIP_TRAILING_WHITESPACE
             WORKING_DIRECTORY "${directory}")
-    if (NOT shallow EQUAL 0)
-        execute_process(
-                COMMAND "${GIT_EXECUTABLE}" fetch origin
-                COMMAND_ERROR_IS_FATAL ANY
-                WORKING_DIRECTORY "${directory}")
+    if (NOT "${result}" STREQUAL "0")
+        message(FATAL_ERROR "Git failed in ${directory}: ${ARGN} (${result})")
     endif ()
-
-    execute_process(
-            COMMAND "${GIT_EXECUTABLE}" checkout --quiet --detach "${revision}"
-            COMMAND_ERROR_IS_FATAL ANY
-            WORKING_DIRECTORY "${directory}")
+    set(ODR_TEST_DATA_GIT_OUTPUT "${output}" PARENT_SCOPE)
 endfunction()
 
-# Move an existing clean checkout onto `revision`. Refuses on a dirty tree.
 function(odr_test_data_update directory revision)
-    execute_process(
-            COMMAND "${GIT_EXECUTABLE}" status --porcelain
-            OUTPUT_VARIABLE dirty
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            COMMAND_ERROR_IS_FATAL ANY
-            WORKING_DIRECTORY "${directory}")
-    if (NOT dirty STREQUAL "")
+    odr_test_data_git("${directory}" status --porcelain)
+    if (NOT ODR_TEST_DATA_GIT_OUTPUT STREQUAL "")
         message(FATAL_ERROR
                 "${directory} has uncommitted changes. Commit or stash them "
                 "before updating the test data.")
@@ -97,25 +48,33 @@ function(odr_test_data_update directory revision)
             COMMAND "${GIT_EXECUTABLE}" fetch --depth 1 origin "${revision}"
             RESULT_VARIABLE shallow
             WORKING_DIRECTORY "${directory}")
-    if (NOT shallow EQUAL 0)
-        execute_process(
-                COMMAND "${GIT_EXECUTABLE}" fetch origin
-                COMMAND_ERROR_IS_FATAL ANY
-                WORKING_DIRECTORY "${directory}")
+    if (NOT "${shallow}" STREQUAL "0")
+        # Some mirrors do not allow shallow fetches of a bare SHA.
+        odr_test_data_git("${directory}" fetch origin)
     endif ()
 
     message(STATUS "test data: ${directory} -> ${revision}")
+    odr_test_data_git("${directory}" checkout --quiet --detach "${revision}")
+endfunction()
+
+function(odr_test_data_clone directory url revision)
+    message(STATUS "test data: initializing ${url} at ${revision}")
+    file(MAKE_DIRECTORY "${directory}")
+    if (NOT EXISTS "${directory}/.git")
+        odr_test_data_git("${directory}" init --quiet)
+    endif ()
     execute_process(
-            COMMAND "${GIT_EXECUTABLE}" checkout --quiet --detach "${revision}"
-            COMMAND_ERROR_IS_FATAL ANY
+            COMMAND "${GIT_EXECUTABLE}" remote get-url origin
+            RESULT_VARIABLE origin
+            OUTPUT_QUIET ERROR_QUIET
             WORKING_DIRECTORY "${directory}")
+    if (NOT "${origin}" STREQUAL "0")
+        odr_test_data_git("${directory}" remote add origin "${url}")
+    endif ()
+    odr_test_data_update("${directory}" "${revision}")
 endfunction()
 
-# One test-data repository. Called from `test/data.cmake`.
-#
-#   PATH      below `test/data`, also the name the tests refer to it by
-#   URL       clone url
-#   REVISION  the pinned commit; advance it here after regenerating output
+# PATH is relative to the data root; URL and REVISION identify its pinned source.
 function(odr_test_data)
     cmake_parse_arguments(PARSE_ARGV 0 ARG "" "PATH;URL;REVISION" "")
     set(directory "${ODR_TEST_DATA_ROOT}/${ARG_PATH}")
@@ -128,11 +87,16 @@ function(odr_test_data)
     endif ()
 
     execute_process(
-            COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD
+            COMMAND "${GIT_EXECUTABLE}" rev-parse --verify HEAD
+            RESULT_VARIABLE resolved
             OUTPUT_VARIABLE head
             OUTPUT_STRIP_TRAILING_WHITESPACE
-            COMMAND_ERROR_IS_FATAL ANY
+            ERROR_QUIET
             WORKING_DIRECTORY "${directory}")
+    if (NOT "${resolved}" STREQUAL "0")
+        odr_test_data_clone("${directory}" "${ARG_URL}" "${ARG_REVISION}")
+        return()
+    endif ()
     if ("${head}" STREQUAL "${ARG_REVISION}")
         return()
     endif ()

scripts/tests/test_test_data.py

@@ -0,0 +1,74 @@
+import shutil
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+@unittest.skipUnless(shutil.which("cmake") and shutil.which("git"), "needs CMake and Git")
+class TestDataCheckoutTest(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory(prefix="odr test data ")
+        self.addCleanup(self.temporary.cleanup)
+        self.root = Path(self.temporary.name)
+        self.remote = self.root / "remote"
+        self.remote.mkdir()
+        self.git(self.remote, "init", "--quiet")
+        self.git(self.remote, "config", "user.email", "test@example.invalid")
+        self.git(self.remote, "config", "user.name", "Test")
+        self.git(self.remote, "config", "commit.gpgsign", "false")
+        self.git(self.remote, "config", "core.hooksPath", str(self.root / "no-hooks"))
+        self.revisions = []
+        for value in ("first", "second"):
+            (self.remote / "fixture").write_text(value)
+            self.git(self.remote, "add", "fixture")
+            self.git(self.remote, "commit", "--quiet", "-m", value)
+            self.revisions.append(self.git(self.remote, "rev-parse", "HEAD"))
+        self.checkout = self.root / "data" / "fixture-repo"
+        self.pins = self.root / "pins.cmake"
+
+    @staticmethod
+    def git(directory, *args):
+        return subprocess.run(["git", "-C", str(directory), *args], check=True,
+                              capture_output=True, text=True).stdout.strip()
+
+    def run_setup(self, revision, update=False):
+        self.pins.write_text(
+            f'odr_test_data(PATH fixture-repo URL "{self.remote}" REVISION {revision})\n')
+        return subprocess.run(
+            ["cmake", f"-DODR_TEST_DATA_ROOT={self.root / 'data'}",
+             f"-DODR_TEST_DATA_PINS={self.pins}", f"-DODR_TEST_DATA_UPDATE={'ON' if update else 'OFF'}",
+             "-P", str(ROOT / "cmake/test_data.cmake")], capture_output=True, text=True)
+
+    def assert_setup_succeeds(self, revision, update=False):
+        result = self.run_setup(revision, update)
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
+    def test_interrupted_initialization_resumes(self):
+        self.assertNotEqual(self.run_setup("f" * 40).returncode, 0)
+        self.assertTrue((self.checkout / ".git").exists())
+        self.assert_setup_succeeds(self.revisions[0])
+        self.assertEqual(self.git(self.checkout, "rev-parse", "HEAD"), self.revisions[0])
+
+        shutil.rmtree(self.checkout)
+        self.checkout.mkdir()
+        self.git(self.checkout, "init", "--quiet")
+        (self.checkout / "fixture").write_text("untracked work")
+        self.assertNotEqual(self.run_setup(self.revisions[0]).returncode, 0)
+        self.assertEqual((self.checkout / "fixture").read_text(), "untracked work")
+        (self.checkout / "fixture").unlink()
+        self.assert_setup_succeeds(self.revisions[0])
+        self.assertEqual((self.checkout / "fixture").read_text(), "first")
+
+    def test_updates_preserve_local_changes_and_require_opt_in(self):
+        self.assert_setup_succeeds(self.revisions[0])
+        self.assert_setup_succeeds(self.revisions[1])
+        self.assertEqual(self.git(self.checkout, "rev-parse", "HEAD"), self.revisions[0])
+        (self.checkout / "fixture").write_text("local changes")
+        self.assertNotEqual(self.run_setup(self.revisions[1], update=True).returncode, 0)
+        self.assertEqual((self.checkout / "fixture").read_text(), "local changes")
+        self.git(self.checkout, "checkout", "--", "fixture")
+        self.assert_setup_succeeds(self.revisions[1], update=True)
+        self.assertEqual(self.git(self.checkout, "rev-parse", "HEAD"), self.revisions[1])