From 14f23096883c9e3bd2d11af82b489414347a94fa Mon Sep 17 00:00:00 2001 From: Skilly Date: Sun, 6 Sep 2026 15:53:18 +0200 Subject: [PATCH] Fix .gitignore swallowing the pipeline build package The unanchored 'build/' pattern matched src/apworld_tester/pipeline/build/, so the package was never committed and a fresh clone failed at import with ModuleNotFoundError. Anchor build/ and dist/ to the repo root and add the missing module files. Also ignore .venv/. --- .gitignore | 7 +- src/apworld_tester/pipeline/build/__init__.py | 0 .../pipeline/build/archipelago.py | 193 ++++++++++++++++++ src/apworld_tester/pipeline/build/docker.py | 53 +++++ 4 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 src/apworld_tester/pipeline/build/__init__.py create mode 100644 src/apworld_tester/pipeline/build/archipelago.py create mode 100644 src/apworld_tester/pipeline/build/docker.py diff --git a/.gitignore b/.gitignore index 36df193..5044efe 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ failed_apworlds.txt CLAUDE.md # Build output. -build/ -dist/ +/build/ +/dist/ *.egg-info/ + +# Virtual environment. +.venv/ diff --git a/src/apworld_tester/pipeline/build/__init__.py b/src/apworld_tester/pipeline/build/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/apworld_tester/pipeline/build/archipelago.py b/src/apworld_tester/pipeline/build/archipelago.py new file mode 100644 index 0000000..fcf91bc --- /dev/null +++ b/src/apworld_tester/pipeline/build/archipelago.py @@ -0,0 +1,193 @@ +"""Updating the Archipelago checkout and building its images.""" + +import os + +import requests + +from apworld_tester.core.config.secrets import Secrets +from apworld_tester.core.state.keys import StateKeys +from apworld_tester.pipeline.build.docker import Docker + + +class ArchipelagoBuild: + """The Archipelago version this run tests against, and its image. + + The checkout is only `docker build`'s context - a machine that + already has the image needs no clone at all. + """ + + #: config.yaml values that mean "no pin - use the newest release". + UNPINNED = (None, "", "latest") + + #: Appended to image_name for the unprivileged wrapper the tests + #: run in. Not configuration: the two images are one build, and a + #: name that did not derive from the base image's would only let + #: them drift apart. + TEST_SUFFIX = "-test" + + #: GitHub's API, where releases are looked up. + API_ROOT = "https://api.github.com" + + def __init__(self, config, paths, session=None): + self.config = config + self.paths = paths + self.session = session or self.anonymous_session(paths) + + @staticmethod + def anonymous_session(paths): + """A session for the release lookup, with a token if there is + one. Only one request is made, so an anonymous one is fine - + the token just keeps it off the 60-an-hour limit. + """ + session = requests.Session() + token = Secrets(paths).github_token + if token: + session.headers["Authorization"] = f"token {token}" + return session + + def setting(self, key, default=None): + return self.config.value("archipelago", key, default) + + @property + def repository(self): + return self.setting("repository") + + @property + def repo_url(self): + return self.setting("repo_url") + + @property + def image_name(self): + return self.setting("image_name", "archipelago") + + @property + def test_image_name(self): + """The unprivileged image the generation tests run in.""" + return f"{self.image_name}{self.TEST_SUFFIX}" + + def image_tag(self, tag): + return f"{self.image_name}:{tag}" + + def test_image_tag(self, tag): + return f"{self.test_image_name}:{tag}" + + @property + def pinned_version(self): + """The tag to run against, or None to follow the newest release.""" + value = self.config.section("archipelago").get("version") + return None if value in self.UNPINNED else str(value) + + def latest_release(self): + """The newest release's tag and publication date.""" + response = self.session.get( + f"{self.API_ROOT}/repos/{self.repository}/releases/latest", + timeout=30, + ) + response.raise_for_status() + data = response.json() + if not data.get("tag_name"): + raise RuntimeError("GitHub returned no tag_name for the " + "latest release.") + return data["tag_name"], (data.get("published_at") + or data.get("created_at")) + + def published_at(self, tag): + """When one release was published, or None if unreadable.""" + response = self.session.get( + f"{self.API_ROOT}/repos/{self.repository}/releases/tags/{tag}", + timeout=30, + ) + if response.status_code != 200: + return None + return response.json().get("published_at") + + def resolve_version(self): + """The version to run against, and when it was published.""" + pinned = self.pinned_version + if pinned: + return pinned, self.published_at(pinned) + return self.latest_release() + + def ensure_checkout(self): + """Clone Archipelago, or fetch into the clone already there.""" + if os.path.isdir(os.path.join(self.paths.checkout, ".git")): + Docker.run(["git", "fetch", "--tags", "--quiet"], + cwd=self.paths.checkout) + else: + Docker.run(["git", "clone", "--quiet", self.repo_url, + self.paths.checkout]) + + def checkout_tag(self, tag): + """Put the checkout exactly on one tag. + + Force and clean, because the working tree is automation-owned: + it has to match the tag recorded in state, not whatever a + previous run left behind. + """ + Docker.run( + ["git", "checkout", "--quiet", "--force", "--detach", tag], + cwd=self.paths.checkout, + ) + Docker.run(["git", "clean", "--quiet", "-fd"], + cwd=self.paths.checkout) + + def build_base_image(self, tag): + """Build the image straight from the checkout.""" + Docker.run(["docker", "build", "-t", self.image_tag(tag), + self.paths.checkout]) + + def build_test_image(self, tag): + """Build the unprivileged wrapper the tests run in.""" + Docker.run([ + "docker", "build", + "-f", self.paths.test_dockerfile, + "--build-arg", f"BASE_IMAGE={self.image_tag(tag)}", + "--build-arg", f"RUNNER_UID={os.getuid()}", + "--build-arg", f"RUNNER_GID={os.getgid()}", + "-t", self.test_image_tag(tag), + os.path.dirname(self.paths.test_dockerfile), + ]) + + def ensure_test_image(self, tag): + """Build the test image if this machine does not have it.""" + if not Docker.image_exists(self.test_image_tag(tag)): + self.build_test_image(tag) + + def local_tags(self): + """Versions already built as test images here, newest first.""" + return Docker.local_tags(self.test_image_name) + + def build_version(self, tag): + """Put the checkout on this exact version and build its image. + + Resolves nothing and records nothing: the caller has already + decided which version it needs, so a run whose image went + missing rebuilds that version rather than quietly upgrading to + whatever is newest. + """ + self.ensure_checkout() + self.checkout_tag(tag) + if not Docker.image_exists(self.image_tag(tag)): + self.build_base_image(tag) + return tag + + def update_and_build(self, store): + """Put the checkout on the right version and build its image.""" + tag, published_at = self.resolve_version() + self.ensure_checkout() + self.checkout_tag(tag) + built = not Docker.image_exists(self.image_tag(tag)) + if built: + self.build_base_image(tag) + previous = self.record_version(store, tag, published_at) + return tag, previous, tag != previous, built + + def record_version(self, store, tag, published_at): + """Write the resolved version into the state file.""" + state = store.load() + previous = state.get(StateKeys.ARCHIPELAGO_TAG) + state[StateKeys.ARCHIPELAGO_TAG] = tag + if published_at is not None: + state[StateKeys.ARCHIPELAGO_TAG_PUBLISHED_AT] = published_at + store.save(state) + return previous diff --git a/src/apworld_tester/pipeline/build/docker.py b/src/apworld_tester/pipeline/build/docker.py new file mode 100644 index 0000000..e5b2a81 --- /dev/null +++ b/src/apworld_tester/pipeline/build/docker.py @@ -0,0 +1,53 @@ +"""Running docker and git for the build.""" + +import subprocess + + +class Docker: + """The docker and git commands the build needs.""" + + @staticmethod + def run(command, cwd=None): + """Run a command, raising with its exit code if it fails.""" + result = subprocess.run(command, cwd=cwd) + if result.returncode != 0: + raise RuntimeError( + f"Command failed ({result.returncode}): " + f"{' '.join(command)}") + + @classmethod + def image_exists(cls, reference): + """Whether docker already has this image locally.""" + result = subprocess.run( + ["docker", "image", "inspect", reference], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return result.returncode == 0 + + @staticmethod + def version_key(tag): + """Sort key ordering 0.6.10 above 0.6.9. + + A string compare gets that pair backwards. A tag that is not + all-numeric sorts last rather than crashing. + """ + try: + return (1, tuple(int(part) for part in tag.split("."))) + except ValueError: + return (0, ()) + + @classmethod + def local_tags(cls, name): + """Versions of an image already built here, newest first.""" + result = subprocess.run( + ["docker", "images", "--format", "{{.Tag}}", name], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + if result.returncode != 0: + return [] + tags = {line.strip() for line in result.stdout.splitlines() + if line.strip()} + return sorted(tags - {""}, key=cls.version_key, reverse=True)