diff --git a/pyproject.toml b/pyproject.toml index 3b4d7f3..ae566bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ apworld-tester = "apworld_tester.cli:main" [tool.setuptools.packages.find] where = ["src"] +include = ["apworld_tester*"] [tool.setuptools.package-data] apworld_tester = [ diff --git a/src/archipelago_tester/__init__.py b/src/archipelago_tester/__init__.py deleted file mode 100644 index e4c6310..0000000 --- a/src/archipelago_tester/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Tests Archipelago community worlds by generating seeds with them. - -The publishing half - archipelago-world-site - imports this package as a -dependency. Nothing here knows that it exists. -""" diff --git a/src/archipelago_tester/core/__init__.py b/src/archipelago_tester/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/core/config/__init__.py b/src/archipelago_tester/core/config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/core/config/config.py b/src/archipelago_tester/core/config/config.py deleted file mode 100644 index bfb4fa0..0000000 --- a/src/archipelago_tester/core/config/config.py +++ /dev/null @@ -1,39 +0,0 @@ -"""The project's configuration.""" - -import json -import os - - -class Config: - """config.json, read once and passed to whatever needs it.""" - - def __init__(self, values, path=None): - self.values = values - self.path = path - - @classmethod - def load(cls, path=None): - """Read config.json from the checkout.""" - path = path or cls.default_path() - with open(path, "r", encoding="utf-8") as handle: - return cls(json.load(handle), path) - - @staticmethod - def default_path(): - """config.json at the repo root, above the installed package.""" - here = os.path.dirname(os.path.abspath(__file__)) - root = os.path.dirname(os.path.dirname(os.path.dirname(here))) - return os.path.join(os.path.dirname(root), "config.json") - - def get(self, key, default=None): - """One top-level value.""" - value = self.values.get(key) - return default if value is None else value - - def section(self, name): - return self.values.get(name) or {} - - def value(self, section, key, default=None): - """One value from a section.""" - value = self.section(section).get(key) - return default if value is None else value diff --git a/src/archipelago_tester/core/config/paths.py b/src/archipelago_tester/core/config/paths.py deleted file mode 100644 index 8286b3d..0000000 --- a/src/archipelago_tester/core/config/paths.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Every path the pipeline uses.""" - -import os - - -class Paths: - """Where everything lives, derived from one Config. - - The checkout holds code and configuration; the data root holds - everything a run generates, and defaults outside the checkout - because none of that is source. Every path is absolute. - """ - - def __init__(self, config): - self.config = config - - @property - def project(self): - return os.path.dirname(self.config.path) - - @property - def data(self): - """Where everything a run generates lives.""" - configured = self.config.get( - "data_root", - "~/.local/share/archipelago-world-tester", - ) - return os.path.abspath(os.path.expanduser(configured)) - - @property - def state(self): - return os.path.join(self.data, "state.json") - - @property - def scratch_state(self): - return os.path.join(self.data, "scratch_state.json") - - @property - def lock(self): - return os.path.join(self.data, "pipeline.lock") - - @property - def downloads(self): - return os.path.join(self.data, "downloads") - - @property - def output(self): - return os.path.join(self.data, "test_output") - - @property - def history(self): - """Every apworld version this pipeline has seen, per game. - - Beside the downloads rather than inside them: that directory is - a cache and gets cleared to force a clean re-download, and this - is the one thing that cannot be rebuilt afterwards. - """ - return os.path.join(self.data, "history") - - @property - def checkout(self): - return os.path.join(self.data, "archipelago_src") - - @property - def core_worlds(self): - return os.path.join(self.checkout, "worlds") - - @property - def common_client(self): - return os.path.join(self.checkout, "CommonClient.py") - - @property - def failed_report(self): - return os.path.join(self.data, "failed_apworlds.txt") - - @property - def sheet_html(self): - return os.path.join(self.data, "playable_worlds.html") - - @property - def roms(self): - """Base ROMs for the worlds that need one.""" - configured = self.config.get("roms_directory") - if not configured: - return os.path.join(self.data, "roms") - return os.path.abspath(os.path.expanduser(configured)) - - @property - def archipelago_location(self): - """The Archipelago install the approved apworlds are placed in. - - Someone's own install rather than anything a run generates, so - it has no default: without archipelago_location there is no - install to speak of, which is a different thing from one at a - path this pipeline picked. - """ - configured = self.config.value("archipelago", "archipelago_location") - if not configured: - return None - return os.path.abspath(os.path.expanduser(configured)) - - @property - def custom_worlds(self): - """Where that install loads apworlds from.""" - location = self.archipelago_location - return location and os.path.join(location, "custom_worlds") - - @property - def env(self): - return os.path.join(self.project, ".env") - - @property - def container(self): - return os.path.join(self.project, "container") - - @property - def drivers(self): - return os.path.join(self.container, "drivers") - - @property - def test_dockerfile(self): - return os.path.join(self.container, "Dockerfile.test") - - @property - def driver(self): - return os.path.join(self.drivers, "run_test.py") - - @property - def identify_driver(self): - return os.path.join(self.drivers, "identify_apworld.py") - - @property - def verify_driver(self): - return os.path.join(self.drivers, "verify_companions.py") diff --git a/src/archipelago_tester/core/config/run_settings.py b/src/archipelago_tester/core/config/run_settings.py deleted file mode 100644 index 48f5e49..0000000 --- a/src/archipelago_tester/core/config/run_settings.py +++ /dev/null @@ -1,66 +0,0 @@ -"""How much testing one run asks for. - -Companions are drawn at random from the worlds Archipelago ships. Core -worlds are the safe choice: part of the build itself, so always present -and always matching the running version, unlike an apworld that might -be missing or broken for reasons of its own. A fresh draw per attempt -matters - ten runs against one fixed pair only ever test that pair. -""" - - -class RunSettings: - """The testing section, as the numbers a run is built from.""" - - def __init__(self, config): - self.config = config - - def number(self, key, default): - return int(self.config.value("testing", key, default)) - - @property - def companion_range(self): - """How many core worlds join the world under test. - - Redrawn every attempt, so ten repeats sample ten combinations. - Clamped so a mistyped config cannot ask for an empty or - backwards range. - """ - low = max(self.number("companion_min", 2), 1) - return low, max(self.number("companion_max", 5), low) - - @property - def repeats(self): - """How often the single and multi tests are repeated. - - Stability is a pass rate across runs, not one verdict: each - repeat gets a different seed, so the fill differs every time. - """ - return self.number("repeats", 10) - - @property - def random_repeats(self): - """How often the two randomized modes are repeated. - - Lower on purpose: single and multi re-roll only the seed, so - ten runs measure one thing precisely, while each randomized run - rolls a different point in the option space and three samples - say much more than one did. - """ - return self.number("random_repeats", 3) - - @property - def spoiler(self): - """2 computes the full playthrough, exercising accessibility - and reachability logic that plain generation skips - so a world - whose logic is wrong fails here rather than in a player's seed. - """ - return self.number("spoiler", 2) - - @property - def jobs(self): - """How many apworlds are tested at once. - - Each test is one isolated `docker run` and subprocess.run - releases the GIL while it waits, so threads are all this needs. - """ - return self.number("jobs", 4) diff --git a/src/archipelago_tester/core/config/secrets.py b/src/archipelago_tester/core/config/secrets.py deleted file mode 100644 index a5f53b5..0000000 --- a/src/archipelago_tester/core/config/secrets.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Credentials, kept out of config.json.""" - -import os - - -class Secrets: - """The .env file at the checkout's root. - - Kept out of config.json: that file gets opened, edited and shared, - and a token in it leaks the first time it is. - """ - - #: The environment variable a GitHub token is read from. - GITHUB_TOKEN = "GITHUB_TOKEN" - - def __init__(self, paths): - self.paths = paths - - def values(self): - """NAME -> value for every line in the file.""" - if not os.path.exists(self.paths.env): - return {} - values = {} - with open(self.paths.env, "r", encoding="utf-8") as handle: - for line in handle: - line = line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - name, value = line.split("=", 1) - values[name.strip()] = value.strip().strip("\"'") - return values - - def get(self, name, default=None): - """One secret, or `default` when it is unset or empty.""" - value = self.values().get(name) - return default if value in (None, "") else value - - @property - def github_token(self): - """The GitHub token, or None.""" - return self.get(self.GITHUB_TOKEN) diff --git a/src/archipelago_tester/core/display/__init__.py b/src/archipelago_tester/core/display/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/core/display/progress.py b/src/archipelago_tester/core/display/progress.py deleted file mode 100644 index ca0dfc2..0000000 --- a/src/archipelago_tester/core/display/progress.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Progress reporting for the long stages.""" - -import shutil -import sys -import time - - -class Progress: - """One stage's progress, tallied by outcome as it goes. - - Both the download stage and the generation tests walk ~650 games, so - per-line output scrolls past uselessly when a run is watched. Under - cron the opposite is true - stdout is a log file, where a bar - redrawn with carriage returns is noise - so each caller supplies the - line it would have printed, and that is what a non-terminal gets. - """ - - def __init__(self, total, config=None, labels=None, stream=None): - self.total = total - self.config = config - self.labels = dict(labels or self.configured_labels()) - self.stream = stream or sys.stdout - self.counts = {} - self.started = None - - def configured_labels(self): - """Short labels for each outcome.""" - if self.config is None: - return {} - return self.config.value("display", "outcome_labels", {}) - - @property - def width(self): - """How many characters wide the bar is drawn.""" - if self.config is None: - return 28 - return int(self.config.value("display", "progress_bar_width", 28)) - - @staticmethod - def duration(seconds): - """Seconds as m:ss, or h:mm:ss past an hour.""" - seconds = int(seconds) - hours, rest = divmod(seconds, 3600) - minutes, seconds = divmod(rest, 60) - if hours: - return f"{hours}:{minutes:02d}:{seconds:02d}" - return f"{minutes:02d}:{seconds:02d}" - - @property - def tally(self): - """The running counts, labelled keys first.""" - ordered = [key for key in self.labels if self.counts.get(key)] - ordered += [key for key in self.counts if key not in self.labels] - return " ".join( - f"{self.labels.get(key, key)} {self.counts[key]}" - for key in ordered - ) - - def bar(self, fraction): - """The bar itself, filled to `fraction`.""" - filled = int(self.width * fraction) - return "#" * filled + "-" * (self.width - filled) - - def eta(self, index, elapsed): - """Time remaining at the current rate, blank once finished.""" - if not index or index >= self.total: - return "" - return f" eta {self.duration(elapsed / index * (self.total - index))}" - - def line(self, index, text): - """The whole status line for this update.""" - fraction = index / self.total if self.total else 1 - elapsed = time.monotonic() - self.started - return (f"[{self.bar(fraction)}] {index}/{self.total} " - f"{fraction * 100:3.0f}% {self.tally} " - f"{self.duration(elapsed)}{self.eta(index, elapsed)} {text}") - - def draw(self, index, text): - """Redraw the bar in place, ending the line when finished. - - Padded to overwrite a previously longer line, then trimmed to - the terminal: a wrapped line cannot be overwritten by a carriage - return, and the bar starts spilling down the screen. - """ - columns = shutil.get_terminal_size((100, 24)).columns - padded = self.line(index, text)[:columns - 1].ljust(columns - 1) - self.stream.write("\r" + padded) - self.stream.flush() - if index >= self.total: - self.stream.write("\n") - self.stream.flush() - - def update(self, index, key=None, text="", fallback_line=None): - """Record one finished item and show where the stage is.""" - if self.started is None: - self.started = time.monotonic() - if key is not None: - self.counts[key] = self.counts.get(key, 0) + 1 - if not self.stream.isatty(): - if fallback_line is not None: - print(fallback_line, file=self.stream) - return - self.draw(index, text) diff --git a/src/archipelago_tester/core/model/__init__.py b/src/archipelago_tester/core/model/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/core/model/apworld.py b/src/archipelago_tester/core/model/apworld.py deleted file mode 100644 index 40bb2fa..0000000 --- a/src/archipelago_tester/core/model/apworld.py +++ /dev/null @@ -1,120 +0,0 @@ -"""The filename an apworld needs before Archipelago will import it.""" - -import json -import os -import re -import zipfile - - -class ApworldFile: - """One downloaded apworld, and what its name has to be. - - Archipelago derives the module name from the FILENAME, so a version - in the name reads as package separators and a backslash is just a - character. The right name is the zip's own package folder. - """ - - #: The extension an apworld is recognised by. - SUFFIX = ".apworld" - - def __init__(self, path): - self.path = path - - @property - def entries(self): - """Every entry name in the zip, empty if it will not open.""" - try: - with zipfile.ZipFile(self.path) as archive: - return archive.namelist() - except (OSError, zipfile.BadZipFile): - return [] - - @property - def uses_backslash_separators(self): - """Whether the zip stores its paths with backslashes. - - Invisible on Windows and fatal on Linux: zipimport swaps "/" for - the platform separator when reading, so on Windows both forms - match and on Linux the backslash entry never matches the - "pkg/__init__.py" it looks for. The author sees a working - apworld and every Linux user sees ModuleNotFoundError. - """ - names = self.entries - if not names: - return False - return (not any("/" in name for name in names) - and any("\\" in name for name in names)) - - @property - def manifest(self): - """The apworld's own archipelago.json, or None. - - Its self-declared "game" and "world_version", sitting in the - file's content regardless of which release was resolved for it - - or whether one ever was. Older apworlds predate the convention - and ship none, so this is best-effort display information rather - than something worth failing over. - """ - try: - with zipfile.ZipFile(self.path) as archive: - name = next((entry for entry in archive.namelist() - if entry.endswith("/archipelago.json")), None) - if name is None: - return None - with archive.open(name) as handle: - return json.load(handle) - except (OSError, zipfile.BadZipFile, json.JSONDecodeError, KeyError): - return None - - @property - def package_folder(self): - """The single top-level package inside the zip, or None.""" - names = [name.replace("\\", "/") for name in self.entries] - packages = { - name.split("/", 1)[0] - for name in names - if name.count("/") == 1 and name.endswith("/__init__.py") - } - return packages.pop() if len(packages) == 1 else None - - @property - def importable_name(self): - """What this file must be called for Archipelago to import it.""" - base = os.path.basename(self.path.replace("\\", "/")) - if base.lower().endswith(ApworldFile.SUFFIX): - stem = base[:-len(ApworldFile.SUFFIX)] - else: - stem = base - stem = self.package_folder or self.sanitize(stem) - return stem + ApworldFile.SUFFIX - - def sanitize(self, stem): - """A filename stem reduced to something importable.""" - base = os.path.basename(stem.replace("\\", "/")) - cleaned = re.sub(r"[^0-9A-Za-z_]", "_", base).strip("_") - if cleaned and cleaned[0].isdigit(): - cleaned = "_" + cleaned - return cleaned or "apworld" - - def prepare(self, working_directory): - """The path to hand Archipelago, and whether it was repaired.""" - if not self.uses_backslash_separators: - return self.path, False - os.makedirs(working_directory, exist_ok=True) - destination = os.path.join(working_directory, self.importable_name) - return self.write_normalized(destination), True - - def write_normalized(self, destination): - """Copy the apworld with its separators repaired.""" - with zipfile.ZipFile(self.path) as source: - with zipfile.ZipFile(destination, "w", - zipfile.ZIP_DEFLATED) as target: - for item in source.infolist(): - repaired = zipfile.ZipInfo( - filename=item.filename.replace("\\", "/"), - date_time=item.date_time, - ) - repaired.compress_type = item.compress_type - repaired.external_attr = item.external_attr - target.writestr(repaired, source.read(item.filename)) - return destination diff --git a/src/archipelago_tester/core/model/blacklist.py b/src/archipelago_tester/core/model/blacklist.py deleted file mode 100644 index 283631b..0000000 --- a/src/archipelago_tester/core/model/blacklist.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Worlds this pipeline declines to test.""" - -from archipelago_tester.core.model.name import Name - - -class Blacklist: - """Worlds not tested, and why. - - A world that times out is the most expensive thing in a run and the - least informative. Entries are permanent until removed by hand - - "do not retry" is the whole reason one is listed. - """ - - #: Recorded verdict for a world that was never tested. - OUTCOME = "unknown" - - def __init__(self, config): - self.config = config - - @property - def entries(self): - """Folded name -> reason.""" - listed = self.config.get("blacklist", {}) - return { - Name(name).slug: reason - for name, reason in listed.items() - if Name(name).slug - } - - def reason_for(self, game_name): - return self.entries.get(Name(game_name).slug) - - def detail(self, reason): - return f"not tested: {reason}" diff --git a/src/archipelago_tester/core/model/fingerprint.py b/src/archipelago_tester/core/model/fingerprint.py deleted file mode 100644 index 14c3c90..0000000 --- a/src/archipelago_tester/core/model/fingerprint.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Content hashes, used to decide what has changed.""" - -import hashlib -import os - - -class Fingerprint: - """The content of a file or a directory, as one hash.""" - - @staticmethod - def of_file(path): - """One file's sha256, read in chunks.""" - digest = hashlib.sha256() - with open(path, "rb") as handle: - for chunk in iter(lambda: handle.read(65536), b""): - digest.update(chunk) - return digest.hexdigest() - - @classmethod - def of_directory(cls, directory): - """One hash covering every file directly inside a directory.""" - digest = hashlib.sha256() - if not os.path.isdir(directory): - return digest.hexdigest() - for name in sorted(os.listdir(directory)): - path = os.path.join(directory, name) - if os.path.isfile(path): - digest.update(name.encode("utf-8")) - digest.update(cls.of_file(path).encode("utf-8")) - return digest.hexdigest() diff --git a/src/archipelago_tester/core/model/game.py b/src/archipelago_tester/core/model/game.py deleted file mode 100644 index e1d0661..0000000 --- a/src/archipelago_tester/core/model/game.py +++ /dev/null @@ -1,47 +0,0 @@ -"""One game from the worlds sheet.""" - -import re - - -class Game: - """A sheet row: a name, its links, and what the sheet says about it.""" - - def __init__(self, name, links=None, release=None, stability=None, - pr_status=None): - self.name = name - self.links = list(links or []) - self.release = release - self.stability = stability - self.pr_status = pr_status - for link in self.links: - self.remember_release(link) - - @staticmethod - def is_release_link(link): - """Whether a link points at a GitHub releases page.""" - return bool(re.match( - r"^https?://(?:www\.)?github\.com/[^/]+/[^/]+/releases" - r"(?:[/?#]|$)", - link or "", - re.IGNORECASE, - )) - - @property - def is_core(self): - return (self.pr_status or "").strip().lower() == "core" - - def remember_release(self, link): - """Keep this link as the release URL if none was found yet.""" - if self.release is None and self.is_release_link(link): - self.release = link - return self.release - - def add_link(self, link): - """Add a link, ignoring one already recorded.""" - if link not in self.links: - self.links.append(link) - self.remember_release(link) - return self.links - - def __str__(self): - return f"{self.name} ({len(self.links)} links)" diff --git a/src/archipelago_tester/core/model/instant.py b/src/archipelago_tester/core/model/instant.py deleted file mode 100644 index 14b97be..0000000 --- a/src/archipelago_tester/core/model/instant.py +++ /dev/null @@ -1,35 +0,0 @@ -"""One timestamp, as the moment it names.""" - -import datetime - - -class Instant: - """An ISO 8601 timestamp parsed into a comparable moment. - - Hosts do not agree on how to write the offset: GitHub sends a - trailing "Z", this pipeline's own clock writes "+00:00", and GitLab - and Gitea send a local one ("...19:55:36-04:00" is really 23:55Z). - Compared as text those sort by the numbers on the clock face rather - than by when they happened, so anything comparing two timestamps - from different sources has to parse them first. - """ - - @staticmethod - def of(text): - """One timestamp, or None when it is not a usable one. - - A naive timestamp is read as UTC: everything written here is, - and the alternative is a comparison that raises rather than one - that is merely approximate. "Z" is spelled out because - fromisoformat only accepts it from 3.11. - """ - if not text: - return None - try: - parsed = datetime.datetime.fromisoformat( - str(text).replace("Z", "+00:00")) - except (TypeError, ValueError): - return None - if parsed.tzinfo is None: - return parsed.replace(tzinfo=datetime.timezone.utc) - return parsed diff --git a/src/archipelago_tester/core/model/modes.py b/src/archipelago_tester/core/model/modes.py deleted file mode 100644 index 8433e71..0000000 --- a/src/archipelago_tester/core/model/modes.py +++ /dev/null @@ -1,22 +0,0 @@ -"""The generation modes a complete result carries.""" - - -class Modes: - """What the driver runs against every world. - - Not configuration: each name is a branch in the container driver - and in Verdict, so a mode cannot be added or removed by editing a - list. Changing this would only desynchronise the cache check and - the results table from what actually ran. - """ - - SINGLE = "single" - DUPLICATE = "duplicate" - MULTI = "multi" - SINGLE_RANDOM = "single_random" - MULTI_RANDOM = "multi_random" - - ALL = (SINGLE, DUPLICATE, MULTI, SINGLE_RANDOM, MULTI_RANDOM) - - #: The modes that cannot demote a world on their own. - SECONDARY = (DUPLICATE, SINGLE_RANDOM, MULTI_RANDOM) diff --git a/src/archipelago_tester/core/model/name.py b/src/archipelago_tester/core/model/name.py deleted file mode 100644 index 6453438..0000000 --- a/src/archipelago_tester/core/model/name.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Folding a game name down to a comparison key.""" - -import re -import unicodedata - - -class Name: - """One game name, and the two ways it gets folded. - - One implementation, because five copies once disagreed and a world - silently lost its blacklist entry. - """ - - def __init__(self, text): - self.text = text or "" - - @property - def slug(self): - """The key for matching this name against another.""" - folded = unicodedata.normalize("NFKD", self.text.lower()) - return re.compile(r"[^a-z0-9]").sub("", folded) - - @property - def report_slug(self): - return re.compile(r"[^a-z0-9]+").sub("-", self.text.lower()).strip("-") - - @property - def directory(self): - return re.sub(r'[<>:"/\\|?*]', "_", self.text).strip().rstrip(".") - - def matches(self, other): - """Whether an internally registered name means this game. - - One-directional: the internal name is usually the shorter, - canonical one and the sheet's is often longer, so checking - containment the other way would let a short internal name match - an unrelated row that happens to start the same way. - """ - if not self.text or not other: - return False - internal = Name(other).slug - return internal == self.slug or internal in self.slug diff --git a/src/archipelago_tester/core/model/verdict.py b/src/archipelago_tester/core/model/verdict.py deleted file mode 100644 index 1dc700a..0000000 --- a/src/archipelago_tester/core/model/verdict.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Where a world's test record lands on the stability ladder.""" - -from archipelago_tester.core.model.modes import Modes - - -class Verdict: - """One world's tested stability. - - The ladder: broken, unknown, flaky, solo_only, unstable, stable. - Single and multi decide the first four, because those are what a - player does; the other modes can only stop a world being stable. - """ - - #: Outcomes that count as a failure in a secondary mode. - FAILURES = ("failed", "invalid_options", "flaky") - - def __init__(self, record, config): - self.record = record or {} - self.config = config - - @property - def modes(self): - return self.record.get("tests") or {} - - @property - def never_imported(self): - return self.record.get("outcome") == "unverified" - - def outcome_of(self, mode): - return (self.modes.get(mode) or {}).get("outcome") - - @property - def primary(self): - """What the single and multi runs decide, or None if they pass.""" - single = self.outcome_of(Modes.SINGLE) - multi = self.outcome_of(Modes.MULTI) - if single == "needs_input" or multi == "needs_input": - return "unknown" - if single == "failed": - return "broken" - if single == "flaky" or multi == "flaky": - return "flaky" - if multi == "failed": - return "solo_only" - if single != "passed": - return "unknown" - return None - - @property - def secondary_failed(self): - """Whether a mode that cannot demote alone still failed.""" - return any(self.outcome_of(mode) in self.FAILURES - for mode in Modes.SECONDARY) - - @property - def name(self): - """Where this record lands on the ladder.""" - if self.never_imported: - return "broken" - if not self.modes or not self.outcome_of(Modes.SINGLE): - return "unknown" - decided = self.primary - if decided is not None: - return decided - return "unstable" if self.secondary_failed else "stable" - - @property - def label(self): - labels = self.config.value("stability", "labels", {}) - return labels.get(self.name, "Unknown") - diff --git a/src/archipelago_tester/core/state/__init__.py b/src/archipelago_tester/core/state/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/core/state/keys.py b/src/archipelago_tester/core/state/keys.py deleted file mode 100644 index 312ce02..0000000 --- a/src/archipelago_tester/core/state/keys.py +++ /dev/null @@ -1,23 +0,0 @@ -"""The keys this pipeline writes into state.json.""" - - -class StateKeys: - """Names of the entries in the state file. - - Not configuration: state.json is written and read only by this - pipeline, so renaming a key here would not migrate the file, it - would orphan every record already in it. - """ - - APWORLD_TESTS = "apworld_tests" - UPGRADE_SNAPSHOT = "apworld_tests_at_last_upgrade" - ARCHIPELAGO_TAG = "archipelago_tag" - ARCHIPELAGO_TAG_PUBLISHED_AT = "archipelago_tag_published_at" - PREVIOUS_ARCHIPELAGO_TAG = "previous_archipelago_tag" - COMPANION_POOL = "verified_companions" - GAME_RELEASES = "game_releases" - DISCONTINUED = "discontinued" - LAST_CHECKED_AT = "last_checked_at" - FAILED_RELEASE = "failed_release" - FAILED_REASON = "failed_reason" - STABILITY_HISTORY = "stability_history" diff --git a/src/archipelago_tester/core/state/lock.py b/src/archipelago_tester/core/state/lock.py deleted file mode 100644 index c073a55..0000000 --- a/src/archipelago_tester/core/state/lock.py +++ /dev/null @@ -1,37 +0,0 @@ -"""The exclusive lock a whole run holds.""" - -import fcntl - - -class LockHeldError(RuntimeError): - """Another run already holds the lock.""" - - -class PipelineLock: - """Stops two runs, or a run and a promotion, from overlapping. - - Non-blocking: a second run fails fast rather than queuing. Queuing - would be worse - a full pass takes hours, so a queued one would - start against an Archipelago version that had moved on. - """ - - def __init__(self, path): - self.path = path - self.handle = None - - def __enter__(self): - self.handle = open(self.path, "a+") - try: - fcntl.flock(self.handle, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as error: - self.handle.close() - raise LockHeldError( - f"Another run already holds {self.path} - refusing to " - "start." - ) from error - return self - - def __exit__(self, *unused): - fcntl.flock(self.handle, fcntl.LOCK_UN) - self.handle.close() - return False diff --git a/src/archipelago_tester/core/state/scratch.py b/src/archipelago_tester/core/state/scratch.py deleted file mode 100644 index b70fe08..0000000 --- a/src/archipelago_tester/core/state/scratch.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Choosing between the live record and a scratch copy of it.""" - -import os -import shutil - - -class ScratchState: - """Where a run's results go, live or not. - - A run is an experiment unless it says otherwise: promotion and the - status page read the live file, so a hand-run test that wrote there - would put a world nobody can download into the deployable record. - - Seeding the scratch copy from the live one is what makes that cheap - rather than punishing - without it a scratch run has no recorded - Archipelago version to test against and no verified companion pool, - so it would stop for the first and re-check every core world for - the second. - """ - - def __init__(self, paths, live=False): - self.paths = paths - self.live = live - - @property - def path(self): - if self.live: - return self.paths.state - scratch = self.paths.scratch_state - if not os.path.exists(scratch) and os.path.exists(self.paths.state): - shutil.copy(self.paths.state, scratch) - return scratch diff --git a/src/archipelago_tester/core/state/store.py b/src/archipelago_tester/core/state/store.py deleted file mode 100644 index 8e4b39f..0000000 --- a/src/archipelago_tester/core/state/store.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Reading and writing the pipeline's JSON state.""" - -import json -import os -import tempfile - - -class StateStore: - """What every world last did, and the companion pool behind it.""" - - def __init__(self, path): - self.path = path - - def load(self): - return self.read(self.path, {}) - - def save(self, data): - self.write(self.path, data) - - @staticmethod - def read(path, default=None): - """A JSON file's contents, or `default` when it is absent.""" - if not os.path.exists(path): - return {} if default is None else default - with open(path, "r", encoding="utf-8") as handle: - return json.load(handle) - - @staticmethod - def write(path, data): - """Write JSON so a reader never sees a partial file. - - A kill mid-write used to leave state.json truncated mid-object - - valid up to the cut point and invalid after it - silently - costing every recorded result on the next load. The batch saves - after every single test, so that window was hit often. - - os.replace is atomic: the file is always either the old complete - version or the new one. - """ - directory = os.path.dirname(os.path.abspath(path)) or "." - handle_id, temporary = tempfile.mkstemp( - dir=directory, prefix=".state_", suffix=".tmp") - try: - with os.fdopen(handle_id, "w", encoding="utf-8") as handle: - json.dump(data, handle, indent=2, sort_keys=True) - handle.write("\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - except BaseException: - try: - os.remove(temporary) - except OSError: - pass - raise diff --git a/src/archipelago_tester/pipeline/__init__.py b/src/archipelago_tester/pipeline/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/pipeline/batch/__init__.py b/src/archipelago_tester/pipeline/batch/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/pipeline/batch/apworld_finder.py b/src/archipelago_tester/pipeline/batch/apworld_finder.py deleted file mode 100644 index 50f9a3a..0000000 --- a/src/archipelago_tester/pipeline/batch/apworld_finder.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Finding the apworlds already downloaded.""" - -import os - - -class ApworldFinder: - """The download directory, as the worlds a run can test.""" - - def __init__(self, root): - self.root = root - - @staticmethod - def matches(game_name, only, exact): - if not only: - return True - if exact: - return any(t.casefold() == game_name.casefold() for t in only) - return any(t.lower() in game_name.lower() for t in only) - - def in_folder(self, game_dir): - """Every apworld directly inside one game's folder. - - A directory where a file should be is skipped rather than - crashing the batch: that is almost always docker auto-creating - an empty directory at a bind-mount source that did not exist - yet, and a fresh download replaces it next time. - """ - found = [] - for entry in sorted(os.listdir(game_dir)): - if not entry.lower().endswith(".apworld"): - continue - path = os.path.join(game_dir, entry) - if not os.path.isfile(path): - print(f"Skipping {path} - not a regular file (directory?)") - continue - found.append(path) - return found - - def find(self, only=None, exact=False): - """Every downloaded apworld, optionally narrowed to `only`. - - `only` matches on substring, which is what --only wants. That - is ambiguous when one game's name contains another's - "Hollow - Knight" also selects "Hollow Knight_ Silksong" - so a caller - that already resolved a term to a real folder passes exact. - """ - found = [] - if not os.path.isdir(self.root): - return found - for game_name in sorted(os.listdir(self.root)): - if not self.matches(game_name, only, exact): - continue - game_dir = os.path.join(self.root, game_name) - if not os.path.isdir(game_dir): - continue - found += [(game_name, path) for path in self.in_folder(game_dir)] - return found - - @property - def available(self): - return sorted({name for name, _ in self.find()}) - - def resolve(self, term): - """The folders a name selects, an exact match winning.""" - matches = sorted({name for name, _ in self.find(only=[term])}) - exact = [n for n in matches if n.casefold() == term.casefold()] - return exact or matches diff --git a/src/archipelago_tester/pipeline/batch/batch.py b/src/archipelago_tester/pipeline/batch/batch.py deleted file mode 100644 index a7d2846..0000000 --- a/src/archipelago_tester/pipeline/batch/batch.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Testing every downloaded apworld and recording what each did.""" - -from archipelago_tester.core.model.fingerprint import Fingerprint -from archipelago_tester.core.state.keys import StateKeys -from archipelago_tester.core.state.store import StateStore -from archipelago_tester.pipeline.batch.companion_pool import CompanionPool -from archipelago_tester.pipeline.batch.failed_report import FailedReport -from archipelago_tester.pipeline.batch.options import Options -from archipelago_tester.pipeline.batch.planner import Planner -from archipelago_tester.pipeline.batch.runner import Runner -from archipelago_tester.pipeline.batch.work_list import WorkList - - -class Batch: - """One run: plan it, serve what is cached, test the rest. - - An apworld whose file content and Archipelago version both match - its last recorded test is served from cache. The caller forces a - full re-test when the version changes, because last week's passes - no longer apply to anything. - """ - - def __init__(self, config, paths, settings, image, tag, force=False, - on_progress=None, **asked): - self.config = config - self.paths = paths - self.settings = settings - self.image = image - self.tag = tag - self.force = force or bool(asked.get("only")) - self.on_progress = on_progress - self.options = Options(paths, settings, **asked).resolve() - self.store = StateStore(self.options["state_path"]) - - @property - def writes_live_report(self): - """Whether this run owns the "what is broken" report. - - The report is about the deployable, current-version state: a - previous-version comparison writing here would conflate "not - promotable now" with "does not work on a release nobody runs", - and a run pointed at a scratch state file is not the live - record either. - """ - return (self.options["state_key"] - == StateKeys.APWORLD_TESTS - and self.options["state_path"] == self.paths.state) - - def pool(self): - return CompanionPool( - config=self.config, - paths=self.paths, - store=self.store, - image=self.image, - tag=self.tag, - roms_directory=self.options["roms_directory"], - ).resolve() - - def plan(self, tests, fingerprint, state): - """Everything to consider, what is known, what to run. - - Blacklisted rows and the confirmation date on a cached one are - the entries written outside the runner, which is what normally - persists state - without the save here they would exist only - in memory, and a run that tested nothing would write nothing. - """ - work = WorkList( - root=self.options["root_directory"], - only=self.options["only"], - exact=self.options["exact_only"], - core_games=self.options["core_games"], - ).items() - planner = Planner( - config=self.config, - settings=self.settings, - tests=tests, - tag=self.tag, - roms_fingerprint=fingerprint, - force=self.force, - repeats=self.options["repeats"], - random_repeats=self.options["random_repeats"], - companion_range=self.options["companion_range"], - # A release published after a world's last result - # invalidates it, whatever the file on disk hashes to. - releases=state.get(StateKeys.GAME_RELEASES, {}), - ) - results, to_test = planner.plan(work) - if planner.wrote_state: - self.store.save(state) - return work, results, to_test - - def report_cached(self, results, total): - """Free to produce, so reported without waiting on the pool.""" - for index, report in enumerate(results, start=1): - if self.on_progress: - self.on_progress(index, total, report) - - def run(self): - self.options["pool"] = self.pool() - state = self.store.load() - tests = state.setdefault(self.options["state_key"], {}) - fingerprint = Fingerprint.of_directory(self.options["roms_directory"]) - work, results, to_test = self.plan(tests, fingerprint, state) - self.report_cached(results, len(work)) - if to_test: - results += Runner( - config=self.config, - paths=self.paths, - store=self.store, - options=self.options, - state=state, - tests=tests, - tag=self.tag, - image=self.image, - roms_fingerprint=fingerprint, - ).run(to_test, len(work), len(results), self.on_progress) - if self.writes_live_report: - FailedReport(state, self.paths.failed_report).write() - return results diff --git a/src/archipelago_tester/pipeline/batch/cache_check.py b/src/archipelago_tester/pipeline/batch/cache_check.py deleted file mode 100644 index 3d6da09..0000000 --- a/src/archipelago_tester/pipeline/batch/cache_check.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Whether a previous result still answers this run's question.""" - -import re - -from archipelago_tester.core.model.instant import Instant -from archipelago_tester.core.model.modes import Modes - - -class CacheCheck: - """One world's recorded result, against what is being asked now.""" - - def __init__(self, settings, previous): - self.settings = settings - self.previous = previous - - #: Failure text pointing at a base ROM, not the world's logic. A - #: wrong or missing ROM surfaces under many exception types, so the - #: type cannot recognise one and the message is what is left. - #: Word-bounded so "from" does not read as "rom". - ROM_FAILURE = re.compile( - r"\b(rom|md5|sha1|checksum)\b" - r"|\.(sfc|smc|z64|n64|nes|gba|gbc|gb|gen|md|iso|bin)\b", - re.IGNORECASE, - ) - - def mentions_rom(self, detail): - return bool(detail and self.ROM_FAILURE.search(str(detail))) - - def needs_rom_retry(self, roms_fingerprint): - """Whether a changed ROM folder invalidates this result. - - A missing ROM raises FileNotFoundError and lands in - "needs_input", but a WRONG one raises whatever the world throws - after reading it and lands in "failed" - so retrying only - needs_input meant replacing a bad dump could never clear the - result. A "failed" result additionally has to look like a ROM - problem: most failures carrying a stale fingerprint were fill - failures and deprecated APIs that no ROM will fix. - """ - if self.previous is None: - return False - outcome = self.previous.get("outcome") - rom_related = (outcome == "needs_input" - or (outcome == "failed" - and self.mentions_rom(self.previous.get("detail")))) - return (rom_related and self.previous.get("roms_fingerprint") - != roms_fingerprint) - - def released_since(self, released_at): - """Whether the row's apworld was published after it was tested. - - The hash catches a file whose content changed, but not a host - that offered a new one and had it turn out identical - and - "identical" is only ever this pipeline's word for it, resting - on the copy it happened to hold. A publication later than the - answer means the answer was reached before whatever is being - offered now, and testing again is the only thing that settles - it. It settles it for good, too: the new result is dated after - the release, so this stops firing until the release moves - again. - """ - if self.previous is None: - return False - released = Instant.of(released_at) - tested = Instant.of(self.previous.get("tested_at")) - if released is None or tested is None: - return False - return tested < released - - @staticmethod - def attempts(recorded_modes): - """How many attempts a cached record was reached with. - - The randomized modes used to run exactly once and carried no - count at all, so a missing value there means "one". - """ - if not recorded_modes: - return None, None - return ( - (recorded_modes.get("single") or {}).get("requested_attempts"), - (recorded_modes.get("single_random") or {}) - .get("requested_attempts"), - ) - - def answers(self, repeats, random_repeats, companion_range): - """Whether the record answers what this run is asking. - - It does not when the record predates per-mode results, when the - set of modes has changed, when it was reached with fewer - repeats - "it generated once" is not "it generated ten times - out of ten" - or when the companion range differs. A world that - never imported is exempt: "unverified" is a complete answer - about this exact file, and the file has to change for the - answer to. - """ - if self.previous is None: - return True - if self.previous.get("outcome") == "unverified": - return True - recorded = self.previous.get("tests") - if recorded is None or set(recorded) != set(Modes.ALL): - return False - single, randomized = self.attempts(recorded) - if single != repeats: - return False - if randomized is not None and randomized != random_repeats: - return False - recorded_range = (recorded.get("multi") or {}).get( - "companion_range") or [] - return list(recorded_range) == list(companion_range) - - def serves(self, current_hash, tag, roms_fingerprint, repeats, - random_repeats, companion_range, released_at=None): - """Whether this world can be served from its previous result.""" - return (self.previous is not None - and self.previous.get("hash") == current_hash - and self.previous.get("tag") == tag - and not self.needs_rom_retry(roms_fingerprint) - and not self.released_since(released_at) - and self.answers(repeats, random_repeats, companion_range)) diff --git a/src/archipelago_tester/pipeline/batch/cli.py b/src/archipelago_tester/pipeline/batch/cli.py deleted file mode 100644 index 90e2449..0000000 --- a/src/archipelago_tester/pipeline/batch/cli.py +++ /dev/null @@ -1,152 +0,0 @@ -"""The command line for a full batch run.""" - -import argparse -import json - -from archipelago_tester.core.config.config import Config -from archipelago_tester.core.config.paths import Paths -from archipelago_tester.core.config.run_settings import RunSettings -from archipelago_tester.core.state.keys import StateKeys -from archipelago_tester.core.state.lock import LockHeldError, PipelineLock -from archipelago_tester.core.state.scratch import ScratchState -from archipelago_tester.core.state.store import StateStore -from archipelago_tester.pipeline.batch.batch import Batch -from archipelago_tester.pipeline.batch.reporter import Reporter -from archipelago_tester.pipeline.batch.results import Results -from archipelago_tester.pipeline.batch.version import Version -from archipelago_tester.pipeline.build.archipelago import ArchipelagoBuild - - -class Cli: - """Test every downloaded apworld, or the ones named with --only.""" - - def __init__(self): - self.config = Config.load() - self.paths = Paths(self.config) - self.settings = RunSettings(self.config) - self.build = ArchipelagoBuild(self.config, self.paths) - - def parser(self): - parser = argparse.ArgumentParser(description=self.__doc__) - parser.add_argument("--root-directory", default=self.paths.downloads) - parser.add_argument("--output-dir", default=self.paths.output) - parser.add_argument( - "--state-path", default=None, - help="Where results are recorded. Defaults to the scratch " - "file, or to the live state.json with --live; naming a " - "path here overrides both.") - parser.add_argument( - "--live", action="store_true", - help="Record into the real state.json, which promotion and " - "the status page read. Off by default: a run started " - "by hand is an experiment until it says otherwise.") - parser.add_argument("--timeout", type=int, default=None) - parser.add_argument( - "--force", action="store_true", - help="Re-test every apworld, ignoring the unchanged-since-" - "last-time skip.") - parser.add_argument( - "--only", action="append", - help="Only check games whose name contains this substring " - "(repeatable). Always retests matches regardless of cache.") - parser.add_argument( - "--update", action="store_true", - help="Resolve the newest Archipelago release, build it if " - "needed, and record it. Off by default: a new version " - "invalidates every cached result.") - parser.add_argument( - "--image", - help="Image to test against; defaults to archipelago-test:" - " for the version recorded in state.json (which " - "only --update changes).") - parser.add_argument( - "--jobs", "-j", type=int, default=self.settings.jobs, - help=f"Apworlds to test at once (default: {self.settings.jobs}).") - return parser - - def state_path(self, args): - """The file this run records into. - - An explicit --state-path wins, so a developer can keep a run - entirely to themselves; otherwise --live decides, and its - default sends results to a scratch copy of the live record. - """ - return args.state_path or ScratchState(self.paths, - live=args.live).path - - def announce(self, state_path): - """Say where results land, because the default is not the - live record and a silent run would not show which it chose.""" - if state_path == self.paths.state: - print(f"results {state_path}") - return - print(f"results {state_path} (scratch, not the live " - "record - pass --live to record for real)") - - def batch(self, args, tag, state_path): - return Batch( - config=self.config, - paths=self.paths, - settings=self.settings, - image=args.image or self.build.test_image_tag(tag), - tag=tag, - force=args.force, - on_progress=Reporter(self.config, bar=True), - root_directory=args.root_directory, - output_directory=args.output_dir, - state_path=state_path, - timeout=args.timeout, - only=args.only, - jobs=args.jobs, - ).run() - - def report(self, results): - """Print the summary a finished run ends on.""" - summary = Results(results, self.config).summary - print(f"\n{json.dumps(summary)}") - failed = [r["game"] for r in results if r.get("outcome") == "failed"] - if failed: - print(f"failed: {', '.join(failed)}") - if not results: - print("No apworlds found to test.") - return 0 if summary.get("failed", 0) == 0 else 1 - - def version(self, args, store): - """The version to test against, or None with a reason said. - - Without --update the recorded version is used as-is: nothing - upgrades behind the caller's back, because a version change - forces every world to be tested again. - """ - if args.update: - tag, source = Version(self.config, self.paths, store).prepare( - update=True) - print(f"archipelago {tag} ({source})") - return tag - tag = store.load().get(StateKeys.ARCHIPELAGO_TAG) - if not tag: - print("No Archipelago version recorded in state.json - run " - "archipelago-test --update to resolve and build one.") - return tag - - def run(self): - args = self.parser().parse_args() - state_path = self.state_path(args) - self.announce(state_path) - store = StateStore(state_path) - tag = self.version(args, store) - if not tag: - return 1 - if not args.image: - self.build.ensure_test_image(tag) - try: - with PipelineLock(self.paths.lock): - return self.report(self.batch(args, tag, state_path)) - except LockHeldError as error: - print(error) - return 1 - - @classmethod - def main(cls): - """The console script's entry point.""" - return cls().run() diff --git a/src/archipelago_tester/pipeline/batch/companion_pool.py b/src/archipelago_tester/pipeline/batch/companion_pool.py deleted file mode 100644 index 52bf099..0000000 --- a/src/archipelago_tester/pipeline/batch/companion_pool.py +++ /dev/null @@ -1,65 +0,0 @@ -"""The core worlds a run may use as companions.""" - -from datetime import datetime, timezone - -from archipelago_tester.core.model.fingerprint import Fingerprint -from archipelago_tester.core.state.keys import StateKeys -from archipelago_tester.pipeline.generation.companion_check import ( - CompanionCheck, -) - - -class CompanionPool: - """Verified once per run and cached against version and ROMs. - - The answer only changes when Archipelago or the ROMs folder does, - and what it rejects would otherwise fail seeds belonging to the - world under test. - """ - - def __init__(self, config, paths, store, image, tag, roms_directory): - self.config = config - self.paths = paths - self.store = store - self.image = image - self.tag = tag - self.fingerprint = Fingerprint.of_directory(roms_directory) - - @property - def key(self): - return StateKeys.COMPANION_POOL - - def cached(self, state): - cached = state.get(self.key) or {} - if (cached.get("tag") == self.tag - and cached.get("roms_fingerprint") == self.fingerprint): - return cached.get("games") or [] - return None - - def store_result(self, state, result): - state[self.key] = { - "tag": self.tag, - "roms_fingerprint": self.fingerprint, - "games": result["verified"], - "rejected": result["rejected"], - "checked_at": datetime.now(timezone.utc).isoformat(), - } - self.store.save(state) - print(f"{len(result['verified'])} core worlds usable as companions, " - f"{len(result['rejected'])} rejected") - for game, reason in sorted(result["rejected"].items()): - print(f" {game}: {reason[:90]}") - return result["verified"] - - def resolve(self): - state = self.store.load() - cached = self.cached(state) - if cached is not None: - return cached - result = CompanionCheck(self.config, self.paths, self.image).run( - self.paths.output) - if not result.get("ran"): - print(f"companion verification failed ({result.get('detail')}) - " - "multi-game tests will draw from every core world") - return [] - return self.store_result(state, result) diff --git a/src/archipelago_tester/pipeline/batch/discontinued.py b/src/archipelago_tester/pipeline/batch/discontinued.py deleted file mode 100644 index a5f321c..0000000 --- a/src/archipelago_tester/pipeline/batch/discontinued.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Flagging test records whose game left the sheet.""" - -from archipelago_tester.core.model.name import Name -from archipelago_tester.core.state.keys import StateKeys - - -class Discontinued: - """A world taken off the sheet stops being one this pipeline - tracks: it should leave the status page and stop being deployed. - The record is kept and labelled rather than deleted, so its history - survives and a row that comes back - or that vanished because of a - bad sheet parse - is one flag away from being live again. - """ - - def __init__(self, store): - self.store = store - - @staticmethod - def sheet_name(key, record): - """The sheet row a record's key came from. - - A download- or core-stage record is keyed on the bare sheet - name, since nothing was ever fetched for it. Everything else is - keyed "/.apworld". The record's own "game" is - deliberately not used: that is the name Archipelago registered - the world under, which can differ from the sheet's wording. - """ - if (record or {}).get("stage") in ("download", "core"): - return key - return key.split("/", 1)[0] - - def apply(self, tests, live): - """Set or clear the flag, reporting only what moved. - - Comparison is on the folded directory name of both sides, which - is what turned the sheet name into the download folder in the - first place, so punctuation and case cannot make a still-listed - game look discontinued. An unchanged run leaves the state file - byte-identical. - """ - flag = StateKeys.DISCONTINUED - changed = {} - for key, record in tests.items(): - name = self.sheet_name(key, record) - gone = Name(name).directory not in live - if gone != bool(record.get(flag)): - changed[key] = gone - if gone: - record[flag] = True - else: - record.pop(flag, None) - return changed - - def mark(self, sheet_names): - """Flag every record whose game is no longer listed. - - Does nothing when handed no sheet names: an empty list means - the sheet was not read, not that every world on earth was - retired, and acting on it would empty the page. - """ - if not sheet_names: - return {} - state = self.store.load() - tests = state.get(StateKeys.APWORLD_TESTS, {}) - changed = self.apply( - tests, {Name(name).directory for name in sheet_names}) - if changed: - self.store.save(state) - return changed diff --git a/src/archipelago_tester/pipeline/batch/download_failures.py b/src/archipelago_tester/pipeline/batch/download_failures.py deleted file mode 100644 index 62c9f37..0000000 --- a/src/archipelago_tester/pipeline/batch/download_failures.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Recording games whose apworld could never be fetched.""" - -from datetime import datetime, timezone - -from archipelago_tester.core.model.name import Name -from archipelago_tester.core.state.keys import StateKeys -from archipelago_tester.pipeline.batch.record import TestRecord - - -class DownloadFailures: - """A game with no file has nothing to key a result on the way a - tested world does, so it is keyed on its own sheet name - and still - shows up as an unknown row rather than vanishing between the sheet - and the results. Real result keys always contain "/", so a bare - name cannot collide with one. - """ - - def __init__(self, config, store): - self.config = config - self.store = store - - @staticmethod - def tested_already(tests, name): - """Whether a real result already answers for this game. - - A download row exists so a game with no file still shows up - rather than vanishing between the sheet and the results. When - an apworld is on disk and was tested, nothing is vanishing - - the link broke, which the download report says, while the file - that world was last tested from is still there and still the - answer. Writing a second row then reports one game twice, and - the two rows collide at promotion because they claim the same - game name. - """ - folded = Name(name).directory - return any( - "/" in key and Name(key.split("/", 1)[0]).directory == folded - for key in tests - ) - - @staticmethod - def detail_for(reason): - return ("apworld could not be downloaded, so it was never tested: " - f"{reason}") - - def clear_stale(self, tests, games, skipped): - """Drop rows for games that downloaded this time. - - Only ever touches entries this class wrote, never a real - generation-test result. - """ - failed = {name for name, _ in skipped} - for game in games: - previous = tests.get(game.name) - if (game.name not in failed and previous is not None - and previous.get("stage") == "download"): - del tests[game.name] - - def note(self, tests, name, reason, tag, now): - """Record one failure, keeping an unchanged row's date. - - A row failing the same way as last time keeps the date of the - attempt that reached that answer and only has its last-checked - stamp moved: a test date that advanced every run would claim a - world was looked at when nothing about it was. - """ - detail = self.detail_for(reason) - previous = tests.get(name) - if (previous is not None and previous.get("stage") == "download" - and previous.get("detail") == detail): - previous["tag"] = tag - previous[StateKeys.LAST_CHECKED_AT] = now - return - tests[name] = TestRecord(name).download_failure( - detail, tag) - - def record(self, games, skipped, tag): - state = self.store.load() - tests = state.setdefault(StateKeys.APWORLD_TESTS, {}) - self.clear_stale(tests, games, skipped) - now = datetime.now(timezone.utc).isoformat() - for name, reason in skipped: - if self.tested_already(tests, name): - tests.pop(name, None) - continue - self.note(tests, name, reason, tag, now) - self.store.save(state) - return state diff --git a/src/archipelago_tester/pipeline/batch/failed_report.py b/src/archipelago_tester/pipeline/batch/failed_report.py deleted file mode 100644 index 0700fa7..0000000 --- a/src/archipelago_tester/pipeline/batch/failed_report.py +++ /dev/null @@ -1,36 +0,0 @@ -"""The list of worlds that were tested and did not pass.""" - -from archipelago_tester.core.state.keys import StateKeys - - -class FailedReport: - """Written after every run, from the whole accumulated state. - - Not just this run's results: a targeted run touches a couple of - entries and the report must still reflect every other known - failure. Kept out of state.json, which is bookkeeping rather than a - report, so answering the question does not mean writing a query. - """ - - def __init__(self, state, path): - self.state = state - self.path = path - - def failed(self): - tests = self.state.get(StateKeys.APWORLD_TESTS, {}) - return sorted( - ((key, record) for key, record in tests.items() - if record.get("outcome") == "failed"), - key=lambda item: item[1].get("game") or item[0], - ) - - def write(self): - with open(self.path, "w", encoding="utf-8") as handle: - failed = self.failed() - if not failed: - handle.write("No failed apworlds.\n") - return - for key, record in failed: - detail = (record.get("detail") or "").splitlines()[0] - handle.write( - f"{record.get('game') or key} ({key}): {detail}\n") diff --git a/src/archipelago_tester/pipeline/batch/fetcher.py b/src/archipelago_tester/pipeline/batch/fetcher.py deleted file mode 100644 index 3f8c953..0000000 --- a/src/archipelago_tester/pipeline/batch/fetcher.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Downloading the worlds a run was asked for but does not have.""" - -import os - -import requests - -from archipelago_tester.core.config.secrets import Secrets -from archipelago_tester.pipeline.batch.apworld_finder import ApworldFinder -from archipelago_tester.pipeline.download.download_run import DownloadRun -from archipelago_tester.pipeline.sheet.fetcher import SheetFetcher -from archipelago_tester.pipeline.sheet.reader import SheetReader - - -class Fetcher: - """Fills the download directory from the sheet, on demand. - - That directory is a working directory this pipeline fills itself, - from the same sheet a full pass reads - not somewhere a user - points at their own copy. A world never fetched is work not done - yet rather than an error. - """ - - def __init__(self, config, paths, root=None): - self.config = config - self.paths = paths - self.root = root or paths.downloads - self.finder = ApworldFinder(self.root) - - def session(self): - session = requests.Session() - token = Secrets(self.paths).get("GITHUB_TOKEN") - if token: - session.headers["Authorization"] = f"token {token}" - return session - - def listed(self): - """The sheet's rows, fetching the sheet if this box has none.""" - fetcher = SheetFetcher(self.config, self.paths) - if not os.path.exists(self.paths.sheet_html): - print(f"fetching the {fetcher.tab_name} sheet " - f"-> {self.paths.sheet_html}") - fetcher.ensure() - return SheetReader(self.config, self.paths.sheet_html).games() - - @staticmethod - def row_for(term, listed): - """The one sheet row a name selects. - - An exact name wins over a substring, and a term still matching - several rows is refused rather than guessed at - downloading - and testing two worlds because a name was ambiguous is worse - than saying so. - """ - rows = [g for g in listed if term.lower() in g.name.lower()] - exact = [g for g in rows if g.name.casefold() == term.casefold()] - rows = exact or rows - if not rows: - raise LookupError( - f"{term!r} is not on the worlds sheet - nothing to " - "download and nothing to test") - if len(rows) > 1: - names = ", ".join(g.name for g in rows) - raise LookupError(f"{term!r} matches {len(rows)} sheet rows: " - f"{names}. Name one of them exactly.") - return rows[0] - - def report(self, terms): - """Say where each requested world landed, or raise.""" - for term in terms: - found = self.finder.find(only=[term]) - if not found: - raise LookupError( - f"no apworld could be obtained for {term!r}") - for name, path in found: - print(f" {name}: {path}") - - def ensure(self, terms): - """Download whichever of `terms` is not already here.""" - missing = [t for t in terms if not self.finder.find(only=[t])] - if not missing: - return [] - listed = self.listed() - wanted = [self.row_for(term, listed) for term in missing] - print(f"downloading {', '.join(g.name for g in wanted)} " - f"-> {self.root}") - run = DownloadRun(self.config, self.paths, self.session(), - root_directory=self.root) - _, skipped, _ = run.all(wanted) - for name, reason in skipped: - print(f" {name} could not be downloaded: {reason}") - self.report(missing) - return [g.name for g in wanted] diff --git a/src/archipelago_tester/pipeline/batch/options.py b/src/archipelago_tester/pipeline/batch/options.py deleted file mode 100644 index 41ac364..0000000 --- a/src/archipelago_tester/pipeline/batch/options.py +++ /dev/null @@ -1,48 +0,0 @@ -"""What one batch run was asked for, with the gaps filled in.""" - -from archipelago_tester.core.state.keys import StateKeys - - -class Options: - """The caller's choices, over config.json's defaults. - - `repeats` of None means "whatever config.json says". An explicit - count trades confidence for time, and the cache check compares - against it, so a cheap run cannot masquerade as a full one. - """ - - def __init__(self, paths, settings, **asked): - self.paths = paths - self.settings = settings - self.asked = asked - - def chosen(self, key, default): - value = self.asked.get(key) - return default if value is None else value - - def counted(self, key, default): - value = self.asked.get(key) - return default if value is None else int(value) - - def resolve(self): - return { - "root_directory": self.chosen("root_directory", - self.paths.downloads), - "output_directory": self.chosen("output_directory", - self.paths.output), - "state_path": self.chosen("state_path", self.paths.state), - "roms_directory": self.chosen("roms_directory", self.paths.roms), - "jobs": self.chosen("jobs", self.settings.jobs), - "state_key": self.chosen( - "state_key", StateKeys.APWORLD_TESTS), - "companion_range": self.chosen("companion_range", - self.settings.companion_range), - "repeats": self.counted("repeats", self.settings.repeats), - "random_repeats": self.counted("random_repeats", - self.settings.random_repeats), - "spoiler": self.settings.spoiler, - "timeout": self.asked.get("timeout"), - "only": self.asked.get("only"), - "exact_only": bool(self.asked.get("exact_only")), - "core_games": self.asked.get("core_games") or (), - } diff --git a/src/archipelago_tester/pipeline/batch/orphaned_records.py b/src/archipelago_tester/pipeline/batch/orphaned_records.py deleted file mode 100644 index 02b6f87..0000000 --- a/src/archipelago_tester/pipeline/batch/orphaned_records.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Dropping test records whose apworld is no longer on disk.""" - -import os - -from archipelago_tester.core.state.keys import StateKeys - - -class OrphanedRecords: - """Results for files that have since gone. - - A record is keyed on the path its apworld had under the downloads - root, so a file that is removed - a release that renamed its asset, - a folder cleared by hand - leaves its result behind. Nothing reads - that record as stale: it still names a tag and a verdict, so the - status page goes on showing the world, and where the replacement - file sits beside it the game is reported twice. - - Only records with a file behind them are considered. A download- or - core-stage record is keyed on the bare sheet name and never had one, - and Discontinued is what retires those. - """ - - def __init__(self, store, downloads): - self.store = store - self.downloads = downloads - - def orphaned(self, tests): - """Every keyed-on-a-path record whose file is missing.""" - return [key for key, record in tests.items() - if "/" in key - and (record or {}).get("stage") not in ("download", "core") - and not os.path.isfile(os.path.join(self.downloads, key))] - - def safe(self): - """Whether the downloads folder looks like itself. - - An unreadable or empty root would make every record an orphan - and empty the page, which is the one outcome worth refusing: - a genuinely empty downloads directory has nothing to prune for - anyway, so declining costs nothing and guessing costs the lot. - """ - if not os.path.isdir(self.downloads): - return False - return any(os.scandir(self.downloads)) - - def prune(self): - """Remove them, returning the keys that went.""" - if not self.safe(): - return [] - state = self.store.load() - tests = state.get(StateKeys.APWORLD_TESTS, {}) - gone = self.orphaned(tests) - for key in gone: - del tests[key] - if gone: - self.store.save(state) - return gone diff --git a/src/archipelago_tester/pipeline/batch/planner.py b/src/archipelago_tester/pipeline/batch/planner.py deleted file mode 100644 index 94d29ec..0000000 --- a/src/archipelago_tester/pipeline/batch/planner.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Splitting the work into known results and tests still to run.""" - -from archipelago_tester.core.model.blacklist import Blacklist -from archipelago_tester.core.model.name import Name -from archipelago_tester.core.state.keys import StateKeys -from archipelago_tester.pipeline.batch.cache_check import CacheCheck -from archipelago_tester.pipeline.batch.record import TestRecord - - -class Planner: - """What is already answered, and what has to be generated. - - The blacklist is checked before the cache and is unaffected by a - force flag: "do not retry" is the entire reason a world is listed, - so a flag that retried it would defeat the point. - """ - - def __init__(self, config, settings, tests, tag, roms_fingerprint, - force, repeats, random_repeats, companion_range, - releases=None): - self.config = config - self.settings = settings - self.tests = tests - self.tag = tag - self.roms_fingerprint = roms_fingerprint - self.force = force - self.repeats = repeats - self.random_repeats = random_repeats - self.companion_range = companion_range - self.blacklist = Blacklist(config) - # Keyed on the folded name: the work list names a row by - # its download folder, while game_releases keys on the - # sheet's own wording, and the two differ wherever the - # sheet uses a colon or its own capitalisation. - self.releases = { - Name(name).directory: (release or {}).get("published_at") - for name, release in (releases or {}).items() - } - self.wrote_state = False - - def record_for(self, item): - game_name, apworld_path, key, _ = item - return TestRecord(game_name, key, - is_core=apworld_path is None) - - def decline(self, item, reason): - """Write and return the result for a declined world.""" - _, _, key, current_hash = item - record = self.record_for(item) - self.tests[key] = record.blacklisted( - blacklist=self.blacklist, - current_hash=current_hash, - tag=self.tag, - reason=reason, - roms_fingerprint=self.roms_fingerprint, - ) - self.wrote_state = True - return record.as_result(self.tests[key]) - - def confirm(self, previous): - """Note that this answer was looked at and still holds. - - A cached result is not re-run, so its tested_at stays at the - run that reached it - which on a page refreshed today reads as - a world nobody has looked at in weeks rather than as a result - nothing has been able to change. This is the date that says - which of the two it is, and only a download failure was - recording it, so it was missing from every row that passed. - """ - previous[StateKeys.LAST_CHECKED_AT] = TestRecord.now() - self.wrote_state = True - - def known(self, item): - """This world's answer already, or None if it must be tested.""" - game_name, _, key, current_hash = item - reason = self.blacklist.reason_for(game_name) - if reason: - return self.decline(item, reason) - previous = self.tests.get(key) - cache = CacheCheck(self.settings, previous) - served = not self.force and cache.serves( - current_hash=current_hash, - tag=self.tag, - roms_fingerprint=self.roms_fingerprint, - repeats=self.repeats, - random_repeats=self.random_repeats, - companion_range=self.companion_range, - # A core world has no release of its own here, so this - # is None for one and the rule cannot fire: what dates - # it is the Archipelago build, already compared above. - released_at=self.releases.get(game_name), - ) - if not served: - return None - self.confirm(previous) - return self.record_for(item).as_result(previous) - - def plan(self, work): - results, to_test = [], [] - for item in work: - answer = self.known(item) - if answer is None: - to_test.append(item) - else: - results.append(answer) - return results, to_test diff --git a/src/archipelago_tester/pipeline/batch/record.py b/src/archipelago_tester/pipeline/batch/record.py deleted file mode 100644 index 79d1d5a..0000000 --- a/src/archipelago_tester/pipeline/batch/record.py +++ /dev/null @@ -1,111 +0,0 @@ -"""The state entry written for one world.""" - -from datetime import datetime, timezone - -from archipelago_tester.core.model.blacklist import Blacklist -from archipelago_tester.core.state.keys import StateKeys - - -class TestRecord: - """What a finished, declined or undownloadable world records. - - A core world has no file behind it, so it is marked as such and - carries no apworld path. - """ - - def __init__(self, game_name, key=None, is_core=False): - self.game_name = game_name - self.key = key - self.is_core = is_core - - @staticmethod - def now(): - return datetime.now(timezone.utc).isoformat() - - def cored(self, record): - if self.is_core: - record["stage"] = "core" - record["apworld"] = None - return record - - def from_report(self, report, current_hash, tag, internal_game, - roms_fingerprint): - """One finished test. - - Single and multi verdicts stay separate: different questions. - Every mode's own verdict is kept too, so adding one needs no - change here. "repaired" is true when the file had to be fixed - before it would load - a real result, but for a corrected copy - rather than the file the author published. - """ - return self.cored({ - "hash": current_hash, - "tag": tag, - "game": internal_game, - "outcome": report.get("outcome"), - "detail": report.get("detail"), - "elapsed_seconds": report.get("elapsed_seconds"), - "multi_outcome": report.get("multi_outcome"), - "multi_detail": report.get("multi_detail"), - "multi_elapsed_seconds": report.get("multi_elapsed_seconds"), - "multi_companions": report.get("multi_companions"), - "tests": report.get("tests"), - "roms_fingerprint": roms_fingerprint, - "tested_at": self.now(), - "repaired": bool(report.get("repaired")), - }) - - def blacklisted(self, blacklist, current_hash, tag, reason, - roms_fingerprint): - """A world this pipeline declines to test.""" - return self.cored({ - "hash": current_hash, - "tag": tag, - "game": self.game_name, - "outcome": Blacklist.OUTCOME, - "detail": blacklist.detail(reason), - "elapsed_seconds": None, - "multi_outcome": None, - "multi_detail": None, - "multi_elapsed_seconds": None, - "multi_companions": None, - "tests": None, - "roms_fingerprint": roms_fingerprint, - "tested_at": self.now(), - "blacklisted": True, - }) - - def download_failure(self, detail, tag): - """A game whose apworld could not be fetched. - - "unknown", never "failed": nothing was tested, so nothing is - known to be broken. Whether the link was a Discord message, an - unsupported host, or a release with no apworld in it, the - conclusion is the same - it could not be checked. - """ - return { - "game": self.game_name, - "apworld": None, - "outcome": "unknown", - "detail": detail, - "elapsed_seconds": None, - "stage": "download", - "tag": tag, - "hash": None, - "tested_at": self.now(), - StateKeys.LAST_CHECKED_AT: self.now(), - } - - def as_result(self, record): - """One entry of the caller-facing result list.""" - return { - "game": self.game_name, - "apworld": None if self.is_core else self.key, - "outcome": record.get("outcome"), - "detail": record.get("detail"), - "elapsed_seconds": record.get("elapsed_seconds"), - "multi_outcome": record.get("multi_outcome"), - "multi_detail": record.get("multi_detail"), - "tests": record.get("tests"), - "skipped": True, - } diff --git a/src/archipelago_tester/pipeline/batch/reporter.py b/src/archipelago_tester/pipeline/batch/reporter.py deleted file mode 100644 index d61063b..0000000 --- a/src/archipelago_tester/pipeline/batch/reporter.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Saying what each finished world did.""" - -from archipelago_tester.core.display.progress import Progress -from archipelago_tester.core.model.verdict import Verdict - - -class Reporter: - """One run's progress callback. - - Holds its own bar rather than a module variable: two runs in one - process would otherwise share a bar and a tally. - """ - - def __init__(self, config, bar=False): - self.config = config - self.wants_bar = bar - self.bar = None - - @staticmethod - def marker(report): - return ("cached" if report.get("skipped") - else f"{report.get('elapsed_seconds')}s") - - def verdicts(self, report): - text = f"{report['outcome']}" - multi = report.get("multi_outcome") - return f"{text} / multi:{multi}" if multi else text - - def drawn(self, index, total, report): - if self.bar is None or self.bar.total != total: - self.bar = Progress(total=total, config=self.config) - self.bar.update( - index=index, - key=report.get("outcome") or "?", - text=report.get("game") or "", - fallback_line=(f"[{index}/{total}] {report['game']}: " - f"{self.verdicts(report)} ({self.marker(report)})"), - ) - - def printed(self, index, total, report): - print(f"[{index}/{total}] {report.get('game')}: " - f"{Verdict(report, self.config).name} ({self.marker(report)})", - flush=True) - - def __call__(self, index, total, report): - if self.wants_bar: - self.drawn(index, total, report) - else: - self.printed(index, total, report) diff --git a/src/archipelago_tester/pipeline/batch/results.py b/src/archipelago_tester/pipeline/batch/results.py deleted file mode 100644 index 57410fd..0000000 --- a/src/archipelago_tester/pipeline/batch/results.py +++ /dev/null @@ -1,69 +0,0 @@ -"""The reports one run produced.""" - -from archipelago_tester.core.model.modes import Modes -from archipelago_tester.core.model.verdict import Verdict - - -class Results(list): - """A run's reports, printable as a table. - - A plain list of the same dicts the batch returns, so anything that - worked on those still works. The repr is the addition, because the - first thing anyone does with these is look at them. - """ - - def __init__(self, reports, config): - super().__init__(reports) - self.config = config - - def verdict(self, report): - return Verdict(report, self.config).name - - @property - def counts(self): - tally = {} - for report in self: - name = self.verdict(report) - tally[name] = tally.get(name, 0) + 1 - return dict(sorted(tally.items(), key=lambda item: -item[1])) - - @property - def summary(self): - """Outcome counts, multi verdicts under their own prefix. - - They are a separate question, so they are not merged into one - total. - """ - counts = {} - for report in self: - outcome = report.get("outcome") - counts[outcome] = counts.get(outcome, 0) + 1 - multi = report.get("multi_outcome") - if multi: - counts[f"multi:{multi}"] = counts.get(f"multi:{multi}", 0) + 1 - return counts - - def row(self, report, width): - """One world's line: name, verdict, every mode's outcome.""" - mode_results = report.get("tests") or {} - per_mode = " ".join( - f"{mode}={(mode_results.get(mode) or {}).get('outcome') or '-'}" - for mode in Modes.ALL - ) - marker = " (cached)" if report.get("skipped") else "" - return (f"{report.get('game') or '?':{width}s} " - f"{Verdict(report, self.config).label:12s} " - f"{per_mode}{marker}") - - def __repr__(self): - if not self: - return "no worlds tested" - width = max(len(report.get("game") or "") for report in self) - lines = [ - self.row(report, width) - for report in sorted(self, key=lambda r: r.get("game") or "") - ] - if len(self) > 1: - lines += ["", " ".join(f"{name}: {count}" - for name, count in self.counts.items())] - return "\n".join(lines) diff --git a/src/archipelago_tester/pipeline/batch/runner.py b/src/archipelago_tester/pipeline/batch/runner.py deleted file mode 100644 index 3d4a8d2..0000000 --- a/src/archipelago_tester/pipeline/batch/runner.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Running the tests the cache could not answer.""" - -import threading -from concurrent.futures import ThreadPoolExecutor, as_completed - -from archipelago_tester.pipeline.batch.record import TestRecord -from archipelago_tester.pipeline.generation.tester import WorldTester - - -class Runner: - """Tests each outstanding world and persists what it did.""" - - def __init__(self, config, paths, store, options, state, tests, tag, - image, roms_fingerprint): - self.config = config - self.paths = paths - self.store = store - self.options = options - self.state = state - self.tests = tests - self.tag = tag - self.tester = WorldTester(config, paths, image) - self.roms_fingerprint = roms_fingerprint - self.lock = threading.Lock() - - @staticmethod - def promotion_name(report, game_name): - """The name Archipelago itself registered this world under. - - Recorded for promotion's collision check, while the report's - own "game" becomes the sheet folder name for display. A world - that never loaded has no real name, so it falls back to a - marked sheet name - otherwise every such entry would collide - under one None key and none would promote. - """ - return report.get("game") or f"unverified:{game_name}" - - def store_record(self, key, record): - """Persist after every test, not just at the end, so a killed - run does not lose the tests it already completed. - """ - with self.lock: - self.tests[key] = record - self.store.save(self.state) - - def test_one(self, game_name, apworld_path, key, current_hash): - options = self.options - report = self.tester.test( - apworld_path=apworld_path, - game=None if apworld_path else game_name, - output_dir=options["output_directory"], - timeout=options["timeout"], - companion_range=options["companion_range"], - spoiler=options["spoiler"], - repeats=options["repeats"], - companion_pool=options["pool"], - random_repeats=options["random_repeats"], - ) - record = TestRecord(game_name, key, - is_core=apworld_path is None) - self.store_record(key, record.from_report( - report=report, - current_hash=current_hash, - tag=self.tag, - internal_game=self.promotion_name(report, game_name), - roms_fingerprint=self.roms_fingerprint, - )) - report["game"] = game_name - report["apworld"] = key - report["skipped"] = False - return report - - def run(self, to_test, total, completed, on_progress): - """Run the outstanding tests, reporting as they land.""" - results = [] - reporting = threading.Lock() - with ThreadPoolExecutor(max_workers=self.options["jobs"]) as pool: - futures = [pool.submit(self.test_one, *args) for args in to_test] - for future in as_completed(futures): - report = future.result() - results.append(report) - with reporting: - completed += 1 - if on_progress: - on_progress(completed, total, report) - return results diff --git a/src/archipelago_tester/pipeline/batch/session.py b/src/archipelago_tester/pipeline/batch/session.py deleted file mode 100644 index 415d765..0000000 --- a/src/archipelago_tester/pipeline/batch/session.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Testing worlds by hand, from the notebook or a shell.""" - -from archipelago_tester.core.config.config import Config -from archipelago_tester.core.config.paths import Paths -from archipelago_tester.core.config.run_settings import RunSettings -from archipelago_tester.core.state.lock import PipelineLock -from archipelago_tester.core.state.scratch import ScratchState -from archipelago_tester.core.state.store import StateStore -from archipelago_tester.pipeline.batch.apworld_finder import ApworldFinder -from archipelago_tester.pipeline.batch.batch import Batch -from archipelago_tester.pipeline.batch.fetcher import Fetcher -from archipelago_tester.pipeline.batch.reporter import Reporter -from archipelago_tester.pipeline.batch.results import Results -from archipelago_tester.pipeline.batch.version import Version -from archipelago_tester.pipeline.build.archipelago import ArchipelagoBuild - - -class TestSession: - """Test one world, several, or every downloaded one. - - `live` records into the real state file - off by default, because a - hand-run test is an experiment. `cached` only affects a whole-set - run; a named world is always retested. - """ - - def __init__(self, game=None, repeats=None, random_repeats=None, jobs=2, - live=False, cached=False, root_directory=None, quiet=False, - update=False): - self.config = Config.load() - self.paths = Paths(self.config) - self.settings = RunSettings(self.config) - self.game = game - self.repeats = repeats - self.random_repeats = random_repeats - self.jobs = jobs - self.live = live - self.cached = cached - self.root = root_directory or self.paths.downloads - self.quiet = quiet - self.update = update - - @property - def state_path(self): - return ScratchState(self.paths, live=self.live).path - - def worlds(self): - """The folders to test, downloading any that are missing. - - Resolved to real folder names and matched exactly from here on, - so the set tested is the set that was just named. - """ - if self.game is None: - return None - terms = [self.game] if isinstance(self.game, str) else list(self.game) - Fetcher(self.config, self.paths, self.root).ensure(terms) - finder = ApworldFinder(self.root) - return [name for term in terms for name in finder.resolve(term)] - - def announce(self, tag, source, state_path, only): - print(f"archipelago {tag} ({source})") - print(f"project {self.paths.project}") - print(f"downloads {self.root}") - scratch = "" if self.live else " (scratch, not the live record)" - print(f"results {state_path}{scratch}") - named = ", ".join(only) if only else "every downloaded apworld" - print(f"testing {named}") - - def run(self): - """The reports, printable as a table.""" - store = StateStore(self.paths.state) - build = ArchipelagoBuild(self.config, self.paths) - tag, source = Version(self.config, self.paths, store).prepare( - update=self.update) - only = self.worlds() - build.ensure_test_image(tag) - state_path = self.state_path - if not self.quiet: - self.announce(tag, source, state_path, only) - with PipelineLock(self.paths.lock): - return Results(Batch( - config=self.config, - paths=self.paths, - settings=self.settings, - image=build.test_image_tag(tag), - tag=tag, - force=not self.cached, - on_progress=None if self.quiet else Reporter(self.config), - root_directory=self.root, - state_path=state_path, - only=only, - exact_only=only is not None, - jobs=self.jobs, - repeats=self.repeats, - random_repeats=self.random_repeats, - ).run(), self.config) diff --git a/src/archipelago_tester/pipeline/batch/version.py b/src/archipelago_tester/pipeline/batch/version.py deleted file mode 100644 index 9590be4..0000000 --- a/src/archipelago_tester/pipeline/batch/version.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Which Archipelago version a run tests against.""" - -from archipelago_tester.core.state.keys import StateKeys -from archipelago_tester.pipeline.build.archipelago import ArchipelagoBuild -from archipelago_tester.pipeline.build.docker import Docker - - -class Version: - """Resolve the version, and build it if this machine lacks it.""" - - def __init__(self, config, paths, store): - self.paths = paths - self.store = store - self.build = ArchipelagoBuild(config, paths) - - def resolve(self): - """The version to test, and where that choice came from. - - A pin is a decision already made, so nothing else revisits it. - Failing that the recorded version wins, then whichever image is - on this machine - a test runs entirely inside the container, so - an image already here is everything it needs. - """ - pinned = self.build.pinned_version - if pinned: - return pinned, "pinned in config.json" - recorded = self.store.load().get(StateKeys.ARCHIPELAGO_TAG) - if recorded: - return recorded, "from state.json" - local = next(iter(self.build.local_tags()), None) - return (local, "newest image on this machine") if local else (None, - None) - - def ensure_built(self, tag): - """Build what this version needs, if anything is missing. - - Building needs the Archipelago source, which is the one job the - checkout has - so this is also the only path that clones. - """ - if tag is None: - print("no Archipelago image on this machine - cloning the " - "newest release to build one (several minutes) ...") - built, _, _, _ = self.build.update_and_build(self.store) - return built, "just built" - if not Docker.image_exists(self.build.image_tag(tag)): - print(f"Archipelago {tag} is not built on this machine - " - "cloning and building it (several minutes) ...") - self.build.build_version(tag) - return tag, None - - def note_others(self, tag, source): - """Say so when an unpinned run is not using the newest image. - - An unpinned tag can come from an image that happens to be lying - around, so a run can quietly answer for a version that is not - the current one. A pinned tag was chosen deliberately. - """ - local = self.build.local_tags() - if self.build.pinned_version or not local or tag == local[0]: - return source - return f"{source}, but {local[0]} is also built here" - - def update(self): - """Resolve the newest release, build it, and record it. - - Never automatic. A new version invalidates every cached result, - so the full re-test it forces is the caller's decision, not a - side effect of running. A pinned version stays pinned: the - resolution behind this honours the pin, so an update then only - ensures that version is built. - """ - tag, previous, changed, _ = self.build.update_and_build(self.store) - if self.build.pinned_version: - return tag, "pinned in config.json" - if not changed: - return tag, "already the newest release" - if previous is None: - return tag, "newest release, recorded for the first time" - return tag, f"updated from {previous}" - - def prepare(self, update=False): - """The version to test against, built if it is missing.""" - if update: - return self.update() - tag, source = self.resolve() - tag, built = self.ensure_built(tag) - return tag, built or self.note_others(tag, source) diff --git a/src/archipelago_tester/pipeline/batch/work_list.py b/src/archipelago_tester/pipeline/batch/work_list.py deleted file mode 100644 index 356adec..0000000 --- a/src/archipelago_tester/pipeline/batch/work_list.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Everything one run will consider testing.""" - -import os - -from archipelago_tester.core.model.fingerprint import Fingerprint -from archipelago_tester.pipeline.batch.apworld_finder import ApworldFinder - - -class WorkList: - """Each world to consider, with its cache key and hash. - - A core world is tested by name with no file behind it, so it is - keyed on the bare sheet name - the same "no slash means no apworld" - convention download failures follow - and gets a stand-in hash - whose real content is the Archipelago build beside it. - """ - - #: Stands in for a core world's file hash. Its real content is - #: the Archipelago build, so the version compared beside it in the - #: cache check is what decides staleness. - CORE_HASH = "core" - - def __init__(self, root, only=None, exact=False, core_games=()): - self.root = root - self.only = only - self.exact = exact - self.core_games = core_games - - def items(self): - found = ApworldFinder(self.root).find(only=self.only, - exact=self.exact) - work = [ - (name, path, os.path.relpath(path, self.root), - Fingerprint.of_file(path)) - for name, path in found - ] - return work + [ - (name, None, name, WorkList.CORE_HASH) - for name in self.core_games - if self.only is None or name in self.only - ] diff --git a/src/archipelago_tester/pipeline/download/__init__.py b/src/archipelago_tester/pipeline/download/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/pipeline/download/asset_matcher.py b/src/archipelago_tester/pipeline/download/asset_matcher.py deleted file mode 100644 index e7a8747..0000000 --- a/src/archipelago_tester/pipeline/download/asset_matcher.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Choosing which asset in a release belongs to this game.""" - -import re - -from archipelago_tester.core.model.name import Name - - -class AssetMatcher: - """Matches release assets and release titles to one sheet row.""" - - def __init__(self, config, game_name): - self.config = config - self.game = Name(game_name) - - def named(self, assets, exact): - """The assets whose filename names this game.""" - matched = [] - for asset in assets: - name = Name(asset["name"].rsplit(".", 1)[0]).slug - if (name == self.game.slug) if exact else (name in self.game.slug): - matched.append(asset) - return matched - - def select(self, assets): - """The assets to download for this game.""" - if len(assets) <= 1: - return assets - for matches in (self.named(assets, True), self.named(assets, False)): - if len(matches) == 1: - return matches - return assets - - def title_slug(self, release): - """A release's own title, reduced to the game it is for.""" - title = release.get("name") or release.get("tag_name") or "" - title = re.sub(r"\bv?\d+(?:\.\d+)*\b", " ", title) - title = re.sub(r"\bapworld\b", " ", title, flags=re.IGNORECASE) - return Name(title).slug - - def uninformative(self): - """Titles that name no game at all.""" - return set(self.config.value( - "downloads", "uninformative_release_slugs", ())) - - def claimed_by(self, title, sheet_names): - """The other sheet row this title names, or None. - - Exact equality only. This is not "the title looks a bit off" - - it is positive evidence that a DIFFERENT row is named, which is - the only thing strong enough to skip a release the link points - at. "Mega Man X2" is exactly a row and is skipped when resolving - "Mega Man X"; "Mega Man X1" is nobody's row and is accepted. - """ - if not sheet_names or not self.game.text: - return None - if not title or title in self.uninformative(): - return None - if title == self.game.slug: - return None - for other in sheet_names: - slug = Name(other).slug - if slug != self.game.slug and slug == title: - return other - return None - - def release_claimed(self, release, sheet_names): - return self.claimed_by(self.title_slug(release), sheet_names) - - def identity_claimed(self, found_games, sheet_names): - """Whether a downloaded file identifies as another row's game.""" - for found in found_games or []: - other = self.claimed_by(Name(found).slug, sheet_names) - if other: - return other - return None diff --git a/src/archipelago_tester/pipeline/download/auth_error.py b/src/archipelago_tester/pipeline/download/auth_error.py deleted file mode 100644 index 10dd5bb..0000000 --- a/src/archipelago_tester/pipeline/download/auth_error.py +++ /dev/null @@ -1,23 +0,0 @@ -"""A token failure, which is the whole run's problem.""" - -import requests - - -class GitHubAuthError(RuntimeError): - """A bad or expired token affects every remaining request.""" - - @classmethod - def matching(cls, game, error): - """This error as an auth failure, or None if it is not one. - - Raised loudly so the whole pool stops, rather than quietly - failing game after game against a token that will not work. - """ - if not isinstance(error, requests.HTTPError): - return None - response = error.response - status = response.status_code if response is not None else None - if status not in (401, 403): - return None - return cls(f"GitHub rejected the request for {game.name} with " - f"{status} - check .env / token expiry.") diff --git a/src/archipelago_tester/pipeline/download/context.py b/src/archipelago_tester/pipeline/download/context.py deleted file mode 100644 index a4671d5..0000000 --- a/src/archipelago_tester/pipeline/download/context.py +++ /dev/null @@ -1,61 +0,0 @@ -"""What every download shares.""" - -import os - -from archipelago_tester.core.model.name import Name -from archipelago_tester.pipeline.download.asset_matcher import AssetMatcher - - -class DownloadContext: - """Who is downloading what, and where it goes. - - One object rather than the same six arguments threaded through - every handler and helper. - """ - - def __init__(self, config, paths, session, game, identifier=None, - sheet_names=None, root_directory=None): - self.config = config - self.paths = paths - self.session = session - self.game = game - self.identifier = identifier - self.sheet_names = sheet_names - self.root = root_directory or paths.downloads - self.matcher = AssetMatcher(config, game.name) - - @property - def target_directory(self): - return os.path.join(self.root, Name(self.game.name).directory) - - def target_path(self, filename): - return os.path.join(self.target_directory, filename) - - def setting(self, key, default=None): - return self.config.value("downloads", key, default) - - @property - def plain_headers(self): - """Headers for a host that is not GitHub. - - The session carries a GitHub token, which must never be sent to - an unrelated site. - """ - return {"Authorization": None, "Accept": "*/*"} - - @property - def json_headers(self): - """Headers for another forge's JSON API, without the token.""" - return {"Authorization": None, "Accept": "application/json"} - - def wrong_game(self, path): - """Why this file is another game's, or None to keep it.""" - if self.identifier is None: - return None - identity = self.identifier.identify(path) - if not identity.get("ran"): - return None - found = identity.get("games") or [] - if any(Name(self.game.name).matches(name) for name in found): - return None - return found or identity.get("detail") diff --git a/src/archipelago_tester/pipeline/download/download_run.py b/src/archipelago_tester/pipeline/download/download_run.py deleted file mode 100644 index 3d452eb..0000000 --- a/src/archipelago_tester/pipeline/download/download_run.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Fetching every game's apworld, concurrently.""" - -import os -import threading -from concurrent.futures import ThreadPoolExecutor, as_completed - -import requests - -from archipelago_tester.core.state.keys import StateKeys -from archipelago_tester.core.state.store import StateStore -from archipelago_tester.pipeline.download.auth_error import GitHubAuthError -from archipelago_tester.pipeline.download.context import DownloadContext -from archipelago_tester.pipeline.download.downloader import Downloader -from archipelago_tester.pipeline.download.superseded_assets import ( - SupersededAssets, -) - - -class DownloadRun: - """One pass over the sheet's rows. - - `sheet_names` should be the FULL sheet: a file can identify as a - core row, which never reaches here. - """ - - def __init__(self, config, paths, session, root_directory=None, - identifier=None, jobs=None, sheet_names=None, - previous_releases=None): - self.config = config - self.paths = paths - self.session = session - self.root = root_directory or paths.downloads - self.identifier = identifier - self.jobs = jobs or int(config.value("downloads", "jobs", 8)) - self.sheet_names = sheet_names - self.previous_releases = previous_releases or {} - - @staticmethod - def split(games): - """The games worth a network call, and those answered now. - - A row with no link has nothing to look up, so it never takes a - slot in the thread pool. - """ - downloadable, skipped = [], [] - for game in games: - if not game.release and not game.links: - skipped.append((game.name, "no release link in the sheet")) - else: - downloadable.append(game) - return downloadable, skipped - - def context_for(self, game): - return DownloadContext( - config=self.config, - paths=self.paths, - session=self.session, - game=game, - identifier=self.identifier, - sheet_names=self.sheet_names, - root_directory=self.root, - ) - - def one(self, game): - """Download one game, which never raises for that game alone. - - A network failure is that game's own result, not the run's - - except an auth failure, which is everyone's. - """ - try: - downloaded, reason, info = Downloader( - context=self.context_for(game), - previous=self.previous_releases.get(game.name), - ).run() - except requests.RequestException as error: - auth = GitHubAuthError.matching(game, error) - if auth is not None: - raise auth from error - return game, None, str(error), None - return game, downloaded, reason, info - - @staticmethod - def source_url(game): - """Where this apworld came from, for the status page. - - game.release is only ever a real releases link, so for anything - resolved through the plain links the first one stands in. - """ - return game.release or (game.links[0] if game.links else None) - - def record(self, finished, releases, skipped, completed, total, - on_asset, on_game): - """Record one finished game and tell the caller about it. - - The release is recorded even on failure - it was still found, - so its version is worth showing. on_game is called for failures - too, unlike on_asset: a progress bar has to count every game - that finished, not only the ones that produced a file. - """ - game, downloaded, reason, info = finished - releases[game.name] = {"url": self.source_url(game), **(info or {})} - if downloaded is None: - skipped.append((game.name, reason)) - if on_game: - on_game(completed, total, game.name, False) - return 0 - paths = [path for path, _ in downloaded] - for path in SupersededAssets(os.path.dirname(paths[0]), - paths).prune(): - print(f"{game.name}: removed superseded {os.path.basename(path)}" - f" - the release no longer offers it") - for path, status in downloaded: - if on_asset: - on_asset(game.name, path, status) - if on_game: - on_game(completed, total, game.name, True) - return len(downloaded) - - @staticmethod - def result_of(future, executor): - """One finished download, stopping the pool on auth failure.""" - try: - return future.result() - except GitHubAuthError: - executor.shutdown(cancel_futures=True) - raise - - def all(self, games, on_asset=None, on_game=None): - """Every game's apworld: how many files, what was skipped, and - which release each row resolved to. - - Recording holds a lock, so on_asset always runs serialized and - a caller's own progress counter needs none of its own. - """ - self.sheet_names = self.sheet_names or [g.name for g in games] - downloadable, skipped = self.split(games) - releases, lock = {}, threading.Lock() - total = completed = 0 - with ThreadPoolExecutor(max_workers=self.jobs) as executor: - futures = [executor.submit(self.one, g) for g in downloadable] - for future in as_completed(futures): - finished = self.result_of(future, executor) - with lock: - completed += 1 - total += self.record( - finished, releases, skipped, completed, - len(downloadable), on_asset, on_game) - return total, skipped, releases - - def record_releases(self, releases, state_path=None): - """Write each game's resolved release into the state file.""" - store = StateStore(state_path or self.paths.state) - state = store.load() - state.setdefault(StateKeys.GAME_RELEASES, {}).update(releases) - store.save(state) - return state diff --git a/src/archipelago_tester/pipeline/download/downloader.py b/src/archipelago_tester/pipeline/download/downloader.py deleted file mode 100644 index 597689d..0000000 --- a/src/archipelago_tester/pipeline/download/downloader.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Fetching one sheet row's apworld.""" - -from archipelago_tester.pipeline.download.handlers.gitea_release import ( - GiteaRelease, -) -from archipelago_tester.pipeline.download.handlers.github_file import ( - GitHubFile, -) -from archipelago_tester.pipeline.download.handlers.github_release import ( - GitHubRelease, -) -from archipelago_tester.pipeline.download.handlers.github_repo import ( - GitHubRepo, -) -from archipelago_tester.pipeline.download.handlers.gitlab_file import ( - GitLabFile, -) -from archipelago_tester.pipeline.download.handlers.gitlab_release import ( - GitLabRelease, -) -from archipelago_tester.pipeline.download.handlers.plain_file import PlainFile -from archipelago_tester.pipeline.download.unusable_links import UnusableLinks -from archipelago_tester.pipeline.download.url_shapes import UrlShapes - - -class Downloader: - """Whatever shape this game's link is, get the apworld. - - A releases link is resolved directly; anything else is tried - against every handler in turn, most specific first, so an exact - file beats browsing a whole repo and the plain-URL handler can - never shadow a forge-specific one. - """ - - def __init__(self, context, previous=None): - self.context = context - self.shapes = UrlShapes(context.config) - self.previous = previous - - def handlers(self): - return ( - GitHubFile(self.shapes), - GitHubRepo(self.shapes), - GitLabFile(self.shapes), - GitLabRelease(self.shapes), - GiteaRelease(self.shapes, previous=self.previous), - PlainFile(self.shapes), - ) - - def from_release_link(self): - handler = GitHubRelease(self.shapes, previous=self.previous) - parsed = handler.parse(self.context.game.release) - if parsed is None: - return None, "not a recognized releases URL", None - return handler.download(parsed, self.context) - - def from_link(self, link): - """This link's result, or None if no handler recognised it.""" - for handler in self.handlers(): - parsed = handler.parse(link) - if parsed is not None: - return handler.download(parsed, self.context) - return None - - def run(self): - """The files downloaded, why none were, and the release info.""" - game = self.context.game - if game.release: - return self.from_release_link() - for link in game.links: - result = self.from_link(link) - if result is not None: - return result - return None, UnusableLinks(self.context.config, game.links).detail(), \ - None diff --git a/src/archipelago_tester/pipeline/download/handlers/__init__.py b/src/archipelago_tester/pipeline/download/handlers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/pipeline/download/handlers/base.py b/src/archipelago_tester/pipeline/download/handlers/base.py deleted file mode 100644 index e8283dd..0000000 --- a/src/archipelago_tester/pipeline/download/handlers/base.py +++ /dev/null @@ -1,45 +0,0 @@ -"""What every link handler shares.""" - -import os - -from archipelago_tester.pipeline.download.local_file import LocalFile - - -class LinkHandler: - """One recognised shape of sheet link, and how to fetch from it. - - parse() returning None means "not my shape", which is different from - "mine, and it failed": the caller only moves on to the next link in - the first case. - """ - - def __init__(self, shapes): - self.shapes = shapes - - def parse(self, link): - raise NotImplementedError - - def download(self, parsed, context): - raise NotImplementedError - - def file(self, context, filename): - return LocalFile(context.session, context.target_path(filename)) - - def collect(self, candidates, context, fetch, verify): - """Download each candidate, dropping any that is another game.""" - downloaded, kept, rejected = [], [], [] - for candidate in candidates: - path, status = fetch(candidate) - if status in LocalFile.NEEDS_DOWNLOAD and verify: - mismatch = context.wrong_game(path) - if mismatch is not None: - os.remove(path) - rejected.append((candidate["name"], mismatch)) - continue - downloaded.append((path, status)) - kept.append(candidate) - return downloaded, kept, rejected - - def rejected_detail(self, rejected, subject): - details = "; ".join(f"{name} -> {found}" for name, found in rejected) - return f"downloaded {subject} didn't match this game: {details}" diff --git a/src/archipelago_tester/pipeline/download/handlers/gitea_release.py b/src/archipelago_tester/pipeline/download/handlers/gitea_release.py deleted file mode 100644 index 483ab8a..0000000 --- a/src/archipelago_tester/pipeline/download/handlers/gitea_release.py +++ /dev/null @@ -1,44 +0,0 @@ -"""A releases page on a self-hosted Gitea or Forgejo instance.""" - -from archipelago_tester.pipeline.download.handlers.base import LinkHandler -from archipelago_tester.pipeline.download.release_downloader import ( - ReleaseDownloader, -) - - -class GiteaRelease(LinkHandler): - """Codeberg and friends. - - The release JSON is close enough to GitHub's that ReleaseDownloader - handles it unchanged, given headers that omit the GitHub token. - """ - - def __init__(self, shapes, previous=None): - super().__init__(shapes) - self.previous = previous - - def parse(self, link): - return self.shapes.gitea_releases(link) - - def latest(self, context, host, owner, repository): - response = context.session.get( - f"https://{host}/api/v1/repos/{owner}/{repository}" - "/releases/latest", - headers=context.json_headers, - ) - if response.status_code == 404: - return None - response.raise_for_status() - return response.json() - - def download(self, parsed, context): - host, owner, repository = parsed - release = self.latest(context, host, owner, repository) - if release is None: - return None, "no release found on this host", None - return ReleaseDownloader( - context=context, - release=release, - headers=context.plain_headers, - previous_published_at=(self.previous or {}).get("published_at"), - ).run() diff --git a/src/archipelago_tester/pipeline/download/handlers/github_file.py b/src/archipelago_tester/pipeline/download/handlers/github_file.py deleted file mode 100644 index 9f46027..0000000 --- a/src/archipelago_tester/pipeline/download/handlers/github_file.py +++ /dev/null @@ -1,58 +0,0 @@ -"""A link straight at an apworld committed in a GitHub repo.""" - -import os -import urllib.parse - -import requests - -from archipelago_tester.pipeline.build.archipelago import ( - ArchipelagoBuild, -) -from archipelago_tester.pipeline.download.local_file import LocalFile -from archipelago_tester.pipeline.download.handlers.base import LinkHandler - - -class GitHubFile(LinkHandler): - """A committed file, named exactly by the sheet. - - No identity check: the link names one file, so there is no second - candidate a name comparison could choose between. - """ - - def parse(self, link): - return self.shapes.github_file(link) - - def download(self, parsed, context): - """Fetch the file as it currently stands on that branch.""" - owner, repository, branch, path = parsed - filename = os.path.basename(urllib.parse.unquote(path)) - local = self.file(context, filename) - url = ("https://raw.githubusercontent.com/" - f"{owner}/{repository}/{branch}/{path}") - status = local.status(local.remote_size(url)) - if status in LocalFile.NEEDS_DOWNLOAD: - local.download(url) - published = self.published_at(context, owner, repository, path, branch) - return [(local.path, status)], None, {"published_at": published} - - def published_at(self, context, owner, repository, path, branch): - """When this file was last committed, or None. - - Best effort: a blob link has no release to read a date from, and - a download should not fail over display information. - """ - try: - response = context.session.get( - f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}" - "/commits", - params={"path": path, "sha": branch, "per_page": 1}, - ) - response.raise_for_status() - commits = response.json() - except requests.RequestException: - return None - if not commits: - return None - commit = commits[0].get("commit", {}) - return ((commit.get("committer") or {}).get("date") - or (commit.get("author") or {}).get("date")) diff --git a/src/archipelago_tester/pipeline/download/handlers/github_release.py b/src/archipelago_tester/pipeline/download/handlers/github_release.py deleted file mode 100644 index fc63289..0000000 --- a/src/archipelago_tester/pipeline/download/handlers/github_release.py +++ /dev/null @@ -1,81 +0,0 @@ -"""A GitHub releases page, the sheet's usual link.""" - -from archipelago_tester.pipeline.download.handlers.base import LinkHandler -from archipelago_tester.pipeline.download.release_downloader import ( - ReleaseDownloader, -) -from archipelago_tester.pipeline.download.release_identity import ( - ReleaseIdentity, -) -from archipelago_tester.pipeline.download.release_picker import ReleasePicker -from archipelago_tester.pipeline.download.walk_back import WalkBack - - -class GitHubRelease(LinkHandler): - """Resolve which release the link means, then fetch its apworld.""" - - def __init__(self, shapes, previous=None): - super().__init__(shapes) - self.previous = previous - - def parse(self, link): - return self.shapes.releases(link) - - def resolve(self, parsed, context): - """The release this link resolves to, or None. - - A hand-pinned tag prefix wins, then a link naming one exact - tag, then the search term. Runs every pass: it is the one call - that can report a new release, which is the only thing able to - change a previously failed answer. - """ - owner, repository, term, tag = parsed - picker = ReleasePicker(context, owner, repository, term) - overrides = context.config.get("release_overrides") or {} - prefix = overrides.get(context.game.name) - if prefix: - return picker.by_tag_prefix(prefix) - if tag: - return picker.by_tag(tag) - return picker.pick() - - def fetch(self, parsed, context, release): - """This release's apworld, walking back if it ships none. - - Only that one cause is walked back from. "Ships another game's - apworld" must not be, since an older release of a multi-world - repo is precisely the wrong answer there. - """ - owner, repository = parsed[0], parsed[1] - downloader = ReleaseDownloader( - context=context, - release=release, - # What this row's date already says, so a re-upload of - # unchanged bytes keeps it instead of moving it forward. - previous_published_at=(self.previous or {}).get("published_at"), - ) - downloaded, reason, info = downloader.run() - if (downloaded is not None or context.identifier is None - or downloader.cause != ReleaseDownloader.NO_APWORLD): - return downloaded, reason, info - fallback, reason = WalkBack( - context=context, - owner=owner, - repository=repository, - release=release, - ).run(reason) - return fallback if fallback is not None else (None, reason, info) - - def download(self, parsed, context): - release = self.resolve(parsed, context) - if release is None: - return None, "no release matched the search term in the link", None - identity = ReleaseIdentity(release) - repeat = identity.repeated(self.previous) - if repeat is not None: - return None, repeat, identity.failure( - {"published_at": None}, repeat) - downloaded, reason, info = self.fetch(parsed, context, release) - if downloaded is None: - return None, reason, identity.failure(info, reason) - return downloaded, reason, info diff --git a/src/archipelago_tester/pipeline/download/handlers/github_repo.py b/src/archipelago_tester/pipeline/download/handlers/github_repo.py deleted file mode 100644 index 333e351..0000000 --- a/src/archipelago_tester/pipeline/download/handlers/github_repo.py +++ /dev/null @@ -1,115 +0,0 @@ -"""A bare repository link, for a world never cut into a release.""" - -import os - -import requests - -from archipelago_tester.pipeline.build.archipelago import ( - ArchipelagoBuild, -) -from archipelago_tester.pipeline.download.local_file import LocalFile -from archipelago_tester.core.model.apworld import ApworldFile -from archipelago_tester.pipeline.download.handlers.base import LinkHandler - - -class GitHubRepo(LinkHandler): - """Browses the repo's own file tree for an apworld.""" - - def parse(self, link): - return self.shapes.github_repo(link) - - def resolve_branch(self, context, owner, repository, branch): - """The real default branch, so a version reads as a name. - - "HEAD" works as a ref, but resolving it means the status page - shows a branch rather than the literal string. - """ - if branch != "HEAD": - return branch - try: - response = context.session.get( - f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}") - response.raise_for_status() - return response.json().get("default_branch") or "HEAD" - except requests.RequestException: - return "HEAD" - - def tree_entries(self, context, owner, repository, branch): - """Every apworld in the tree, with its size. - - One call: the trees API takes recursive=1 itself, and returns - each blob's size for free. - """ - try: - response = context.session.get( - f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}" - f"/git/trees/{branch}", - params={"recursive": "1"}, - ) - response.raise_for_status() - except requests.RequestException: - return [] - return [ - (entry["path"], entry.get("size")) - for entry in response.json().get("tree", []) - if entry.get("type") == "blob" - and entry["path"].lower().endswith(ApworldFile.SUFFIX) - ] - - def candidates(self, entries): - return [ - {"name": os.path.basename(path), "path": path, "size": size} - for path, size in entries - ] - - def download(self, parsed, context): - owner, repository, branch = parsed - branch = self.resolve_branch(context, owner, repository, branch) - entries = self.tree_entries(context, owner, repository, branch) - if not entries: - return None, ("repo link has no release and no .apworld file " - "found in it"), None - chosen = context.matcher.select(self.candidates(entries)) - downloaded, kept, rejected = self.collect( - candidates=chosen, - context=context, - fetch=lambda candidate: self.fetch( - candidate, context, owner, repository, branch), - verify=len(chosen) > 1, - ) - if not downloaded: - detail = self.rejected_detail(rejected, "file(s) from repo") - return None, detail, None - published = self.published_at( - context, owner, repository, kept[0]["path"], branch) - return downloaded, None, {"published_at": published} - - def fetch(self, candidate, context, owner, repository, branch): - local = self.file(context, candidate["name"]) - status = local.status(candidate.get("size")) - if status in LocalFile.NEEDS_DOWNLOAD: - local.download( - f"https://raw.githubusercontent.com/{owner}/{repository}" - f"/{branch}/{candidate['path']}") - return local.path, status - - def published_at(self, context, owner, repository, path, branch): - """When the first kept file was last committed. - - Only the first: one API call per candidate otherwise. - """ - try: - response = context.session.get( - f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}" - "/commits", - params={"path": path, "sha": branch, "per_page": 1}, - ) - response.raise_for_status() - commits = response.json() - except requests.RequestException: - return None - if not commits: - return None - commit = commits[0].get("commit", {}) - return ((commit.get("committer") or {}).get("date") - or (commit.get("author") or {}).get("date")) diff --git a/src/archipelago_tester/pipeline/download/handlers/gitlab_file.py b/src/archipelago_tester/pipeline/download/handlers/gitlab_file.py deleted file mode 100644 index ded473c..0000000 --- a/src/archipelago_tester/pipeline/download/handlers/gitlab_file.py +++ /dev/null @@ -1,44 +0,0 @@ -"""A link straight at an apworld committed in a GitLab repo.""" - -import os -import urllib.parse - -import requests - -from archipelago_tester.pipeline.download.local_file import LocalFile -from archipelago_tester.pipeline.download.handlers.base import LinkHandler - - -class GitLabFile(LinkHandler): - """A committed file on GitLab, named exactly by the sheet.""" - - def parse(self, link): - return self.shapes.gitlab_file(link) - - def download(self, parsed, context): - project_path, branch, path = parsed - filename = os.path.basename(urllib.parse.unquote(path)) - local = self.file(context, filename) - url = f"https://gitlab.com/{project_path}/-/raw/{branch}/{path}" - headers = context.json_headers - status = local.status(local.remote_size(url, headers=headers)) - if status in LocalFile.NEEDS_DOWNLOAD: - local.download(url, headers=headers) - published = self.published_at(context, project_path, path, branch) - return [(local.path, status)], None, {"published_at": published} - - def published_at(self, context, project_path, path, branch): - """When this file was last committed, or None.""" - encoded = urllib.parse.quote(project_path, safe="") - try: - response = context.session.get( - f"https://gitlab.com/api/v4/projects/{encoded}" - "/repository/commits", - params={"path": path, "ref_name": branch, "per_page": 1}, - headers=context.json_headers, - ) - response.raise_for_status() - commits = response.json() - except requests.RequestException: - return None - return commits[0].get("committed_date") if commits else None diff --git a/src/archipelago_tester/pipeline/download/handlers/gitlab_release.py b/src/archipelago_tester/pipeline/download/handlers/gitlab_release.py deleted file mode 100644 index 0913cfa..0000000 --- a/src/archipelago_tester/pipeline/download/handlers/gitlab_release.py +++ /dev/null @@ -1,74 +0,0 @@ -"""A GitLab releases page.""" - -import urllib.parse - -from archipelago_tester.core.model.apworld import ApworldFile -from archipelago_tester.pipeline.download.handlers.base import LinkHandler -from archipelago_tester.pipeline.download.local_file import LocalFile - - -class GitLabRelease(LinkHandler): - """The newest release of a GitLab project. - - GitLab's assets are shaped nothing like GitHub's - a release-level - list of {name, url, direct_asset_url} with no per-asset timestamp - - so they are reshaped here rather than reusing the release path. - """ - - def parse(self, link): - return self.shapes.gitlab_releases(link) - - def latest(self, context, project_path): - """GitLab's own newest-release shortcut, or None.""" - encoded = urllib.parse.quote(project_path, safe="") - response = context.session.get( - f"https://gitlab.com/api/v4/projects/{encoded}" - "/releases/permalink/latest", - headers=context.json_headers, - ) - if response.status_code == 404: - return None - response.raise_for_status() - return response.json() - - def assets(self, release, context): - """The release's apworld links, in GitHub's asset shape.""" - links = (release.get("assets") or {}).get("links", []) - return [ - { - "name": link["name"], - "browser_download_url": (link.get("direct_asset_url") - or link["url"]), - } - for link in links - if (link.get("name") or "").lower().endswith(ApworldFile.SUFFIX) - ] - - def fetch(self, asset, context): - local = self.file(context, asset["name"]) - url = asset["browser_download_url"] - headers = context.plain_headers - status = local.status(local.remote_size(url, headers=headers)) - if status in LocalFile.NEEDS_DOWNLOAD: - local.download(url, headers=headers) - return local.path, status - - def download(self, parsed, context): - release = self.latest(context, parsed) - if release is None: - return None, "no GitLab release found for this project", None - info = {"published_at": (release.get("released_at") - or release.get("created_at"))} - assets = self.assets(release, context) - if not assets: - return None, "latest GitLab release has no .apworld asset", info - chosen = context.matcher.select(assets) - downloaded, _, rejected = self.collect( - candidates=chosen, - context=context, - fetch=lambda asset: self.fetch(asset, context), - verify=len(chosen) > 1, - ) - if not downloaded: - return None, self.rejected_detail(rejected, "asset(s)"), info - return downloaded, None, info diff --git a/src/archipelago_tester/pipeline/download/handlers/plain_file.py b/src/archipelago_tester/pipeline/download/handlers/plain_file.py deleted file mode 100644 index 81f2984..0000000 --- a/src/archipelago_tester/pipeline/download/handlers/plain_file.py +++ /dev/null @@ -1,48 +0,0 @@ -"""A plain URL ending in .apworld, on any host.""" - -import email.utils - -import requests - -from archipelago_tester.pipeline.download.local_file import LocalFile -from archipelago_tester.pipeline.download.handlers.base import LinkHandler - - -class PlainFile(LinkHandler): - """Any other host. Tried last, so a forge handler always wins.""" - - def parse(self, link): - return self.shapes.plain_file(link) - - def download(self, parsed, context): - url, filename = parsed - local = self.file(context, filename) - size, published = self.remote_facts(context, url) - status = local.status(size) - if status in LocalFile.NEEDS_DOWNLOAD: - local.download(url, headers=context.plain_headers) - return [(local.path, status)], None, {"published_at": published} - - def remote_facts(self, context, url): - """The size and date one HEAD request can offer. - - A plain URL has no version to compare, so size is the whole - staleness check. Neither is required. - """ - try: - response = context.session.head( - url, headers=context.plain_headers, allow_redirects=True) - response.raise_for_status() - except requests.RequestException: - return None, None - length = response.headers.get("Content-Length") - size = int(length) if length and length.isdigit() else None - return size, self.modified_at(response.headers.get("Last-Modified")) - - def modified_at(self, modified): - if not modified: - return None - try: - return email.utils.parsedate_to_datetime(modified).isoformat() - except (TypeError, ValueError): - return None diff --git a/src/archipelago_tester/pipeline/download/local_file.py b/src/archipelago_tester/pipeline/download/local_file.py deleted file mode 100644 index 36b6da8..0000000 --- a/src/archipelago_tester/pipeline/download/local_file.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Deciding whether a local copy is still the current one.""" - -import datetime -import os - -import requests - -from archipelago_tester.core.model.instant import Instant - - -class LocalFile: - """One file on disk, against what the remote currently offers. - - "It exists" is not "it is current": an author can replace an - asset's content in place, same filename, same version. Two signals - catch that - the asset's own timestamp, and its size - and both are - needed. A replacement that happens to be the same length is - invisible to size alone, and a remote that reports no length at all - makes size no signal whatsoever. - """ - - #: The statuses that mean the file has to be fetched. The other - #: two - "verified" and "assumed" - mean the copy on disk stands. - NEEDS_DOWNLOAD = ("missing", "stale") - - def __init__(self, session, path): - self.session = session - self.path = path - - def outdated(self, remote_updated_at): - """Whether the remote was touched after this copy arrived. - - mtime is when download() wrote this file, so a remote timestamp - later than it means the copy on disk predates what is being - offered now. Without this, a release that replaced its asset - with one of the same length was never fetched, and the recorded - test result went on describing content that no longer exists - - while the page showed the new release's date beside it. - """ - stamp = Instant.of(remote_updated_at) - if stamp is None: - return False - arrived = datetime.datetime.fromtimestamp( - os.path.getmtime(self.path), datetime.timezone.utc) - return arrived < stamp - - def status(self, remote_size, remote_updated_at=None): - """Whether this file is missing, stale, verified or assumed. - - The timestamp is checked before the size, and before a missing - size falls through to "assumed": it is the stronger signal, and - the two cases it catches are exactly the ones size cannot. - """ - if not os.path.exists(self.path): - return "missing" - if self.outdated(remote_updated_at): - return "stale" - if remote_size is None: - return "assumed" - if os.path.getsize(self.path) == remote_size: - return "verified" - return "stale" - - def remote_size(self, url, headers=None): - """The size a HEAD request reports, or None.""" - try: - response = self.session.head(url, headers=headers, - allow_redirects=True) - response.raise_for_status() - except requests.RequestException: - return None - length = response.headers.get("Content-Length") - return int(length) if length and length.isdigit() else None - - def download(self, url, headers=None): - """Stream a URL into this path, creating its directory.""" - response = self.session.get(url, stream=True, headers=headers) - response.raise_for_status() - os.makedirs(os.path.dirname(self.path), exist_ok=True) - with open(self.path, "wb") as handle: - for chunk in response.iter_content(chunk_size=65536): - handle.write(chunk) - return self.path diff --git a/src/archipelago_tester/pipeline/download/release_downloader.py b/src/archipelago_tester/pipeline/download/release_downloader.py deleted file mode 100644 index 4e61ede..0000000 --- a/src/archipelago_tester/pipeline/download/release_downloader.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Fetching the apworld a release carries.""" - -import os - -from archipelago_tester.core.model.apworld import ApworldFile -from archipelago_tester.core.model.fingerprint import Fingerprint -from archipelago_tester.pipeline.download.local_file import LocalFile -from archipelago_tester.pipeline.download.zip_search import ZipSearch - - -class ReleaseDownloader: - """A GitHub-shaped release, and the apworld it ships. - - Shared with Gitea and Forgejo, whose release JSON is close enough - that the same logic applies unchanged. - """ - - #: Why run() came back empty. The caller decides what to do about - #: it from this, never from the message beside it: a release that - #: ships no apworld can be walked back from, and one that ships - #: another game's must not be. - NO_APWORLD = "no_apworld" - WRONG_GAME = "wrong_game" - - NO_APWORLD_DETAIL = "latest release has no .apworld asset" - - def __init__(self, context, release, headers=None, - require_identity=False, previous_published_at=None): - self.context = context - self.release = release - self.headers = headers - self.require_identity = require_identity - self.previous_published_at = previous_published_at - self.cause = None - - def named_assets(self): - return [ - asset for asset in self.release.get("assets", []) - if asset["name"].lower().endswith(ApworldFile.SUFFIX) - ] - - def missing_detail(self): - detail = self.NO_APWORLD_DETAIL - if any(asset["name"].lower().endswith(ZipSearch.SUFFIX) - for asset in self.release.get("assets", [])): - detail += " (checked inside its .zip asset(s) too, none found)" - return detail - - def assets(self): - """Every apworld this release offers, zips included.""" - found = self.named_assets() - if found: - return found - return ZipSearch(self.context, self.release, self.headers).assets() - - def verifies(self, assets): - """Whether a file has to prove which game it is. - - With one candidate there is nothing to choose between, and the - sheet's link points at this release - so that file is the row's - apworld whatever its World class calls itself. A walk-back - release overrides that, since the link no longer points at it. - """ - return len(assets) > 1 or self.require_identity - - @staticmethod - def touched_at(asset): - """When this asset was last written, as the remote reports it. - - The same value published_at() reports for the row, read here so - the two cannot disagree: a file whose recorded release date is - newer than the copy on disk is a file that was never fetched. - """ - return asset.get("updated_at") or asset.get("created_at") - - def fetch(self, asset): - """The asset's path, how it got there, and whether it changed. - - An asset whose timestamp moved is fetched again - the copy on - disk can no longer be trusted to be current - and only then can - the two be compared. Identical bytes mean the author re-uploaded - the same file, so nothing about this row actually changed. A - file lifted out of a zip is taken as changed: ZipSearch does its - own fetching, and there is no earlier copy here to compare to. - """ - if "extracted_path" in asset: - return asset["extracted_path"], asset["status"], True - local = LocalFile(self.context.session, - self.context.target_path(asset["name"])) - url = asset["browser_download_url"] - status = local.status( - local.remote_size(url, headers=self.headers), - remote_updated_at=self.touched_at(asset), - ) - if status not in LocalFile.NEEDS_DOWNLOAD: - return local.path, status, False - before = (Fingerprint.of_file(local.path) - if os.path.exists(local.path) else None) - local.download(url, headers=self.headers) - return local.path, status, Fingerprint.of_file(local.path) != before - - def rejection(self, path, assets): - """Why this file is not ours, or None to keep it. - - A walk-back file that registered no world at all is kept: - nothing was proven either way, and generation testing reports - the import failure far more usefully than a silent drop. - """ - identity = self.context.identifier.identify(path) - if not identity.get("ran"): - return None - found = identity.get("games") or [] - if self.require_identity and not found and len(assets) == 1: - return None - other = self.context.matcher.identity_claimed( - found, self.context.sheet_names) - matches = bool(found) and self.context.matcher.game.matches( - found[0]) - if other: - return f'{found} - belongs to the "{other}" row' - if self.verifies(assets) and not matches: - return found or identity.get("detail") - return None - - def checks(self, status, assets): - return (status in LocalFile.NEEDS_DOWNLOAD - and self.context.identifier is not None - and (self.verifies(assets) or self.context.sheet_names)) - - def published_at(self, kept): - """When the content this row now holds was published. - - Each asset carries its own timestamp; a release's own stays - fixed at creation even when its files are replaced later, so - the assets' is the one to read. But that timestamp moves for a - re-upload of identical bytes too, and then it dates the upload - rather than the world: the page would show a release newer than - the test beside it while the tested file never changed, and the - Compatibility column would call a world current on the strength - of an upload that added nothing. So only an asset whose content - actually changed sets this date, and otherwise the one already - recorded stands. With nothing recorded there is nothing to - hold, and the asset's own timestamp is the best available. - """ - stamps = [(self.touched_at(asset), changed) - for asset, changed in kept] - stamps = [pair for pair in stamps if pair[0]] - replaced = [stamp for stamp, changed in stamps if changed] - if replaced: - return max(replaced) - if self.previous_published_at: - return self.previous_published_at - return max((stamp for stamp, _ in stamps), default=None) - - def run(self): - """The files downloaded, why none were, and the release info.""" - assets = self.assets() - if not assets: - self.cause = self.NO_APWORLD - return None, self.missing_detail(), {"published_at": None} - assets = self.context.matcher.select(assets) - downloaded, kept, rejected = [], [], [] - for asset in assets: - path, status, changed = self.fetch(asset) - reason = (self.rejection(path, assets) - if self.checks(status, assets) else None) - if reason is not None: - os.remove(path) - rejected.append((asset["name"], reason)) - continue - downloaded.append((path, status)) - kept.append((asset, changed)) - if not downloaded: - self.cause = self.WRONG_GAME - details = "; ".join(f"{name} -> {found}" - for name, found in rejected) - return None, (f"downloaded asset(s) didn't match this game: " - f"{details}"), {"published_at": None} - return downloaded, None, {"published_at": self.published_at(kept)} diff --git a/src/archipelago_tester/pipeline/download/release_identity.py b/src/archipelago_tester/pipeline/download/release_identity.py deleted file mode 100644 index 500f79b..0000000 --- a/src/archipelago_tester/pipeline/download/release_identity.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Remembering which release a failure was decided on.""" - -from archipelago_tester.core.state.keys import StateKeys - - -class ReleaseIdentity: - """What makes a resolved release the same one as last run. - - Its tag and publication date, plus every asset's name and size. The - sizes matter because an author can replace an asset in place - without cutting a new tag. - """ - - def __init__(self, release): - self.release = release - - @property - def value(self): - assets = sorted( - (asset.get("name"), asset.get("size")) - for asset in self.release.get("assets") or [] - ) - return { - "tag": self.release.get("tag_name"), - "published_at": self.release.get("published_at"), - "assets": [{"name": name, "size": size} for name, size in assets], - } - - def repeated(self, previous): - """The recorded reason, if this exact release failed before. - - A release that ships no apworld, or only another row's, fails - the same way every run - and finding that out costs a download - of every zip asset plus a container run to identify what came - back. None of that can reach a different answer while the - release is unchanged. Keyed on the release rather than the - link, because the link is usually /releases/latest and a newly - published release is exactly what would fix a world in this - state. Only ever holds a verdict about content: a network error - escapes as an exception and a container that could not start - keeps the file, so neither is cached here. - """ - if not previous: - return None - if previous.get(StateKeys.FAILED_RELEASE) != self.value: - return None - return previous.get(StateKeys.FAILED_REASON) or None - - def failure(self, info, reason): - """A failure recorded against the release it was decided on.""" - return { - **(info or {}), - StateKeys.FAILED_RELEASE: self.value, - StateKeys.FAILED_REASON: reason, - } diff --git a/src/archipelago_tester/pipeline/download/release_picker.py b/src/archipelago_tester/pipeline/download/release_picker.py deleted file mode 100644 index fb8156e..0000000 --- a/src/archipelago_tester/pipeline/download/release_picker.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Which release a sheet link means.""" - -import re -import urllib.parse - -from archipelago_tester.pipeline.build.archipelago import ( - ArchipelagoBuild, -) -from archipelago_tester.core.model.apworld import ApworldFile -from archipelago_tester.core.model.name import Name - - -class ReleasePicker: - """One repository's releases, narrowed to this game's. - - A repo dedicated to one world can answer with its latest release. A - repo hosting several has to be matched on the link's search term and - the release's own title, and must never answer with a release that - another sheet row already claims. - """ - - def __init__(self, context, owner, repository, term=None): - self.context = context - self.owner = owner - self.repository = repository - self.term = term - self.matcher = context.matcher - - @property - def repo_url(self): - return (f"{ArchipelagoBuild.API_ROOT}/repos/" - f"{self.owner}/{self.repository}") - - def all(self, max_pages=10): - """Every release, newest first, across every page. - - GitHub pages at 30 by default and several source repos hold - more than that, which would silently hide real releases from - every lookup below. - """ - releases = [] - for page in range(1, max_pages + 1): - response = self.context.session.get( - f"{self.repo_url}/releases", - params={"per_page": 100, "page": page}, - ) - response.raise_for_status() - batch = response.json() - releases.extend(batch) - if len(batch) < 100: - break - return releases - - def by_tag(self, tag): - """The release for exactly this tag, or None.""" - encoded = urllib.parse.quote(tag, safe="") - response = self.context.session.get( - f"{self.repo_url}/releases/tags/{encoded}") - if response.status_code == 404: - return None - response.raise_for_status() - return response.json() - - def by_tag_prefix(self, prefix): - """The newest release whose tag starts with this prefix.""" - for release in self.all(): - if (release.get("tag_name") or "").startswith(prefix): - return release - return None - - def claimed(self, release): - return self.matcher.release_claimed(release, self.context.sheet_names) - - def newest(self): - """GitHub's own newest full release. - - Right whenever it resolves, since it skips in-progress - prereleases. The list's first entry is the fallback for a repo - where every release is flagged prerelease, which /latest - excludes rather than relaxing. - """ - response = self.context.session.get(f"{self.repo_url}/releases/latest") - if response.status_code != 404: - response.raise_for_status() - return response.json() - releases = self.all() - return releases[0] if releases else None - - def titled_for_this_row(self): - for release in self.all(): - if self.matcher.title_slug(release) == self.matcher.game.slug: - return release - return None - - def without_a_term(self): - """The release to use when the link named none. - - "Newest in the repo" is a fine answer for a repo holding one - world and a coin flip for one holding twelve, so a newest that - is titled for somebody else's row gives way to the one titled - for this row. - """ - latest = self.newest() - if latest is None or not self.claimed(latest): - return latest - return self.titled_for_this_row() or latest - - def by_title(self, releases): - """The first release whose title holds every word of the term. - - The sheet's term is built for GitHub's own search box, which - matches per word: requiring the whole phrase rejects real terms - like "Jurassic Park Randomizer (SNES) Jurassic Park", where the - link generator repeats the game name. That looseness is what - lets a term land on a sibling world, so a release titled for - another row is skipped and the search continues. - """ - words = re.findall(r"[a-z0-9]+", self.term.lower()) - if not words: - return None - for release in releases: - title = (f"{release.get('name') or ''} " - f"{release.get('tag_name') or ''}").lower() - if all(word in title for word in words) \ - and not self.claimed(release): - return release - return None - - def by_asset_name(self, releases): - """The release carrying an asset named for this game. - - A fork mirroring many worlds often tags every release by build - date alone, so the game's name appears only in which apworlds - the release bundles. - """ - term = Name(self.term).slug - for release in releases: - for asset in release.get("assets", []): - name = asset["name"] - if not name.lower().endswith(ApworldFile.SUFFIX): - continue - if Name(name.rsplit(".", 1)[0]).slug == term: - return release - return None - - def pick(self): - """The release this link means, or None.""" - if self.term is None: - return self.without_a_term() - releases = self.all() - return self.by_title(releases) or self.by_asset_name(releases) diff --git a/src/archipelago_tester/pipeline/download/superseded_assets.py b/src/archipelago_tester/pipeline/download/superseded_assets.py deleted file mode 100644 index 26a46e5..0000000 --- a/src/archipelago_tester/pipeline/download/superseded_assets.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Clearing apworlds a game's release no longer offers.""" - -import os - - -class SupersededAssets: - """The files left behind when an author renames their asset. - - A download writes each asset under its own filename, so a release - that renamed one leaves the previous name sitting beside it. Both - are then real files in the game's folder, and everything downstream - keys on the path rather than the game - two work items, two test - records, two rows on the status page, and a promotion that refuses - both because they claim one game name between them. - - Only ever run for a game whose download succeeded: with nothing - fetched there is no current set to compare against, and pruning on - a failed network call would delete the copy that still works. - """ - - #: What this class considers. A game's folder holds nothing else, - #: but naming it here keeps a stray file from being deleted on the - #: strength of living in the wrong directory. - SUFFIX = ".apworld" - - def __init__(self, directory, kept): - self.directory = directory - self.kept = {os.path.abspath(path) for path in kept} - - def stale(self): - """Every apworld in the folder that this download did not write.""" - if not self.kept or not os.path.isdir(self.directory): - return [] - found = [] - for entry in sorted(os.listdir(self.directory)): - if not entry.lower().endswith(self.SUFFIX): - continue - path = os.path.join(self.directory, entry) - if os.path.isfile(path) and os.path.abspath(path) not in self.kept: - found.append(path) - return found - - def prune(self): - """Delete them, returning what went. - - A file that cannot be removed is not worth failing a download - over - it means the next run sees a duplicate, which is what - was already happening. - """ - removed = [] - for path in self.stale(): - try: - os.remove(path) - except OSError as error: - print(f"could not remove superseded {path}: {error}") - continue - removed.append(path) - return removed diff --git a/src/archipelago_tester/pipeline/download/unusable_links.py b/src/archipelago_tester/pipeline/download/unusable_links.py deleted file mode 100644 index 4043777..0000000 --- a/src/archipelago_tester/pipeline/download/unusable_links.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Why none of a game's links could be downloaded from.""" - -import urllib.parse - - -class UnusableLinks: - """Categorise links no handler recognised. - - Reaching here means every recognised shape was tried against every - link, so the status page can say something concrete instead of a - generic "failed". - """ - - def __init__(self, config, links): - self.config = config - self.links = links - - @property - def unreachable(self): - """Hosts known in advance never to be downloadable. - - A Discord message link is not a public file however it is - fetched, unlike an unknown host that merely has no handler yet. - """ - return self.config.value("downloads", "unreachable_link_hosts") or {} - - def domains(self): - """Each link's host, or a reason it has none.""" - found = [] - for link in self.links: - parsed = urllib.parse.urlparse(link) - if parsed.scheme not in ("http", "https"): - protocol = parsed.scheme or "(none)" - return None, f"unsupported link protocol {protocol!r}: {link}" - if not parsed.netloc: - return None, f"not a valid URL: {link}" - netloc = parsed.netloc.lower() - found.append(netloc[4:] if netloc.startswith("www.") else netloc) - return found, None - - def detail(self): - domains, problem = self.domains() - if problem is not None: - return problem - for domain in domains: - if domain in self.unreachable: - return self.unreachable[domain] - if "github.com" in domains: - return ("GitHub link found, but not a recognized " - "releases/repo/file URL") - if "gitlab.com" in domains: - return "GitLab link found, but not a recognized releases/blob URL" - hosts = ", ".join(sorted(set(domains))) - return f"no supported download source (custom host: {hosts})" diff --git a/src/archipelago_tester/pipeline/download/update_history.py b/src/archipelago_tester/pipeline/download/update_history.py deleted file mode 100644 index 2596b54..0000000 --- a/src/archipelago_tester/pipeline/download/update_history.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Every distinct apworld file this pipeline has seen, per game. - -One file per game, written only when that game gains content it has not -seen before - so a run touches the handful of games that actually -changed rather than rewriting one shared blob. Deliberately NOT under -downloads/: that directory is a cache and gets cleared to force a clean -re-download, which is harmless for everything else in it. History is the -one thing that cannot be rebuilt afterwards - which version existed on -which date has no other source once it is gone. -""" - -import os -from datetime import datetime, timezone - -from archipelago_tester.core.model.apworld import ApworldFile -from archipelago_tester.core.model.fingerprint import Fingerprint -from archipelago_tester.core.model.name import Name -from archipelago_tester.core.state.keys import StateKeys -from archipelago_tester.core.state.store import StateStore - - -class UpdateHistory: - """What each game's apworld was, and when. - - Identity is the file's sha256, so a re-download of unchanged content - adds nothing and recording is safe to repeat within a run. The - game's sheet name is stored inside the file as well as encoded in - its name, because the filename is a folded directory name and a - sheet rename would otherwise leave a file nothing could identify. - """ - - def __init__(self, paths, directory=None): - self.directory = directory or paths.history - - def path_for(self, game_name): - folded = Name(game_name).directory - return os.path.join(self.directory, f"{folded}.json") - - def load(self, game_name): - return StateStore.read( - self.path_for(game_name), - {"game": game_name, "watching_since": None, - "baseline_at": None, "files": []}) - - def save(self, game_name, history): - os.makedirs(self.directory, exist_ok=True) - StateStore(self.path_for(game_name)).save(history) - - #: Everything recorded before this field existed was gathered from - #: whatever the hosts happened to say, at whatever moment the file - #: was first downloaded, and none of it measures how often a world - #: changes under observation. The first pass to touch a game draws - #: a line: that instant is the game's one starting point, and only - #: changes seen after it count. See UpdateCadence, which reads it. - @staticmethod - def watching_since(history, stamp): - """When this pipeline began watching this game. - - The earliest content it has already recorded, so a file written - before this field existed still gets an honest answer rather - than a window that starts the day the field was added. Only a - game with no history at all starts watching now. - """ - seen = [item.get("first_seen_at") for item in history["files"] - if item.get("first_seen_at")] - return min(seen) if seen else stamp - - def entry_for(self, apworld_path, sha256, released_at, source_url, now): - """One file's record. - - `released_at` is the upload date the download stage saw; - first_seen_at is when this pipeline observed it, which is the - only honest timestamp for a file whose host reports no date. - """ - manifest = ApworldFile(apworld_path).manifest or {} - return { - "version": manifest.get("world_version"), - "released_at": released_at, - "first_seen_at": (now or datetime.now(timezone.utc)).isoformat(), - "sha256": sha256, - "filename": os.path.basename(apworld_path), - "source_url": source_url, - } - - def record_file(self, game_name, apworld_path, sha256, released_at, - source_url=None, now=None): - """Add this file to the game's history if its content is new. - - A game whose content has not changed still records that it was - looked at. How often a world updates is only meaningful against - how long it has been watched, and without that a game seen once - and a game seen all year are indistinguishable. - """ - history = self.load(game_name) - now = now or datetime.now(timezone.utc) - opened = not history.get("watching_since") - if opened: - history["watching_since"] = self.watching_since( - history, now.isoformat()) - if not history.get("baseline_at"): - opened = True - history["baseline_at"] = now.isoformat() - if any(item.get("sha256") == sha256 for item in history["files"]): - if opened: - self.save(game_name, history) - return None - entry = self.entry_for(apworld_path, sha256, released_at, - source_url, now) - history["game"] = game_name - history["files"].append(entry) - # Oldest first. released_at is missing often enough that it - # cannot be the only key - first_seen_at always exists and - # breaks those ties in the order the pipeline saw them. - history["files"].sort( - key=lambda item: (item.get("released_at") or "", - item.get("first_seen_at") or "")) - self.save(game_name, history) - return entry - - def files_in(self, game_directory): - return [ - name for name in sorted(os.listdir(game_directory)) - if name.lower().endswith(ApworldFile.SUFFIX) - and os.path.isfile(os.path.join(game_directory, name)) - ] - - def record_all(self, state, root_directory, now=None): - """Record every apworld on disk that is not yet in history. - - Hashes the files directly rather than reading apworld_tests: - that dict only gains a hash once a world has been TESTED, so a - file downloaded this run would not appear until the next one - - and this runs right after downloading, which is the point at - which a new version is known. - """ - releases = state.get(StateKeys.GAME_RELEASES) or {} - # game_releases is keyed by the sheet name while the download - # folder is that name folded - map one to the other rather than - # assuming they match as written. - by_folder = {Name(name).directory: info - for name, info in releases.items()} - added = [] - if not os.path.isdir(root_directory): - return added - for folder in sorted(os.listdir(root_directory)): - game_directory = os.path.join(root_directory, folder) - if not os.path.isdir(game_directory): - continue - added += self.record_folder( - folder, game_directory, by_folder.get(folder) or {}, now) - return added - - def record_folder(self, folder, game_directory, release, now): - """Every new apworld in one game's download folder.""" - added = [] - for filename in self.files_in(game_directory): - path = os.path.join(game_directory, filename) - entry = self.record_file( - game_name=folder, - apworld_path=path, - sha256=Fingerprint.of_file(path), - released_at=release.get("published_at"), - source_url=release.get("url"), - now=now, - ) - if entry: - added.append((folder, entry)) - return added diff --git a/src/archipelago_tester/pipeline/download/url_shapes.py b/src/archipelago_tester/pipeline/download/url_shapes.py deleted file mode 100644 index 2a2bec4..0000000 --- a/src/archipelago_tester/pipeline/download/url_shapes.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Recognising the shapes a sheet link can take.""" - -import os -import re -import urllib.parse - -from archipelago_tester.core.model.apworld import ApworldFile - - -class UrlShapes: - """The link shapes this pipeline knows how to fetch from.""" - - def __init__(self, config): - self.config = config - - def releases(self, url): - """Owner, repo, search term and tag from a releases link. - - A link naming one exact tag is read as such: without it, a repo - hosting several worlds would fall back to "latest", which drifts - to whichever world was tagged most recently. - """ - match = re.match( - r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+)/releases", - url or "", re.IGNORECASE) - if match is None: - return None - parsed = urllib.parse.urlparse(url) - terms = urllib.parse.parse_qs(parsed.query).get("q", []) - tag_match = re.match( - r"^/[^/]+/[^/]+/releases/tag/([^/?#]+)", parsed.path, - re.IGNORECASE) - return ( - match.group(1), - match.group(2), - terms[0].strip('"') if terms else None, - urllib.parse.unquote(tag_match.group(1)) if tag_match else None, - ) - - def github_file(self, url): - """Owner, repo, branch and path from a committed-file link.""" - match = re.match( - r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+)" - r"/(?:blob|raw)/([^/]+)/(.+\.apworld)$", - url or "", re.IGNORECASE) - return match.groups() if match else None - - def github_repo(self, url): - """Owner, repo and branch from a bare repository link. - - Deliberately strict, so it does not swallow issues or wiki - links: those should keep falling through to the generic "not a - releases URL" message rather than triggering a tree search that - could never succeed. - """ - match = re.match( - r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+?)" - r"(?:/tree/([^/?#]+))?/?$", - url or "", re.IGNORECASE) - if match is None: - return None - owner, repository, branch = match.groups() - return owner, repository, branch or "HEAD" - - def gitlab_file(self, url): - """Project path, branch and file path from a GitLab blob link.""" - match = re.match( - r"^https?://(?:www\.)?gitlab\.com/([^/]+(?:/[^/]+)*?)" - r"/-/blob/([^/]+)/(.+\.apworld)$", - url or "", re.IGNORECASE) - return match.groups() if match else None - - def gitlab_releases(self, url): - """The project path from a GitLab releases link.""" - match = re.match( - r"^https?://(?:www\.)?gitlab\.com/([^/]+(?:/[^/]+)*?)" - r"/-/releases(?:[/?#]|$)", - url or "", re.IGNORECASE) - return match.group(1) if match else None - - def gitea_releases(self, url): - """Host, owner and repo from a Gitea-style releases link.""" - match = re.match( - r"^https?://(?!(?:www\.)?(?:github|gitlab)\.com)" - r"([^/]+)/([^/]+)/([^/]+)/releases(?:[/?#]|$)", - url or "", re.IGNORECASE) - return match.groups() if match else None - - def plain_file(self, url): - """A URL ending in .apworld on any host, and its filename.""" - parsed = urllib.parse.urlparse(url or "") - if parsed.scheme not in ("http", "https") or not parsed.netloc: - return None - path = urllib.parse.unquote(parsed.path) - if not path.lower().endswith(ApworldFile.SUFFIX): - return None - filename = os.path.basename(path) - return (url, filename) if filename != ApworldFile.SUFFIX else None diff --git a/src/archipelago_tester/pipeline/download/walk_back.py b/src/archipelago_tester/pipeline/download/walk_back.py deleted file mode 100644 index 7afee7e..0000000 --- a/src/archipelago_tester/pipeline/download/walk_back.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Falling back to an older release that still ships an apworld.""" - -import requests - -from archipelago_tester.core.model.apworld import ApworldFile -from archipelago_tester.pipeline.download.release_downloader import ( - ReleaseDownloader, -) -from archipelago_tester.pipeline.download.release_picker import ReleasePicker - - -class WalkBack: - """One narrow rescue: the newest release forgot the apworld. - - An author cuts a release and re-attaches only a mod zip, leaving - the world perfectly downloadable one release back. Every other - failure is left alone on purpose - in particular "did not match - this game", where an older release holding a different game's - apworld is precisely the wrong answer. - """ - - def __init__(self, context, owner, repository, release): - self.context = context - self.owner = owner - self.repository = repository - self.release = release - - def candidate(self, release): - """Whether this is an older release that ships an apworld. - - Matches on asset filename alone: this runs over a repo's whole - history, and opening every zip in it would cost far more than - the problem is worth. - """ - cutoff = self.release.get("published_at") - published_at = release.get("published_at") - return bool( - not release.get("draft") - and release.get("tag_name") != self.release.get("tag_name") - and published_at - and (not cutoff or published_at < cutoff) - and any(asset["name"].lower().endswith(ApworldFile.SUFFIX) - for asset in release.get("assets", [])) - ) - - def older(self): - """The newest earlier release with an apworld, by date. - - Ordering comes from published_at rather than the order GitHub - returns, which follows the underlying tag and is not the same - thing. - """ - picker = ReleasePicker(self.context, self.owner, self.repository) - try: - releases = picker.all() - except requests.RequestException: - return None - candidates = [r for r in releases if self.candidate(r)] - if not candidates: - return None - return max(candidates, key=lambda release: release["published_at"]) - - def run(self, reason): - """The fallback's result, or None and a combined reason.""" - older = self.older() - if older is None: - return None, reason - fallback = ReleaseDownloader( - context=self.context, - release=older, - require_identity=True, - ).run() - if fallback[0] is not None: - fallback[2]["fallback_release_tag"] = older.get("tag_name") - return fallback, None - return None, (f"{reason}; older release {older.get('tag_name')!r} " - f"was tried too: {fallback[1]}") diff --git a/src/archipelago_tester/pipeline/download/zip_search.py b/src/archipelago_tester/pipeline/download/zip_search.py deleted file mode 100644 index fb5fcd2..0000000 --- a/src/archipelago_tester/pipeline/download/zip_search.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Looking for an apworld inside a release's zip assets.""" - -import os -import tempfile -import zipfile - -import requests - -from archipelago_tester.core.model.apworld import ApworldFile -from archipelago_tester.pipeline.download.local_file import LocalFile - - -class ZipSearch: - """For a release that publishes a zip instead of an apworld. - - Some repos attach a source snapshot with the built apworld nested - inside. Every zip is re-downloaded to scratch space each time, since - the member's own filename - what a normal download caches on - is - not known until the zip is opened. - """ - - #: The extension a release's archive asset is recognised by. - SUFFIX = ".zip" - - def __init__(self, context, release, headers=None): - self.context = context - self.release = release - self.headers = headers - - def zip_assets(self): - return [ - asset for asset in self.release.get("assets", []) - if asset["name"].lower().endswith(ZipSearch.SUFFIX) - ] - - def members(self, zip_path): - """Every apworld inside the zip, with its size.""" - try: - with zipfile.ZipFile(zip_path) as archive: - return [ - (info.filename, info.file_size) - for info in archive.infolist() - if not info.filename.endswith("/") - and info.filename.lower().endswith(ApworldFile.SUFFIX) - ] - except zipfile.BadZipFile: - return [] - - def extract(self, zip_path, member): - """Unpack one member into the game's folder.""" - target = self.context.target_path(os.path.basename(member)) - os.makedirs(os.path.dirname(target), exist_ok=True) - with zipfile.ZipFile(zip_path) as archive: - with archive.open(member) as source: - with open(target, "wb") as handle: - handle.write(source.read()) - return target - - def unpack(self, zip_path, members, zip_asset): - """The members as assets, already on disk.""" - results = [] - for member, size in members: - target = self.context.target_path(os.path.basename(member)) - status = LocalFile(self.context.session, target).status(size) - if status in LocalFile.NEEDS_DOWNLOAD: - self.extract(zip_path, member) - results.append({ - "name": os.path.basename(member), - "updated_at": zip_asset.get("updated_at"), - "created_at": zip_asset.get("created_at"), - "extracted_path": target, - "status": status, - }) - return results - - def search_one(self, zip_asset): - with tempfile.TemporaryDirectory() as scratch: - zip_path = os.path.join(scratch, zip_asset["name"]) - local = LocalFile(self.context.session, zip_path) - try: - local.download(zip_asset["browser_download_url"], - headers=self.headers) - except requests.RequestException: - return [] - members = self.members(zip_path) - if not members: - return [] - return self.unpack(zip_path, members, zip_asset) - - def assets(self): - """The first zip that holds an apworld wins.""" - for zip_asset in self.zip_assets(): - found = self.search_one(zip_asset) - if found: - return found - return [] diff --git a/src/archipelago_tester/pipeline/generation/__init__.py b/src/archipelago_tester/pipeline/generation/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/pipeline/generation/command.py b/src/archipelago_tester/pipeline/generation/command.py deleted file mode 100644 index 2f4f7f2..0000000 --- a/src/archipelago_tester/pipeline/generation/command.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Building the docker invocations the drivers run in.""" - -import os - -from archipelago_tester.core.model.apworld import ApworldFile - - -class ContainerCommand: - """The `docker run` for one driver, and the mounts it needs. - - The paths below are where this pipeline mounts things inside its - own container. Both ends of that contract live in this repository - - these mount arguments and the drivers that receive them - so they - are part of the code rather than something to configure. - """ - - #: Where each driver script is mounted. - DRIVER_PATH = "/app/run_test.py" - IDENTIFY_DRIVER_PATH = "/app/identify_apworld.py" - VERIFY_DRIVER_PATH = "/app/verify_companions.py" - - #: Where the apworld under test, the seeds and the ROMs go. - APWORLD_DIRECTORY = "/app/custom_worlds" - OUTPUT_DIRECTORY = "/data/output" - DATA_DIRECTORY = "/app/data" - - #: A writable cache directory. The image runs as uid 1000 but - #: leaves HOME as "/", which is root-owned, so anything resolving - #: "~/.cache" fails with PermissionError before generation starts. - CACHE_ENVIRONMENT = {"HOME": "/tmp", "XDG_CACHE_HOME": "/tmp/.cache"} - - def __init__(self, config, paths, image): - self.config = config - self.paths = paths - self.image = image - - def timeout(self, name, default): - return int(self.config.value("testing", name, default)) - - @property - def prefix(self): - """The part of every invocation that never varies.""" - return ["docker", "run", "--rm", "--network", "none", - "--entrypoint", "python", *self.cache_environment] - - @property - def cache_environment(self): - args = [] - for name, value in self.CACHE_ENVIRONMENT.items(): - args += ["-e", f"{name}={value}"] - return args - - @property - def common_client_mount(self): - """CommonClient.py, mounted back in from the checkout. - - The image excludes every client file, correctly - but a few - apworlds import their own during generation and fail with "No - module named 'CommonClient'" purely because it is absent. - Absent source means no mount: it exists only after a clone. - """ - if not os.path.isfile(self.paths.common_client): - return [] - return ["-v", f"{self.paths.common_client}:/app/CommonClient.py:ro"] - - @property - def rom_mounts(self): - """Every base ROM, mounted individually. - - Per file rather than mounting the directory over /app/data, - which would shadow the image's own shipped data. Worlds declare - rom_file inconsistently - some embed "data/", some do not - so - each is mounted at both locations. - """ - directory = self.paths.roms - if not os.path.isdir(directory): - return [] - data_in = self.DATA_DIRECTORY - args = [] - for name in sorted(os.listdir(directory)): - path = os.path.join(directory, name) - if os.path.isfile(path): - args += ["-v", f"{path}:{data_in}/{name}:ro"] - args += ["-v", f"{path}:/app/{name}:ro"] - return args - - def apworld_mount(self, apworld_path): - """The mount and container path for the world under test.""" - if not apworld_path: - return [], None - name = ApworldFile(apworld_path).importable_name - inside = f"{self.APWORLD_DIRECTORY}/{name}" - return ["-v", f"{apworld_path}:{inside}:ro"], inside - - def driver_mounts(self, output_dir): - """The driver, the output directory, the ROMs and the client.""" - return [ - "-v", f"{self.paths.driver}:{self.DRIVER_PATH}:ro", - "-v", f"{output_dir}:{self.OUTPUT_DIRECTORY}", - *self.rom_mounts, - *self.common_client_mount, - ] diff --git a/src/archipelago_tester/pipeline/generation/companion_check.py b/src/archipelago_tester/pipeline/generation/companion_check.py deleted file mode 100644 index 9f41fe1..0000000 --- a/src/archipelago_tester/pipeline/generation/companion_check.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Which core worlds can actually generate in this container.""" - -import os -import subprocess - -from archipelago_tester.core.config.run_settings import RunSettings -from archipelago_tester.pipeline.generation.command import ContainerCommand -from archipelago_tester.pipeline.generation.report import DriverReport - - -class CompanionCheck: - """Tries every core world once, so the rest can trust the pool. - - Several core worlds need a base ROM this container has no copy of, - and drawing one fails the seed - with the failure landing on the - apworld under test rather than on the companion that caused it. - """ - - def __init__(self, config, paths, image): - self.config = config - self.paths = paths - self.image = image - self.command = ContainerCommand(config, paths, image) - - def build(self, output_dir, per_world_timeout, spoiler): - verify_in = ContainerCommand.VERIFY_DRIVER_PATH - return [ - *self.command.prefix, - "-v", f"{self.paths.verify_driver}:{verify_in}:ro", - "-v", (f"{self.paths.driver}:" - f"{ContainerCommand.DRIVER_PATH}:ro"), - "-v", (f"{output_dir}:" - f"{ContainerCommand.OUTPUT_DIRECTORY}"), - *self.command.rom_mounts, - *self.command.common_client_mount, - self.image, - verify_in, - "--output-dir", ContainerCommand.OUTPUT_DIRECTORY, - "--timeout", str(per_world_timeout), - "--spoiler", str(spoiler), - ] - - @staticmethod - def not_verified(detail): - return {"ran": False, "verified": [], "rejected": {}, - "detail": detail} - - def parse(self, result): - data = DriverReport(result).data - if data is not None: - data.setdefault("ran", True) - return data - return self.not_verified( - f"no JSON output (exit {result.returncode}): " - f"{DriverReport.tail(result.stderr)}") - - def run(self, output_dir, timeout=None, per_world_timeout=None, - spoiler=None): - """The pool of usable companions, or why there is none.""" - timeout = timeout or self.command.timeout("verify_timeout", 1800) - per_world_timeout = per_world_timeout or self.command.timeout( - "verify_per_world_timeout", 120) - if spoiler is None: - spoiler = RunSettings(self.config).spoiler - output_dir = os.path.abspath(output_dir) - os.makedirs(output_dir, exist_ok=True) - try: - result = subprocess.run( - self.build(output_dir, per_world_timeout, spoiler), - capture_output=True, - text=True, - timeout=timeout, - ) - except subprocess.TimeoutExpired: - return self.not_verified("companion verification did not " - f"finish within {timeout}s") - return self.parse(result) diff --git a/src/archipelago_tester/pipeline/generation/identifier.py b/src/archipelago_tester/pipeline/generation/identifier.py deleted file mode 100644 index 470d2cc..0000000 --- a/src/archipelago_tester/pipeline/generation/identifier.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Asking an apworld which game it registers as.""" - -import os -import subprocess -import tempfile - -from archipelago_tester.core.model.apworld import ApworldFile -from archipelago_tester.pipeline.generation.command import ContainerCommand -from archipelago_tester.pipeline.generation.report import DriverReport - - -class ApworldIdentifier: - """What a file's own World class says it is. - - Lighter than a test: no ROMs, no output volume, no generation. Used - right after download to check the content matches the row that - fetched it, before the file is trusted at all. - """ - - def __init__(self, config, paths, image): - self.paths = paths - self.image = image - self.command = ContainerCommand(config, paths, image) - - def build(self, apworld_path): - mount, inside = self.command.apworld_mount(apworld_path) - identify_in = ContainerCommand.IDENTIFY_DRIVER_PATH - return [ - *self.command.prefix, - *mount, - "-v", f"{self.paths.identify_driver}:{identify_in}:ro", - *self.command.common_client_mount, - self.image, - identify_in, - "--apworld", inside, - ] - - def failed(self, detail): - return {"ran": False, "games": [], "detail": detail} - - def identify(self, apworld_path, timeout=None): - """Which games this file registers, and whether the check ran.""" - timeout = timeout or self.command.timeout("identify_timeout", 60) - apworld_path = os.path.abspath(apworld_path) - if not os.path.isfile(apworld_path): - return self.failed(f"{apworld_path} does not exist (not a file)") - try: - result = self.run(apworld_path, timeout) - except subprocess.TimeoutExpired: - return self.failed( - f"identify check did not finish within {timeout}s") - data = DriverReport(result).data - if data is not None: - data.setdefault("ran", True) - return data - return self.failed( - f"no JSON output (exit {result.returncode}): " - f"{DriverReport.tail(result.stderr)}") - - def run(self, apworld_path, timeout): - """Run the driver against a prepared copy of the file. - - Repaired here as well as at test time, so a backslash-separated - world is not rejected at download for registering no game and - never reaching the test that could repair it. - """ - with tempfile.TemporaryDirectory(prefix="apworld-id-") as workdir: - checked, _ = ApworldFile(apworld_path).prepare(workdir) - return subprocess.run(self.build(checked), capture_output=True, - text=True, timeout=timeout) diff --git a/src/archipelago_tester/pipeline/generation/report.py b/src/archipelago_tester/pipeline/generation/report.py deleted file mode 100644 index 3b13c41..0000000 --- a/src/archipelago_tester/pipeline/generation/report.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Reading what a driver reported.""" - -import json - - -class DriverReport: - """The JSON one driver run wrote to stdout.""" - - def __init__(self, result): - self.result = result - - @staticmethod - def tail(text, lines=20): - return "\n".join(text.splitlines()[-lines:]) - - @staticmethod - def last_json_object(text): - """The driver's report, however it ended up on the line. - - Not "the last line starting with {": worlds write to raw stdout - during generation, and one that writes without a trailing - newline leaves its output glued to the front of the report - - Super Metroid's randomizer emits a bare "*" per retry, so the - report arrives as "*{...}" and would read as no report at all. - """ - for line in reversed(text.splitlines()): - line = line.strip() - start = line.find("{") - while start != -1: - try: - return json.loads(line[start:]) - except json.JSONDecodeError: - start = line.find("{", start + 1) - return None - - @property - def data(self): - return self.last_json_object(self.result.stdout) - - def parsed(self): - """The report, or a failure built from what it printed.""" - report = self.data - if report is not None: - report.setdefault("stderr_tail", self.tail(self.result.stderr)) - return report - return { - "game": None, - "outcome": "failed", - "detail": ("no JSON report on stdout " - f"(exit code {self.result.returncode})"), - "elapsed_seconds": None, - "stdout_tail": self.tail(self.result.stdout), - "stderr_tail": self.tail(self.result.stderr), - } diff --git a/src/archipelago_tester/pipeline/generation/tester.py b/src/archipelago_tester/pipeline/generation/tester.py deleted file mode 100644 index 83a857e..0000000 --- a/src/archipelago_tester/pipeline/generation/tester.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Testing one world by generating seeds for it.""" - -import os -import subprocess -import tempfile - -from archipelago_tester.core.model.apworld import ApworldFile -from archipelago_tester.pipeline.generation.command import ContainerCommand -from archipelago_tester.pipeline.generation.report import DriverReport - - -class WorldTester: - """Runs every generation mode against one world.""" - - def __init__(self, config, paths, image): - self.config = config - self.paths = paths - self.image = image - self.command = ContainerCommand(config, paths, image) - - def settings(self, key, default): - return self.config.value("testing", key, default) - - def build(self, apworld_path, output_dir, timeout, companion_range, - spoiler, repeats, companion_pool, game, random_repeats): - """The full `docker run` for one world's whole test.""" - mount, inside = self.command.apworld_mount(apworld_path) - selector = ["--apworld", inside] if inside else ["--game", game] - return [ - *self.command.prefix, - *mount, - *self.command.driver_mounts(output_dir), - self.image, - ContainerCommand.DRIVER_PATH, - *selector, - "--output-dir", ContainerCommand.OUTPUT_DIRECTORY, - "--timeout", str(timeout), - "--companion-min", str(companion_range[0]), - "--companion-max", str(companion_range[1]), - "--spoiler", str(spoiler), - "--repeats", str(repeats), - "--random-repeats", str(random_repeats), - "--companion-pool", ",".join(companion_pool or ()), - ] - - def host_timeout(self, timeout, repeats, random_repeats): - """How long to let the whole container run. - - Every mode runs in the one container, each bounded by its own - timeout, so the host allows for all of them plus startup slack. - A ceiling rather than an expected cost. - """ - generations = 2 * repeats + 1 + 2 * random_repeats - buffer = self.settings("host_timeout_buffer", 30) - return timeout * generations + buffer - - def missing_file(self, path): - """The failure for a path that is not a file. - - Docker silently creates an empty directory at a missing mount - source rather than failing, which turns a not-yet-written - apworld into a bogus directory that breaks the content hash - days later with a confusing "Is a directory". - """ - return { - "game": None, - "outcome": "failed", - "detail": f"{path} does not exist (not a file)", - "elapsed_seconds": None, - } - - def test(self, apworld_path=None, game=None, output_dir=None, - timeout=None, companion_range=(2, 5), spoiler=3, repeats=10, - companion_pool=(), random_repeats=3): - """Run every mode against one world, core or downloaded.""" - output_dir = os.path.abspath(output_dir or self.paths.output) - os.makedirs(output_dir, exist_ok=True) - timeout = timeout or self.settings("generation_timeout", 300) - if apworld_path is not None: - apworld_path = os.path.abspath(apworld_path) - if not os.path.isfile(apworld_path): - return self.missing_file(apworld_path) - with tempfile.TemporaryDirectory(prefix="apworld-") as workdir: - tested, repaired = self.prepared(apworld_path, workdir) - report = self.run_once( - apworld_path=tested, - game=game, - output_dir=output_dir, - timeout=timeout, - companion_range=companion_range, - spoiler=spoiler, - repeats=repeats, - companion_pool=companion_pool, - random_repeats=random_repeats, - ) - if repaired: - report["repaired"] = True - return report - - def prepared(self, apworld_path, workdir): - """The path to test, and whether it had to be repaired.""" - if not apworld_path: - return None, False - return ApworldFile(apworld_path).prepare(workdir) - - def run_once(self, apworld_path, game, output_dir, timeout, - companion_range, spoiler, repeats, companion_pool, - random_repeats): - """One container run, and the report it produced.""" - limit = self.host_timeout(timeout, repeats, random_repeats) - command = self.build( - apworld_path=apworld_path, - output_dir=output_dir, - timeout=timeout, - companion_range=companion_range, - spoiler=spoiler, - repeats=repeats, - companion_pool=companion_pool, - game=game, - random_repeats=random_repeats, - ) - try: - result = subprocess.run(command, capture_output=True, text=True, - timeout=limit) - except subprocess.TimeoutExpired: - return { - "game": None, - "outcome": "failed", - "detail": f"docker run did not exit within {limit}s", - "elapsed_seconds": limit, - } - return DriverReport(result).parsed() diff --git a/src/archipelago_tester/pipeline/host/__init__.py b/src/archipelago_tester/pipeline/host/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/pipeline/host/custom_worlds.py b/src/archipelago_tester/pipeline/host/custom_worlds.py deleted file mode 100644 index 0ce4def..0000000 --- a/src/archipelago_tester/pipeline/host/custom_worlds.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Placing the approved apworlds where an Archipelago install finds them.""" - -import os -import shutil - -from archipelago_tester.core.model.apworld import ApworldFile -from archipelago_tester.core.model.verdict import Verdict -from archipelago_tester.core.state.keys import StateKeys - - -class CustomWorlds: - """The install's custom_worlds, kept to the worlds that passed. - - Only apworlds are placed. Nothing else in the install is read or - written - not host.yaml, not options, not the checkout - because the - install is someone else's and this pipeline only decides which - worlds are good enough to be in it. - - update_archipelago says whether to touch it at all and - archipelago_location says which install: with either missing, - nothing is changed and the reason is reported rather than guessed - at. - """ - - def __init__(self, config, paths, store, tag): - self.config = config - self.paths = paths - self.store = store - self.tag = tag - - def setting(self, key, default=None): - return self.config.value("archipelago", key, default) - - @property - def enabled(self): - return bool(self.setting("update_archipelago", False)) - - @property - def directory(self): - return self.paths.custom_worlds - - @property - def approved_labels(self): - """The labels good enough to place, matched case-insensitively. - - Written as the labels read rather than as the verdicts behind - them, so promote_labels and stability.labels are the same - wording and a rename in one is visible in the other. - """ - return {str(label).casefold() - for label in self.setting("promote_labels", [])} - - def approved(self, record): - """Whether this record's world belongs in the install. - - Judged on the verdict its own test produced, for the version - being run now: a result from an older Archipelago says nothing - about whether the world loads in this one. - """ - if record.get(StateKeys.DISCONTINUED): - return False - if record.get("tag") != self.tag: - return False - label = Verdict(record, self.config).label - return label.casefold() in self.approved_labels - - def candidates(self): - """Each approved apworld on disk, by the name it imports as. - - Two apworlds can declare the same importable name - a stray - download beside the real one - and one would silently overwrite - the other in a flat directory, so both are held back and - reported instead of picking a winner here. - """ - tests = self.store.load().get(StateKeys.APWORLD_TESTS, {}) - by_name = {} - for key, record in tests.items(): - if not self.approved(record): - continue - path = os.path.join(self.paths.downloads, key) - if os.path.exists(path): - name = ApworldFile(path).importable_name - by_name.setdefault(name, []).append(path) - placeable = {name: paths[0] for name, paths in by_name.items() - if len(paths) == 1} - collisions = {name: paths for name, paths in by_name.items() - if len(paths) > 1} - return placeable, collisions - - def place(self, placeable): - """Copy each approved apworld in under its importable name. - - Copied rather than linked: Archipelago loads these directly, - and a link into this pipeline's download cache would break the - moment that cache is cleared. - """ - os.makedirs(self.directory, exist_ok=True) - for name, path in placeable.items(): - shutil.copy2(path, os.path.join(self.directory, name)) - return sorted(placeable) - - def withdraw(self, placeable): - """Delete the apworlds that are no longer approved. - - A world that has stopped passing has to leave the install, not - merely stop being refreshed: leaving it there would keep - offering a world this pipeline no longer vouches for. Only - apworlds are ever removed, so anything else a person put in - that directory stays. - """ - present = {name for name in os.listdir(self.directory) - if name.endswith(ApworldFile.SUFFIX)} - withdrawn = present - set(placeable) - for name in withdrawn: - os.remove(os.path.join(self.directory, name)) - return sorted(withdrawn) - - def update(self): - """Bring the install's custom_worlds to the approved set.""" - if not self.enabled: - return {"updated": False, "reason": "update_archipelago is off"} - if not self.directory: - return {"updated": False, - "reason": "archipelago_location is not set"} - placeable, collisions = self.candidates() - placed = self.place(placeable) - withdrawn = self.withdraw(placeable) - return { - "updated": True, - "directory": self.directory, - "placed": placed, - "withdrawn": withdrawn, - "collisions": collisions, - } diff --git a/src/archipelago_tester/pipeline/sheet/__init__.py b/src/archipelago_tester/pipeline/sheet/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/pipeline/sheet/fetcher.py b/src/archipelago_tester/pipeline/sheet/fetcher.py deleted file mode 100644 index 037b006..0000000 --- a/src/archipelago_tester/pipeline/sheet/fetcher.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Downloading the worlds sheet.""" - -import io -import os -import zipfile - -import requests - - -class SheetFetcher: - """The Playable Worlds sheet, as HTML.""" - - def __init__(self, config, paths): - self.config = config - self.paths = paths - - @property - def spreadsheet_id(self): - return self.config.value("sheet", "spreadsheet_id") - - @property - def tab_name(self): - return self.config.value("sheet", "tab_name") - - @property - def export_url(self): - return ("https://docs.google.com/spreadsheets/d/" - f"{self.spreadsheet_id}/export?format=zip") - - def fetch(self): - """The tab's HTML, as bytes. - - Google answers a private sheet with its sign-in page and status - 200 rather than an error, so the content type is checked instead - of the status. - """ - response = requests.get(self.export_url, timeout=120) - response.raise_for_status() - if "application/zip" not in response.headers.get("Content-Type", ""): - raise RuntimeError( - "The server did not return a zip archive. The sheet is " - "most likely not shared with 'anyone with the link'.") - with zipfile.ZipFile(io.BytesIO(response.content)) as archive: - wanted = f"{self.tab_name}.html" - if wanted not in archive.namelist(): - raise RuntimeError( - f"The export has no tab called '{self.tab_name}'. " - f"Available: {', '.join(archive.namelist())}") - return archive.read(wanted) - - def save(self, content=None): - """Write the tab's HTML, returning where it landed.""" - content = self.fetch() if content is None else content - with open(self.paths.sheet_html, "wb") as handle: - handle.write(content) - return self.paths.sheet_html - - def ensure(self): - """Download the sheet only when this machine has no copy.""" - if not os.path.exists(self.paths.sheet_html): - self.save() - return self.paths.sheet_html diff --git a/src/archipelago_tester/pipeline/sheet/reader.py b/src/archipelago_tester/pipeline/sheet/reader.py deleted file mode 100644 index feb5420..0000000 --- a/src/archipelago_tester/pipeline/sheet/reader.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Reading the worlds sheet into Game objects.""" - -from bs4 import BeautifulSoup - -from archipelago_tester.core.model.game import Game - - -class SheetReader: - """The sheet's HTML export, as rows.""" - - def __init__(self, config, path): - self.config = config - self.path = path - - def column(self, name): - return (self.config.value("sheet", "columns") or {})[name] - - def rows(self): - """Every table row, as lists of cells. - - Google repeats each row's frozen-column cells in a spacer, which - would shift every column index by one, so those are dropped. - """ - with open(self.path, "r", encoding="utf-8") as handle: - soup = BeautifulSoup(handle.read(), "html.parser") - found = [] - for row in soup.find_all("tr"): - cells = [ - cell for cell in row.find_all("td") - if "freezebar-cell" not in cell.get("class", []) - ] - if cells: - found.append(cells) - return found - - def header(self, rows): - """Where the header row is, and the headings it holds.""" - for index, cells in enumerate(rows): - texts = [cell.get_text(strip=True) for cell in cells] - if self.column("name") in texts: - return index, texts - raise ValueError( - f"No '{self.column('name')}' header found in the export.") - - def indexes(self, headers): - """Where each field sits in a row.""" - if self.column("link") not in headers: - raise ValueError(f"No '{self.column('link')}' column found.") - optional = {} - for field in ("stability", "pr_status"): - heading = self.column(field) - optional[field] = (headers.index(heading) - if heading in headers else None) - return (headers.index(self.column("name")), - headers.index(self.column("link")), - optional["stability"], - optional["pr_status"]) - - @staticmethod - def text_of(cells, index): - """One cell's text, or None when missing or empty.""" - if index is None or index >= len(cells): - return None - return cells[index].get_text(strip=True) or None - - @staticmethod - def links_in(cell): - """Every hyperlink target in one cell, in the order written.""" - return [anchor.get("href") for anchor in cell.find_all("a") - if anchor.get("href")] - - def build(self, cells, indexes): - """One row as a Game, or None when it has no name.""" - name_index, link_index, stability_index, status_index = indexes - name = self.text_of(cells, name_index) - if name is None: - return None - game = Game( - name=name, - stability=self.text_of(cells, stability_index), - pr_status=self.text_of(cells, status_index), - ) - if link_index < len(cells): - for link in self.links_in(cells[link_index]): - game.add_link(link) - return game - - def games(self): - """Every world listed in the export.""" - rows = self.rows() - header_index, headers = self.header(rows) - indexes = self.indexes(headers) - found = [] - for cells in rows[header_index + 1:]: - game = self.build(cells, indexes) - if game is not None: - found.append(game) - return found diff --git a/src/archipelago_tester/pipeline/update/__init__.py b/src/archipelago_tester/pipeline/update/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/archipelago_tester/pipeline/update/cli.py b/src/archipelago_tester/pipeline/update/cli.py deleted file mode 100644 index e3e2617..0000000 --- a/src/archipelago_tester/pipeline/update/cli.py +++ /dev/null @@ -1,68 +0,0 @@ -"""The command line for a full update pass.""" - -import argparse -import json - -from archipelago_tester.core.config.config import Config -from archipelago_tester.core.config.paths import Paths -from archipelago_tester.core.config.run_settings import RunSettings -from archipelago_tester.core.state.lock import LockHeldError, PipelineLock -from archipelago_tester.pipeline.update.stage_error import StageError -from archipelago_tester.pipeline.update.update_run import UpdateRun - - -class UpdateCli: - """Update Archipelago, the sheet and every apworld, then test.""" - - def __init__(self): - self.config = Config.load() - self.paths = Paths(self.config) - self.settings = RunSettings(self.config) - - def parser(self): - parser = argparse.ArgumentParser(description=self.__doc__) - parser.add_argument( - "--stability", - help="Only consider games with this Stability column value " - "(e.g. 'stable', 'broken on main'). Case-insensitive.") - parser.add_argument( - "--fetch", type=int, - help="Only consider the first N games, after --stability " - "filtering - for a cheap, bounded pass.") - parser.add_argument( - "--jobs", "-j", type=int, default=None, - help="Apworlds to generation-test at once " - f"(default: {self.settings.jobs}).") - parser.add_argument( - "--force-retest", action="store_true", - help="Re-test every apworld, ignoring the cache. Expensive.") - return parser - - def run(self): - args = self.parser().parse_args() - run = UpdateRun( - config=self.config, - paths=self.paths, - stability=args.stability, - fetch=args.fetch, - jobs=args.jobs, - force_retest=args.force_retest, - ) - try: - with PipelineLock(self.paths.lock): - report = run.run() - except LockHeldError as error: - print(error) - return 1 - except StageError as error: - print(f"Update {error}") - return 1 - print("\n=== summary ===") - summary = {key: value for key, value in report.items() - if key != "test_results"} - print(json.dumps(summary, indent=2, default=str)) - return 0 - - @classmethod - def main(cls): - return cls().run() diff --git a/src/archipelago_tester/pipeline/update/download_reporter.py b/src/archipelago_tester/pipeline/update/download_reporter.py deleted file mode 100644 index dac9a92..0000000 --- a/src/archipelago_tester/pipeline/update/download_reporter.py +++ /dev/null @@ -1,37 +0,0 @@ -"""The download stage's progress line.""" - -import os - -from archipelago_tester.core.display.progress import Progress - - -class DownloadReporter: - """One line per game, saying what its apworld did. - - The download run calls back serialized, so this needs no lock of its - own. What each game's last asset did is remembered so the line can - still name it once the game itself finishes. - """ - - #: What each download status means, in words. - STATUS_LABELS = { - "missing": "downloaded (new)", - "stale": "downloaded (updated)", - "verified": "already present (verified up to date)", - "assumed": "already present (not verified)", - } - - def __init__(self, config, total): - self.bar = Progress(total, config=config, - labels={"ok": "ok", "none": "none"}) - self.last_asset = {} - - def on_asset(self, game_name, path, status): - label = self.STATUS_LABELS.get(status, status) - self.last_asset[game_name] = f"{label} {os.path.basename(path)}" - - def on_game(self, index, total, game_name, downloaded): - detail = self.last_asset.pop(game_name, "no apworld") - self.bar.update( - index, "ok" if downloaded else "none", game_name, - fallback_line=f"[{index}/{total}] {game_name}: {detail}") diff --git a/src/archipelago_tester/pipeline/update/stage_error.py b/src/archipelago_tester/pipeline/update/stage_error.py deleted file mode 100644 index 8749153..0000000 --- a/src/archipelago_tester/pipeline/update/stage_error.py +++ /dev/null @@ -1,15 +0,0 @@ -"""A failure, named by the stage it happened in.""" - - -class StageError(RuntimeError): - """Which stage failed, so the run can say so and stop. - - Raised at the stage boundary rather than inside it: a stage can fail - anywhere after it was opened, and an unnamed traceback makes a long - run's failure much harder to place. - """ - - def __init__(self, stage, original): - super().__init__(f"failed during '{stage}': {original}") - self.stage = stage - self.original = original diff --git a/src/archipelago_tester/pipeline/update/update_run.py b/src/archipelago_tester/pipeline/update/update_run.py deleted file mode 100644 index afd1cdb..0000000 --- a/src/archipelago_tester/pipeline/update/update_run.py +++ /dev/null @@ -1,298 +0,0 @@ -"""Everything the testing half does in one pass. - -Update Archipelago to the newest release, refresh the worlds sheet, -download whatever changed, and test each world by generating seeds. -What happens to the results afterwards - promoting, publishing, -deploying - is the other half's job and is not known about here. -""" - -import contextlib - -import requests - -from archipelago_tester.core.config.run_settings import RunSettings -from archipelago_tester.core.config.secrets import Secrets -from archipelago_tester.core.model.name import Name -from archipelago_tester.core.state.keys import StateKeys -from archipelago_tester.core.state.store import StateStore -from archipelago_tester.pipeline.batch.batch import Batch -from archipelago_tester.pipeline.batch.discontinued import Discontinued -from archipelago_tester.pipeline.batch.orphaned_records import ( - OrphanedRecords, -) -from archipelago_tester.pipeline.batch.download_failures import ( - DownloadFailures, -) -from archipelago_tester.pipeline.batch.reporter import Reporter -from archipelago_tester.pipeline.batch.results import Results -from archipelago_tester.pipeline.build.archipelago import ArchipelagoBuild -from archipelago_tester.pipeline.download.download_run import DownloadRun -from archipelago_tester.pipeline.download.update_history import ( - UpdateHistory, -) -from archipelago_tester.pipeline.generation.identifier import ApworldIdentifier -from archipelago_tester.pipeline.host.custom_worlds import CustomWorlds -from archipelago_tester.pipeline.sheet.fetcher import SheetFetcher -from archipelago_tester.pipeline.sheet.reader import SheetReader -from archipelago_tester.pipeline.update.download_reporter import ( - DownloadReporter, -) -from archipelago_tester.pipeline.update.stage_error import StageError - - -class UpdateRun: - """The four stages, in the order they run. - - Takes no lock of its own: a caller that also publishes the results - holds one for the whole thing, so nothing swaps underneath a - generation still in progress. - """ - - def __init__(self, config, paths, stability=None, fetch=None, jobs=None, - force_retest=False): - self.config = config - self.paths = paths - self.settings = RunSettings(config) - self.store = StateStore(paths.state) - self.build = ArchipelagoBuild(config, paths) - self.stability = stability - self.fetch = fetch - self.jobs = jobs or self.settings.jobs - self.force_retest = force_retest - self.report = {} - - @contextlib.contextmanager - def stage(self, name): - print(f"\n=== {name} ===") - try: - yield - except Exception as error: - raise StageError(name, error) from error - - def snapshot_previous_version(self, previous_tag): - """Keep last version's results before this one overwrites them. - - The only source for "did this world's outcome change at the - exact moment Archipelago upgraded": it is the state at the - instant the version changed, which no later re-test can - reconstruct. - """ - state = self.store.load() - state[StateKeys.UPGRADE_SNAPSHOT] = dict( - state.get(StateKeys.APWORLD_TESTS, {})) - state[StateKeys.PREVIOUS_ARCHIPELAGO_TAG] = previous_tag - self.store.save(state) - - def update_archipelago(self): - """Put this machine on the newest release and build its image.""" - with self.stage("archipelago_build"): - tag, previous, changed, built = self.build.update_and_build( - self.store) - self.build.ensure_test_image(tag) - self.report.update({ - "archipelago_tag": tag, - "archipelago_previous_tag": previous, - "archipelago_tag_changed": changed, - "archipelago_image_built": built, - }) - print(f"tag {tag}" + (f" (was {previous})" if previous - else " (first run)")) - if changed and previous: - self.snapshot_previous_version(previous) - return tag, changed - - def announce_retired(self, retired): - gone = sorted(key for key, value in retired.items() if value) - back = sorted(key for key, value in retired.items() if not value) - if gone: - print(f"{len(gone)} no longer on the sheet, marked " - f"discontinued: {', '.join(gone)}") - if back: - print(f"{len(back)} back on the sheet: {', '.join(back)}") - - def select(self, games): - """The rows this pass considers. - - Applied to the whole sheet, core rows included, so --fetch - really does bound the work: core worlds are tested like every - other row, and leaving them out of the limit made a "cheap, - bounded pass" still test all eighty of them. - """ - if self.stability: - games = [game for game in games - if (game.stability or "").lower() - == self.stability.lower()] - return games[:self.fetch] if self.fetch is not None else games - - def read_sheet(self): - """The sheet's rows, split into core and downloadable. - - Discontinued rows are flagged straight after reading, so every - stage below already sees a world taken off the sheet as gone. - Core rows are tested by name but never downloaded: Archipelago - ships them. - """ - with self.stage("sheet_download"): - path = SheetFetcher(self.config, self.paths).save() - games = SheetReader(self.config, path).games() - # Every row, before any filtering: download-time identity - # checks compare against the whole sheet, core rows - # included, and so does the discontinued flag. - sheet_names = [game.name for game in games] - self.report["games_in_sheet"] = len(games) - print(f"{len(games)} games in sheet") - retired = Discontinued(self.store).mark(sheet_names) - self.report["discontinued_changed"] = retired - self.announce_retired(retired) - selected = self.select(games) - core = [game for game in selected if game.is_core] - rest = [game for game in selected if not game.is_core] - self.report["games_core"] = len(core) - self.report["games_selected"] = len(rest) - print(f"{len(core)} core games (tested by name, nothing to download)") - print(f"{len(rest)} games selected (stability={self.stability!r}, " - f"fetch={self.fetch!r})") - return rest, core, sheet_names - - def session(self): - """A session carrying the token, if there is one.""" - session = requests.Session() - token = Secrets(self.paths).github_token - if token: - session.headers["Authorization"] = f"token {token}" - return session - - def downloader(self, image, sheet_names): - return DownloadRun( - config=self.config, - paths=self.paths, - session=self.session(), - identifier=ApworldIdentifier(self.config, self.paths, image), - jobs=self.jobs, - sheet_names=sheet_names, - # Last run's resolved releases, so a game whose attempt - # failed against an unchanged release is not downloaded and - # re-identified all over again. - previous_releases=self.store.load().get( - StateKeys.GAME_RELEASES, {}), - ) - - def download(self, games, sheet_names, image, tag): - """Fetch every selected game's newest apworld.""" - reporter = DownloadReporter(self.config, len(games)) - run = self.downloader(image, sheet_names) - with self.stage("apworld_download"): - total, skipped, releases = run.all( - games, on_asset=reporter.on_asset, on_game=reporter.on_game) - self.report["assets_downloaded"] = total - self.report["games_skipped_download"] = len(skipped) - print(f"{total} assets downloaded, {len(skipped)} games skipped") - run.record_releases(releases) - self.record_versions() - DownloadFailures(self.config, self.store).record(games, skipped, tag) - orphaned = OrphanedRecords(self.store, self.paths.downloads).prune() - self.report["orphaned_records_dropped"] = len(orphaned) - if orphaned: - print(f"{len(orphaned)} result(s) dropped for apworlds no " - f"longer on disk: {', '.join(sorted(orphaned))}") - - def record_versions(self): - """One history entry per game whose apworld content changed. - - Recorded here, not by whatever publishes afterwards: this is - the moment a new version is known, and a pass that never - publishes would lose it outright - the next download overwrites - the file and no later stage can reconstruct which version - existed on which date. - """ - history = UpdateHistory(self.paths) - with self.stage("apworld_history"): - recorded = history.record_all(self.store.load(), - self.paths.downloads) - self.report["new_apworld_versions"] = len(recorded) - print(f"{len(recorded)} new apworld version(s) recorded") - - def selected_only(self, games, core_games): - """This pass's rows, named the way the batch names them. - - None unless something actually narrowed the sheet. The download - directory holds every apworld ever fetched, so an unnarrowed run - has to consider all of them - and Batch reads an explicit - selection as a reason to retest rather than serve from cache, - which would turn a plain pass into a full re-test of everything. - - Downloadable rows are matched on their download folder, core - ones on the bare sheet name, because that is how each is keyed. - """ - if self.fetch is None and not self.stability: - return None - return ([Name(game.name).directory for game in games] - + [game.name for game in core_games]) - - def test(self, image, tag, games, core_games, changed): - """Generation-test the downloaded apworlds, and the core ones. - - Core worlds ship with Archipelago, so nothing was downloaded for - them and they are named rather than mounted, but they run the - same five modes and their result is measured rather than - assumed. - - A narrowed pass tests what it narrowed to and nothing else: - --fetch bounded the download stage from the start, and leaving - the test stage unbounded meant a "cheap, bounded pass" still - generated against every apworld on disk. - """ - only = self.selected_only(games, core_games) - if only is not None: - print(f"{len(only)} selected for testing " - f"(stability={self.stability!r}, fetch={self.fetch!r})") - with self.stage("generation_tests"): - results = Batch( - config=self.config, - paths=self.paths, - settings=self.settings, - image=image, - tag=tag, - force=changed or self.force_retest, - on_progress=Reporter(self.config, bar=True), - jobs=self.jobs, - core_games=[game.name for game in core_games], - only=only, - exact_only=only is not None, - ).run() - summary = Results(results, self.config).summary - self.report["test_counts"] = summary - self.report["test_results"] = results - print(summary) - return results - - def place_apworlds(self, tag): - """Put the worlds that passed into the Archipelago install. - - Last, because it acts on what the tests just found. Off by - default: an install is someone's own, so it is only written to - when update_archipelago says so, and then only its - custom_worlds directory. - """ - worlds = CustomWorlds(self.config, self.paths, self.store, tag) - with self.stage("custom_worlds"): - result = worlds.update() - self.report["custom_worlds"] = result - if not result["updated"]: - print(f"not updating any install - {result['reason']}") - return - print(f"{len(result['placed'])} apworlds in {result['directory']}, " - f"{len(result['withdrawn'])} withdrawn") - if result["collisions"]: - print(f"{len(result['collisions'])} name collision(s) held " - f"back: {', '.join(sorted(result['collisions']))}") - - def run(self): - """Every stage, in order, returning what each one did.""" - tag, changed = self.update_archipelago() - image = self.build.test_image_tag(tag) - games, core_games, sheet_names = self.read_sheet() - self.download(games, sheet_names, image, tag) - self.test(image, tag, games, core_games, changed) - self.place_apworlds(tag) - return self.report