initial commit

src/archipelago_tester/pipeline/update/cli.py

@@ -0,0 +1,68 @@
+"""The command line for a full update pass."""
+
+import argparse
+import json
+
+from archipelago_tester.core.config.config import Config
+from archipelago_tester.core.config.paths import Paths
+from archipelago_tester.core.config.run_settings import RunSettings
+from archipelago_tester.core.state.lock import LockHeldError, PipelineLock
+from archipelago_tester.pipeline.update.stage_error import StageError
+from archipelago_tester.pipeline.update.update_run import UpdateRun
+
+
+class UpdateCli:
+    """Update Archipelago, the sheet and every apworld, then test."""
+
+    def __init__(self):
+        self.config = Config.load()
+        self.paths = Paths(self.config)
+        self.settings = RunSettings(self.config)
+
+    def parser(self):
+        parser = argparse.ArgumentParser(description=self.__doc__)
+        parser.add_argument(
+            "--stability",
+            help="Only consider games with this Stability column value "
+                 "(e.g. 'stable', 'broken on main'). Case-insensitive.")
+        parser.add_argument(
+            "--fetch", type=int,
+            help="Only consider the first N games, after --stability "
+                 "filtering - for a cheap, bounded pass.")
+        parser.add_argument(
+            "--jobs", "-j", type=int, default=None,
+            help="Apworlds to generation-test at once "
+                 f"(default: {self.settings.jobs}).")
+        parser.add_argument(
+            "--force-retest", action="store_true",
+            help="Re-test every apworld, ignoring the cache. Expensive.")
+        return parser
+
+    def run(self):
+        args = self.parser().parse_args()
+        run = UpdateRun(
+            config=self.config,
+            paths=self.paths,
+            stability=args.stability,
+            fetch=args.fetch,
+            jobs=args.jobs,
+            force_retest=args.force_retest,
+        )
+        try:
+            with PipelineLock(self.paths.lock):
+                report = run.run()
+        except LockHeldError as error:
+            print(error)
+            return 1
+        except StageError as error:
+            print(f"Update {error}")
+            return 1
+        print("\n=== summary ===")
+        summary = {key: value for key, value in report.items()
+                   if key != "test_results"}
+        print(json.dumps(summary, indent=2, default=str))
+        return 0
+
+    @classmethod
+    def main(cls):
+        return cls().run()

src/archipelago_tester/pipeline/update/download_reporter.py

@@ -0,0 +1,37 @@
+"""The download stage's progress line."""
+
+import os
+
+from archipelago_tester.core.display.progress import Progress
+
+
+class DownloadReporter:
+    """One line per game, saying what its apworld did.
+
+    The download run calls back serialized, so this needs no lock of its
+    own. What each game's last asset did is remembered so the line can
+    still name it once the game itself finishes.
+    """
+
+    #: What each download status means, in words.
+    STATUS_LABELS = {
+        "missing": "downloaded (new)",
+        "stale": "downloaded (updated)",
+        "verified": "already present (verified up to date)",
+        "assumed": "already present (not verified)",
+    }
+
+    def __init__(self, config, total):
+        self.bar = Progress(total, config=config,
+                            labels={"ok": "ok", "none": "none"})
+        self.last_asset = {}
+
+    def on_asset(self, game_name, path, status):
+        label = self.STATUS_LABELS.get(status, status)
+        self.last_asset[game_name] = f"{label} {os.path.basename(path)}"
+
+    def on_game(self, index, total, game_name, downloaded):
+        detail = self.last_asset.pop(game_name, "no apworld")
+        self.bar.update(
+            index, "ok" if downloaded else "none", game_name,
+            fallback_line=f"[{index}/{total}] {game_name}: {detail}")

src/archipelago_tester/pipeline/update/stage_error.py

@@ -0,0 +1,15 @@
+"""A failure, named by the stage it happened in."""
+
+
+class StageError(RuntimeError):
+    """Which stage failed, so the run can say so and stop.
+
+    Raised at the stage boundary rather than inside it: a stage can fail
+    anywhere after it was opened, and an unnamed traceback makes a long
+    run's failure much harder to place.
+    """
+
+    def __init__(self, stage, original):
+        super().__init__(f"failed during '{stage}': {original}")
+        self.stage = stage
+        self.original = original

src/archipelago_tester/pipeline/update/update_run.py

@@ -0,0 +1,298 @@
+"""Everything the testing half does in one pass.
+
+Update Archipelago to the newest release, refresh the worlds sheet,
+download whatever changed, and test each world by generating seeds.
+What happens to the results afterwards - promoting, publishing,
+deploying - is the other half's job and is not known about here.
+"""
+
+import contextlib
+
+import requests
+
+from archipelago_tester.core.config.run_settings import RunSettings
+from archipelago_tester.core.config.secrets import Secrets
+from archipelago_tester.core.model.name import Name
+from archipelago_tester.core.state.keys import StateKeys
+from archipelago_tester.core.state.store import StateStore
+from archipelago_tester.pipeline.batch.batch import Batch
+from archipelago_tester.pipeline.batch.discontinued import Discontinued
+from archipelago_tester.pipeline.batch.orphaned_records import (
+    OrphanedRecords,
+)
+from archipelago_tester.pipeline.batch.download_failures import (
+    DownloadFailures,
+)
+from archipelago_tester.pipeline.batch.reporter import Reporter
+from archipelago_tester.pipeline.batch.results import Results
+from archipelago_tester.pipeline.build.archipelago import ArchipelagoBuild
+from archipelago_tester.pipeline.download.download_run import DownloadRun
+from archipelago_tester.pipeline.download.update_history import (
+    UpdateHistory,
+)
+from archipelago_tester.pipeline.generation.identifier import ApworldIdentifier
+from archipelago_tester.pipeline.host.custom_worlds import CustomWorlds
+from archipelago_tester.pipeline.sheet.fetcher import SheetFetcher
+from archipelago_tester.pipeline.sheet.reader import SheetReader
+from archipelago_tester.pipeline.update.download_reporter import (
+    DownloadReporter,
+)
+from archipelago_tester.pipeline.update.stage_error import StageError
+
+
+class UpdateRun:
+    """The four stages, in the order they run.
+
+    Takes no lock of its own: a caller that also publishes the results
+    holds one for the whole thing, so nothing swaps underneath a
+    generation still in progress.
+    """
+
+    def __init__(self, config, paths, stability=None, fetch=None, jobs=None,
+                 force_retest=False):
+        self.config = config
+        self.paths = paths
+        self.settings = RunSettings(config)
+        self.store = StateStore(paths.state)
+        self.build = ArchipelagoBuild(config, paths)
+        self.stability = stability
+        self.fetch = fetch
+        self.jobs = jobs or self.settings.jobs
+        self.force_retest = force_retest
+        self.report = {}
+
+    @contextlib.contextmanager
+    def stage(self, name):
+        print(f"\n=== {name} ===")
+        try:
+            yield
+        except Exception as error:
+            raise StageError(name, error) from error
+
+    def snapshot_previous_version(self, previous_tag):
+        """Keep last version's results before this one overwrites them.
+
+        The only source for "did this world's outcome change at the
+        exact moment Archipelago upgraded": it is the state at the
+        instant the version changed, which no later re-test can
+        reconstruct.
+        """
+        state = self.store.load()
+        state[StateKeys.UPGRADE_SNAPSHOT] = dict(
+            state.get(StateKeys.APWORLD_TESTS, {}))
+        state[StateKeys.PREVIOUS_ARCHIPELAGO_TAG] = previous_tag
+        self.store.save(state)
+
+    def update_archipelago(self):
+        """Put this machine on the newest release and build its image."""
+        with self.stage("archipelago_build"):
+            tag, previous, changed, built = self.build.update_and_build(
+                self.store)
+            self.build.ensure_test_image(tag)
+        self.report.update({
+            "archipelago_tag": tag,
+            "archipelago_previous_tag": previous,
+            "archipelago_tag_changed": changed,
+            "archipelago_image_built": built,
+        })
+        print(f"tag {tag}" + (f" (was {previous})" if previous
+                              else " (first run)"))
+        if changed and previous:
+            self.snapshot_previous_version(previous)
+        return tag, changed
+
+    def announce_retired(self, retired):
+        gone = sorted(key for key, value in retired.items() if value)
+        back = sorted(key for key, value in retired.items() if not value)
+        if gone:
+            print(f"{len(gone)} no longer on the sheet, marked "
+                  f"discontinued: {', '.join(gone)}")
+        if back:
+            print(f"{len(back)} back on the sheet: {', '.join(back)}")
+
+    def select(self, games):
+        """The rows this pass considers.
+
+        Applied to the whole sheet, core rows included, so --fetch
+        really does bound the work: core worlds are tested like every
+        other row, and leaving them out of the limit made a "cheap,
+        bounded pass" still test all eighty of them.
+        """
+        if self.stability:
+            games = [game for game in games
+                     if (game.stability or "").lower()
+                     == self.stability.lower()]
+        return games[:self.fetch] if self.fetch is not None else games
+
+    def read_sheet(self):
+        """The sheet's rows, split into core and downloadable.
+
+        Discontinued rows are flagged straight after reading, so every
+        stage below already sees a world taken off the sheet as gone.
+        Core rows are tested by name but never downloaded: Archipelago
+        ships them.
+        """
+        with self.stage("sheet_download"):
+            path = SheetFetcher(self.config, self.paths).save()
+            games = SheetReader(self.config, path).games()
+            # Every row, before any filtering: download-time identity
+            # checks compare against the whole sheet, core rows
+            # included, and so does the discontinued flag.
+            sheet_names = [game.name for game in games]
+        self.report["games_in_sheet"] = len(games)
+        print(f"{len(games)} games in sheet")
+        retired = Discontinued(self.store).mark(sheet_names)
+        self.report["discontinued_changed"] = retired
+        self.announce_retired(retired)
+        selected = self.select(games)
+        core = [game for game in selected if game.is_core]
+        rest = [game for game in selected if not game.is_core]
+        self.report["games_core"] = len(core)
+        self.report["games_selected"] = len(rest)
+        print(f"{len(core)} core games (tested by name, nothing to download)")
+        print(f"{len(rest)} games selected (stability={self.stability!r}, "
+              f"fetch={self.fetch!r})")
+        return rest, core, sheet_names
+
+    def session(self):
+        """A session carrying the token, if there is one."""
+        session = requests.Session()
+        token = Secrets(self.paths).github_token
+        if token:
+            session.headers["Authorization"] = f"token {token}"
+        return session
+
+    def downloader(self, image, sheet_names):
+        return DownloadRun(
+            config=self.config,
+            paths=self.paths,
+            session=self.session(),
+            identifier=ApworldIdentifier(self.config, self.paths, image),
+            jobs=self.jobs,
+            sheet_names=sheet_names,
+            # Last run's resolved releases, so a game whose attempt
+            # failed against an unchanged release is not downloaded and
+            # re-identified all over again.
+            previous_releases=self.store.load().get(
+                StateKeys.GAME_RELEASES, {}),
+        )
+
+    def download(self, games, sheet_names, image, tag):
+        """Fetch every selected game's newest apworld."""
+        reporter = DownloadReporter(self.config, len(games))
+        run = self.downloader(image, sheet_names)
+        with self.stage("apworld_download"):
+            total, skipped, releases = run.all(
+                games, on_asset=reporter.on_asset, on_game=reporter.on_game)
+        self.report["assets_downloaded"] = total
+        self.report["games_skipped_download"] = len(skipped)
+        print(f"{total} assets downloaded, {len(skipped)} games skipped")
+        run.record_releases(releases)
+        self.record_versions()
+        DownloadFailures(self.config, self.store).record(games, skipped, tag)
+        orphaned = OrphanedRecords(self.store, self.paths.downloads).prune()
+        self.report["orphaned_records_dropped"] = len(orphaned)
+        if orphaned:
+            print(f"{len(orphaned)} result(s) dropped for apworlds no "
+                  f"longer on disk: {', '.join(sorted(orphaned))}")
+
+    def record_versions(self):
+        """One history entry per game whose apworld content changed.
+
+        Recorded here, not by whatever publishes afterwards: this is
+        the moment a new version is known, and a pass that never
+        publishes would lose it outright - the next download overwrites
+        the file and no later stage can reconstruct which version
+        existed on which date.
+        """
+        history = UpdateHistory(self.paths)
+        with self.stage("apworld_history"):
+            recorded = history.record_all(self.store.load(),
+                                          self.paths.downloads)
+        self.report["new_apworld_versions"] = len(recorded)
+        print(f"{len(recorded)} new apworld version(s) recorded")
+
+    def selected_only(self, games, core_games):
+        """This pass's rows, named the way the batch names them.
+
+        None unless something actually narrowed the sheet. The download
+        directory holds every apworld ever fetched, so an unnarrowed run
+        has to consider all of them - and Batch reads an explicit
+        selection as a reason to retest rather than serve from cache,
+        which would turn a plain pass into a full re-test of everything.
+
+        Downloadable rows are matched on their download folder, core
+        ones on the bare sheet name, because that is how each is keyed.
+        """
+        if self.fetch is None and not self.stability:
+            return None
+        return ([Name(game.name).directory for game in games]
+                + [game.name for game in core_games])
+
+    def test(self, image, tag, games, core_games, changed):
+        """Generation-test the downloaded apworlds, and the core ones.
+
+        Core worlds ship with Archipelago, so nothing was downloaded for
+        them and they are named rather than mounted, but they run the
+        same five modes and their result is measured rather than
+        assumed.
+
+        A narrowed pass tests what it narrowed to and nothing else:
+        --fetch bounded the download stage from the start, and leaving
+        the test stage unbounded meant a "cheap, bounded pass" still
+        generated against every apworld on disk.
+        """
+        only = self.selected_only(games, core_games)
+        if only is not None:
+            print(f"{len(only)} selected for testing "
+                  f"(stability={self.stability!r}, fetch={self.fetch!r})")
+        with self.stage("generation_tests"):
+            results = Batch(
+                config=self.config,
+                paths=self.paths,
+                settings=self.settings,
+                image=image,
+                tag=tag,
+                force=changed or self.force_retest,
+                on_progress=Reporter(self.config, bar=True),
+                jobs=self.jobs,
+                core_games=[game.name for game in core_games],
+                only=only,
+                exact_only=only is not None,
+            ).run()
+        summary = Results(results, self.config).summary
+        self.report["test_counts"] = summary
+        self.report["test_results"] = results
+        print(summary)
+        return results
+
+    def place_apworlds(self, tag):
+        """Put the worlds that passed into the Archipelago install.
+
+        Last, because it acts on what the tests just found. Off by
+        default: an install is someone's own, so it is only written to
+        when update_archipelago says so, and then only its
+        custom_worlds directory.
+        """
+        worlds = CustomWorlds(self.config, self.paths, self.store, tag)
+        with self.stage("custom_worlds"):
+            result = worlds.update()
+        self.report["custom_worlds"] = result
+        if not result["updated"]:
+            print(f"not updating any install - {result['reason']}")
+            return
+        print(f"{len(result['placed'])} apworlds in {result['directory']}, "
+              f"{len(result['withdrawn'])} withdrawn")
+        if result["collisions"]:
+            print(f"{len(result['collisions'])} name collision(s) held "
+                  f"back: {', '.join(sorted(result['collisions']))}")
+
+    def run(self):
+        """Every stage, in order, returning what each one did."""
+        tag, changed = self.update_archipelago()
+        image = self.build.test_image_tag(tag)
+        games, core_games, sheet_names = self.read_sheet()
+        self.download(games, sheet_names, image, tag)
+        self.test(image, tag, games, core_games, changed)
+        self.place_apworlds(tag)
+        return self.report