initial commit

.env.example

@@ -0,0 +1,5 @@
+# Copy to .env and fill in. .env is gitignored; this file is not.
+#
+# A GitHub personal access token, needing no scopes at all - it is only
+# used to raise the API rate limit from 60 requests an hour to 5000.
+GITHUB_TOKEN=

.gitignore

@@ -1,176 +1,26 @@
-# ---> Python
-# Byte-compiled / optimized / DLL files
 __pycache__/
-*.py[cod]
+*.pyc
-*$py.class
 
-# C extensions
+# Credentials. Never committed - see .env.example.
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# UV
-#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#uv.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
-.pdm.toml
-.pdm-python
-.pdm-build/
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
 .env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
 
-# Spyder project settings
+# Nothing a run generates lives here any more: config.yaml's "data_root"
-.spyderproject
+# points outside the checkout (the Archipelago clone, downloads, seeds
-.spyproject
+# and state.json all live there). These remain only to catch a config
+# that points data_root back inside it.
+state.json
+scratch_state.json
+pipeline.lock
+downloads/
+test_output/
+archipelago_src/
+playable_worlds.html
+failed_apworlds.txt
 
-# Rope project settings
+# Claude file
-.ropeproject
+CLAUDE.md
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-
-# Ruff stuff:
-.ruff_cache/
-
-# PyPI configuration file
-.pypirc
 
+# Build output.
+build/
+dist/
+*.egg-info/

LICENSE

@@ -1,18 +1,21 @@
 MIT License
 
-Copyright (c) 2026 pascal
+Copyright (c) 2026 Pascal
 
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and 
+Permission is hereby granted, free of charge, to any person obtaining a copy
-associated documentation files (the "Software"), to deal in the Software without restriction, including 
+of this software and associated documentation files (the "Software"), to deal
-without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 
+in the Software without restriction, including without limitation the rights
-copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the 
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-following conditions:
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in all copies or substantial 
+The above copyright notice and this permission notice shall be included in all
-portions of the Software.
+copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO 
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-USE OR OTHER DEALINGS IN THE SOFTWARE.
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,162 @@
-# APWorldTester
+# Archipelago World Tester
+
+This project checks whether community Archipelago worlds can generate a seed.
+
+It reads the community worlds spreadsheet, downloads each world's newest `.apworld` file, and runs Archipelago's generator against it in a Docker container.
+
+Each world is tested five ways:
+
+- by itself
+- with two players of the same game
+- with 2–5 of Archipelago's built-in worlds
+- by itself with its options randomized
+- with other worlds and its options randomized
+
+Each test is repeated with different seeds. The results are combined into one of six stability values:
+
+| Value | Meaning |
+| --- | --- |
+| Stable | Every attempt succeeded |
+| Minor issues | Solo and multiworld tests worked, but a randomized test failed |
+| Flaky | Some attempts succeeded and others failed |
+| Solo only | Works by itself, but fails with other worlds |
+| Broken | Could not generate at all |
+| Unknown | No usable result, for example because the `.apworld` was missing or a ROM was required |
+
+A successful test means the world loaded, generated a seed, and computed the spoiler log at the configured level. With the default `playthrough` level, this also checks that the seed is completable.
+
+## Requirements
+
+- **Python 3.10 or newer**
+- **Git** — used to clone Archipelago and check out the version being tested
+- **Docker** — must be usable by your user without `sudo`
+- **A GitHub token** — required to access the GitHub API. It does not need any scopes
+- **Base ROMs** — optional; some worlds cannot generate without them
+
+## Installation
+
+### System packages
+
+On Ubuntu or Debian:
+
+```sh
+sudo apt update
+sudo apt install -y git python3 python3-venv
+```
+
+Install Docker:
+
+```sh
+curl -fsSL https://get.docker.com | sudo sh
+sudo usermod -aG docker "$USER"
+newgrp docker
+docker run --rm hello-world
+```
+
+The last command should work without `sudo`.
+
+### Install the package
+
+```sh
+git clone <repository-url> archipelago-world-tester
+cd archipelago-world-tester
+```
+
+Using `uv`:
+
+```sh
+uv sync
+```
+
+Or using `pip`:
+
+```sh
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e .
+```
+
+### Configuration
+
+Write a starter config and edit the paths in it:
+
+```bash
+apworld-tester init
+```
+
+That writes `~/.config/apworld-tester/config.yaml` (or
+`$XDG_CONFIG_HOME/apworld-tester/config.yaml`) and prints the path.
+Set `general.output_directory` - everything a run generates lives there,
+including the Archipelago checkout and the downloaded apworlds - and
+`general.roms_directory` for the worlds that need a base ROM.
+
+The config is looked for in this order, first hit wins:
+
+1. `$APWORLD_TESTER_CONFIG`
+2. `~/.config/apworld-tester/config.yaml`
+3. `config.yaml` beside a source checkout
+
+`apworld-tester config-path` prints which one is in use, or lists
+every location it tried.
+
+Create `.env` with your GitHub token:
+
+```text
+GITHUB_TOKEN=ghp_...
+```
+
+## Test worlds
+
+Use `TestRun` to test one or more worlds:
+
+```python
+from apworld_tester import TestRun
+
+TestRun("Baba Is You").run()
+```
+
+Multiple worlds:
+
+```python
+TestRun(["Baba Is You", "Anodyne"]).run()
+```
+
+To test all worlds that have already been downloaded:
+
+```python
+TestRun(cached=True).run()
+```
+
+The first run clones Archipelago and builds the Docker image, which can take several minutes.
+
+## Test all worlds
+
+`UpdateRun` updates Archipelago, refreshes the community worlds spreadsheet, downloads changed worlds, and tests everything:
+
+```python
+from apworld_tester import UpdateRun
+
+UpdateRun().run()
+```
+
+## Test a local `.apworld`
+
+For a world you are developing, put the `.apworld` in its own directory:
+
+```text
+~/my-apworlds/
+└── My Game/
+    └── my_game.apworld
+```
+
+Then run:
+
+```python
+from apworld_tester import TestRun
+
+TestRun(
+    "My Game",
+    root_directory="/home/you/my-apworlds",
+).run()
+```
 
-Test Suite for Archipelago Worlds

container/Dockerfile.test

@@ -0,0 +1,20 @@
+# Unprivileged wrapper for the generation tests.
+#
+# Upstream's image runs as root and leaves /.local root-owned, so running
+# it as the host user fails before generation starts (PermissionError on
+# /.local) and running it as root leaves root-owned seeds in the mounted
+# output directory. Two lines fix both, and they are the only difference
+# from the base image.
+#
+# Built once per Archipelago tag, not per apworld test, so the cost of
+# the recursive chown is paid once and amortised over a few hundred
+# generations.
+ARG BASE_IMAGE=archipelago:latest
+FROM ${BASE_IMAGE}
+
+ARG RUNNER_UID=1000
+ARG RUNNER_GID=1000
+
+RUN chown -R ${RUNNER_UID}:${RUNNER_GID} /app /.local 2>/dev/null || chown -R ${RUNNER_UID}:${RUNNER_GID} /app
+
+USER ${RUNNER_UID}:${RUNNER_GID}

container/drivers/identify_apworld.py

@@ -0,0 +1,49 @@
+"""Runs inside the archipelago Docker image. Reports which game (if any)
+a single mounted .apworld registers itself as under Archipelago's own
+World registry - the import/registration step only, no seed generation.
+
+Used to verify a freshly downloaded .apworld's actual content matches
+the sheet row that fetched it, rather than trusting release/asset
+name-matching alone (see AssetMatcher): the filename
+and release title are just text an author chose, but AutoWorldRegister
+is filled from the World class's own "game" attribute, which is what
+Archipelago itself will treat this file as - the authoritative source.
+"""
+
+import argparse
+import json
+import sys
+import warnings
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--apworld", required=True)
+    args = parser.parse_args()
+
+    warnings.simplefilter("ignore")
+
+    try:
+        import worlds
+        from worlds.AutoWorld import AutoWorldRegister
+    except BaseException as error:
+        # The apworld (or something it imports) crashed outright - that's
+        # itself a meaningful signal (this file is broken/wrong), not an
+        # infrastructure failure, so it's still reported as a clean JSON
+        # result rather than a non-zero exit / stack trace.
+        print(json.dumps({"games": [], "detail": f"{type(error).__name__}: {error}"}))
+        return 0
+
+    games = [
+        name for name, world_type in AutoWorldRegister.world_types.items()
+        if str(getattr(world_type, "zip_path", "") or "") == args.apworld
+    ]
+    print(json.dumps({
+        "games": games,
+        "detail": None if games else f"failed_world_loads={worlds.failed_world_loads}",
+    }))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

container/drivers/run_test.py

@@ -0,0 +1,827 @@
+"""Runs inside the archipelago Docker image. Generates one seed for a
+single installed apworld and reports the outcome as one JSON line on
+stdout: passed, needs_input (missing base ROM or similar), or failed.
+
+The logic mirrors Archipelago's own test/hosting/generate.py, but is
+copied here rather than imported from it: the project's .dockerignore
+excludes test/ from the image, so that module isn't present at runtime.
+"""
+
+import argparse
+import io
+import json
+import logging
+import random
+import re
+import shutil
+import sys
+import unicodedata
+import time
+import warnings
+from multiprocessing import Manager, Process, set_start_method
+from pathlib import Path
+
+def ensure_tutorials_default():
+    # Monkeypatch applied AFTER whichever Archipelago version's own real
+    # worlds/AutoWorld.py has already loaded normally - defaults
+    # WebWorld.tutorials to [] if that core doesn't already declare it,
+    # without needing to know or duplicate anything else about that
+    # core's own AutoWorld.py content. Previously this was a full-file
+    # bind-mount replacement (container/AutoWorld.py (absent - see config.py)) built against one
+    # specific version's source; that broke outright when tested against
+    # an older Archipelago release whose real AutoWorld.py imports
+    # differently (e.g. no rule_builder module before it was added
+    # upstream) - replacing the whole file assumes it stays
+    # source-compatible with everything else that release's core
+    # expects, which isn't true across versions. This achieves the same
+    # fix (see the original container/AutoWorld.py (absent - see config.py) comment for why it's
+    # needed - WebHost.py's invalid_worlds filter vs.
+    # network_data_package snapshot timing) without that assumption.
+    from worlds.AutoWorld import WebWorld
+    if "tutorials" not in vars(WebWorld):
+        WebWorld.tutorials = []
+
+
+class WebHostIncompatibleError(RuntimeError):
+    """Generation succeeds, but this world would crash WebHost's own
+    startup - a bug class Generate.py never exercises (see
+    check_webhost_compatibility below)."""
+
+
+def check_webhost_compatibility(game):
+    # Reproduces WebHost's own generate_yaml_templates() (Options.py),
+    # scoped to just this one world, so a bug like Option.visibility
+    # being set to a bare int instead of a Visibility flag - which
+    # crashes generate_yaml_templates() for the ENTIRE deployed site,
+    # not just the offending world, since it iterates every visible
+    # world in one loop - is caught here as a normal test failure
+    # instead of only in production. (A hand-run diagnostic that located
+    # the offending worlds used to live in container/tools/; removed.)
+    # which found this bug class live on the deployed instance.
+    from inspect import cleandoc
+
+    import yaml
+    from jinja2 import Template
+
+    import Options
+    from Utils import local_path, __version__
+    from worlds.AutoWorld import AutoWorldRegister
+
+    world = AutoWorldRegister.world_types[game]
+    if not hasattr(world.web, "tutorials"):
+        # WebHost.py strips any world missing this from AutoWorldRegister
+        # entirely at startup (its invalid_worlds set) - so it can't
+        # crash generate_yaml_templates (never part of that loop), but
+        # the world is now gone from WebHost's registry altogether. That
+        # breaks something worse than templates: MultiServer can't find
+        # the game's data (item_name_groups etc.) when starting a room,
+        # so *hosting* any seed generated with this world crashes with
+        # a KeyError - even though generation itself succeeded fine.
+        raise WebHostIncompatibleError(
+            "passes generation but has no WebWorld.tutorials - WebHost "
+            "strips it from its world registry entirely at startup, so "
+            "hosting a room for a seed generated with it crashes "
+            "MultiServer with a KeyError on the game name"
+        )
+
+    def dictify_range(option, option_val):
+        data = {option_val: 50}
+        for sub_option in ["random", "random-low", "random-high",
+                            f"random-range-{option.range_start}-{option.range_end}"]:
+            if sub_option != option_val:
+                data[sub_option] = 0
+        notes = {}
+        for name, number in getattr(option, "special_range_names", {}).items():
+            notes[name] = f"equivalent to {number}"
+            if number in data:
+                data[name] = data[number]
+                del data[number]
+            elif name in data:
+                pass
+            else:
+                data[name] = 0
+        return data, notes
+
+    def yaml_dump_scalar(scalar):
+        return yaml.dump(scalar).replace("...\n", "").strip()
+
+    with open(local_path("data", "options.yaml")) as f:
+        template = Template(f.read())
+
+    try:
+        option_groups = Options.get_option_groups(world)
+        presets = world.web.options_presets.copy()
+        presets.update({"": {}})
+        for name, preset in presets.items():
+            template.render(
+                option_groups=option_groups,
+                __version__=__version__,
+                game=game,
+                world_version=world.world_version.as_simple_string(),
+                yaml_dump=yaml_dump_scalar,
+                dictify_range=dictify_range,
+                cleandoc=cleandoc,
+                preset_name=name,
+                preset=preset,
+            )
+    except Exception as error:
+        raise WebHostIncompatibleError(
+            f"passes generation but would crash WebHost's own startup "
+            f"(generate_yaml_templates): {type(error).__name__}: {error}"
+        ) from error
+
+
+def core_worlds(exclude):
+    """Every world Archipelago ships, minus the one under test.
+
+    Core worlds are the safe companions: they are part of the build
+    itself, so they are always present and always match the running
+    version - unlike an apworld, which might be missing or broken for
+    reasons that have nothing to do with the world being tested.
+    """
+    from worlds.AutoWorld import AutoWorldRegister
+    return sorted(
+        name for name, world_type in AutoWorldRegister.world_types.items()
+        if not str(getattr(world_type, "zip_path", "") or "") and name != exclude
+    )
+
+
+def random_options(game):
+    """One concrete random value per option of `game`.
+
+    Only the option types with a defined value space are randomized -
+    toggles, choices and ranges. Free text, option lists/sets and item
+    dicts (start inventory, plando, exclusions...) have no meaningful
+    "random", and forcing one would fail generation for reasons that have
+    nothing to do with the world being tested.
+
+    The values are RESOLVED here rather than written into the yaml as the
+    string "random". Both randomized modes then run the exact same option
+    values, so a difference between them is attributable to the extra
+    players and nothing else - writing "random" would have each mode roll
+    its own values from its own seed, and SR-passes-while-MR-fails would
+    say nothing about the multiworld.
+
+    Resolution goes through the option's own from_text("random"), so the
+    values are exactly the ones Archipelago itself would roll: weighted
+    ranges, named-range specials and all.
+    """
+    from worlds.AutoWorld import AutoWorldRegister
+    import Options
+
+    randomizable = (Options.Toggle, Options.Choice, Options.Range)
+    world_type = AutoWorldRegister.world_types[game]
+    chosen = {}
+    for name, option in getattr(world_type.options_dataclass, "type_hints", {}).items():
+        if not issubclass(option, randomizable):
+            continue
+        try:
+            chosen[name] = option.from_text("random").value
+        except Exception:
+            # A world can define an option whose randomization raises;
+            # leaving it out means it keeps its default rather than
+            # failing the whole mode for one unrollable option.
+            continue
+    return chosen
+
+
+def generate_one(games, dest, results, options_per_game=None, seed=None, spoiler=3):
+    # games is the full player list: the world under test first, then any
+    # companions (see RunSettings). One entry is the single-player
+    # test, the same game twice is the duplicate test, and the world plus
+    # companions is the multi-game test - all the same code path, differing
+    # only in who is in the seed and what options they roll.
+    warnings.simplefilter("ignore")
+    try:
+        from tempfile import TemporaryDirectory
+
+        import Generate
+        import Main
+
+        ensure_tutorials_default()
+
+        with TemporaryDirectory() as players_dir, TemporaryDirectory() as output_dir:
+            for number, game in enumerate(games, start=1):
+                player_path = Path(players_dir) / f"{number}.yaml"
+                # {} means every option at its default - the only point in
+                # option space the pipeline used to cover.
+                game_options = (options_per_game or {}).get(game, {})
+                player_path.write_text(json.dumps({
+                    "name": f"Tester{number}",
+                    "game": game,
+                    game: game_options,
+                    "description": f"ArchiUpdater compatibility test: {game}",
+                }), encoding="utf-8")
+
+            sys.argv = [
+                sys.argv[0],
+                "--player_files_path", players_dir,
+                "--outputpath", output_dir,
+                # Spoiler 3 computes the full playthrough, which exercises
+                # accessibility/reachability logic that plain generation
+                # never reaches - "a seed came out" becomes "the seed is
+                # actually completable".
+                "--spoiler", str(spoiler),
+            ]
+            if seed is not None:
+                sys.argv += ["--seed", str(seed)]
+            Main.main(*Generate.main())
+            check_webhost_compatibility(games[0])
+
+            output_files = list(Path(output_dir).glob("*.zip"))
+            if len(output_files) != 1:
+                raise RuntimeError(
+                    f"expected exactly one output file, found {len(output_files)}")
+            final_file = Path(dest) / output_files[0].name
+            # rename() can't cross the tmpdir -> bind-mounted output volume
+            # boundary, so move (copy + delete) instead.
+            shutil.move(str(output_files[0]), str(final_file))
+            results.append(str(final_file))
+    except BaseException as error:
+        results.append(error)
+        raise
+
+
+def describe_load_failure(log_text, apworld_path):
+    """The real reason an apworld did not load, out of the swallowed log.
+
+    worlds/__init__.py catches an import failure per world, formats the
+    traceback, and hands it to logging.exception - so by default it goes
+    nowhere and the pipeline could only report "found 0 worlds", which
+    says nothing an author could act on. Untitled Goose Game, for
+    instance, defines `option_random` on a Choice, which Archipelago
+    reserves; the assert fires at class-definition time during import.
+
+    Returns (summary, traceback) - summary being the exception and the
+    line in the world's own code that raised it.
+    """
+    marker = "Could not load world"
+    TRACEBACK_HEADER = "Traceback (most recent call last):"
+    blocks = [block for block in log_text.split(marker) if apworld_path in block]
+    if not blocks:
+        return None, None
+    block = marker + blocks[-1]
+    lines = [line for line in block.splitlines() if line.strip()]
+
+    # The exception itself is the last line of a traceback.
+    exception = next((line.strip() for line in reversed(lines)
+                      if line.strip() and not line.startswith((" ", "\t"))
+                      and "Traceback" not in line and marker not in line), None)
+
+    # The deepest frame inside the apworld itself - core frames above it
+    # are just the import machinery and tell an author nothing.
+    where = None
+    for line in lines:
+        match = re.search(r'File "([^"]*%s[^"]*)", line (\d+)' % re.escape(apworld_path), line)
+        if match:
+            inside = match.group(1).split(apworld_path + "/", 1)[-1]
+            where = f"{inside}, line {match.group(2)}"
+
+    summary = exception or "import failed"
+    if where:
+        summary += f" (in {where})"
+
+    # logging.exception both formats the traceback into the message AND
+    # appends exc_info's own copy, so the block holds it twice - keep one.
+    first = block.find(TRACEBACK_HEADER)
+    if first != -1:
+        second = block.find(TRACEBACK_HEADER, first + len(TRACEBACK_HEADER))
+        if second != -1:
+            block = block[:second]
+
+    # Drop the import machinery. Every one of these frames is identical
+    # for every failing world and tells an author nothing; what is left is
+    # their own code and the line that actually raised.
+    # A frame is its File line plus the indented source and caret lines
+    # under it, so drop the whole frame - removing only the File line
+    # leaves orphaned source fragments behind.
+    noise = ("<frozen importlib", "importlib/__init__.py", "worlds/__init__.py", "_bootstrap")
+    kept = []
+    skipping = False
+    for line in block.splitlines():
+        stripped = line.lstrip()
+        if stripped.startswith("File "):
+            skipping = any(n in line for n in noise)
+            if skipping:
+                continue
+        elif skipping:
+            # Continuation of a dropped frame: still indented, and not the
+            # start of the next frame.
+            if line.startswith(" ") or not line.strip():
+                continue
+            skipping = False
+        kept.append(line)
+    return summary, "\n".join(kept)
+
+
+def rooted_in(error, exception_type):
+    seen = set()
+    while error is not None and id(error) not in seen:
+        seen.add(id(error))
+        if isinstance(error, exception_type):
+            return True
+        error = error.__cause__ or error.__context__
+    return False
+
+
+def rooted_in_named(error, class_name):
+    # Some apworlds define their own exception class inside their own
+    # zipimported package (e.g. CTJoT's InvalidYamlException) rather
+    # than a stable top-level module - there's no fixed path to import
+    # and isinstance-check it the way Fill.FillError works, so match by
+    # class name instead.
+    seen = set()
+    while error is not None and id(error) not in seen:
+        seen.add(id(error))
+        if type(error).__name__ == class_name:
+            return True
+        error = error.__cause__ or error.__context__
+    return False
+
+
+def run_with_timeout(games, dest, timeout, options_per_game=None, seed=None, spoiler=3):
+    # fork, not spawn. Every mode runs in its own process so a generation
+    # can be killed on timeout and cannot leak state into the next one -
+    # but spawn re-imports the whole of Archipelago in each child, five
+    # times per apworld, which measured as the single largest cost in a
+    # run. Forking inherits the import the parent already did.
+    #
+    # The isolation that matters is unchanged: the child still gets its
+    # own copy-on-write memory, so whatever generation mutates dies with
+    # it and the parent's registry stays pristine for the next mode.
+    #
+    # Safe here specifically because the forking process is
+    # single-threaded: main() starts no threads of its own, and the
+    # Manager below is a separate process rather than a thread in this
+    # one. Forking a multi-threaded parent is the case to avoid, and this
+    # is not one.
+    try:
+        set_start_method("fork")
+    except RuntimeError:
+        pass
+    manager = Manager()
+    results = manager.list()
+    process = Process(target=generate_one,
+                      args=(games, dest, results, options_per_game, seed, spoiler))
+    started = time.monotonic()
+    process.start()
+    process.join(timeout)
+    timed_out = process.is_alive()
+    if timed_out:
+        process.terminate()
+        process.join(5)
+        if process.is_alive():
+            process.kill()
+            process.join()
+    elapsed = time.monotonic() - started
+    return list(results), timed_out, elapsed
+
+
+def classify(results, timed_out, elapsed, single_player=True, randomized=False):
+    if timed_out:
+        return "failed", f"timed out after {elapsed:.0f}s"
+    if not results:
+        return "failed", "generation process exited without a result"
+    result = results[0]
+    if isinstance(result, BaseException):
+        if rooted_in(result, FileNotFoundError):
+            return "needs_input", str(result)
+        if randomized and rooted_in_named(result, "OptionError"):
+            # The world rejected the option combination we rolled - not a
+            # crash and not a fill failure, but the world validating its
+            # own input. Archipelago rolls every option independently, so
+            # a world with interdependent options (Blender's min/max
+            # similarity percent, for example) can always be handed a
+            # combination it considers invalid. A player writing
+            # "random" for both hits exactly this, so it is worth
+            # reporting - but as its own thing, not as a defect.
+            #
+            # Only when the options were randomized: a world that rejects
+            # its OWN DEFAULTS is genuinely broken and stays "failed".
+            return "invalid_options", f"rejected the rolled options: {result}"
+        from Fill import FillError
+        if rooted_in(result, FillError):
+            # A fill failure is a failure in both modes. This used to be
+            # excused in single-player on the theory that some games'
+            # item/location balance "only works out in a real multiworld"
+            # - the multi-game test exists precisely to check that theory,
+            # and it does not hold (Dead Cells fails both ways), so the
+            # excuse is gone: a seed that cannot be filled is a seed that
+            # cannot be generated, whoever else is in it.
+            where = "a multi-game seed" if not single_player else "a single-player seed"
+            return "failed", f"fill failed in {where}: {result}"
+        if rooted_in_named(result, "InvalidYamlException"):
+            # Some worlds (e.g. CTJoT) require a yaml pre-generated by an
+            # external tool and reject our generic single-player yaml
+            # outright - not a defect, just incompatible with this
+            # testing methodology.
+            return "passed", f"requires an externally-generated yaml (not a defect): {result}"
+        return "failed", f"{type(result).__name__}: {result}"
+    return "passed", str(result)
+
+
+def resolve_core_game(name, world_types):
+    """Match a sheet name to a world Archipelago actually registers.
+
+    The sheet and the worlds disagree about wording often enough that an
+    exact comparison finds only 72 of the 81 core rows. The rest differ
+    by an accent ("Pokemon Emerald"), a hyphen ("Choo-Choo Charles"),
+    capitalisation ("EarthBound"), or a series prefix the sheet adds and
+    the world does not ("The Legend of Zelda: Ocarina of Time").
+
+    Tried in order: the name as written, then folded to letters and
+    digits, then either side of a colon folded the same way. Either side,
+    because the sheet puts the distinguishing part before the colon as
+    often as after it - "Super Mario Land 2: The Golden Coins" is
+    registered as "Super Mario Land 2", while "The Legend of Zelda:
+    Ocarina of Time" is registered as "Ocarina of Time".
+    """
+    if name in world_types:
+        return name
+    by_slug = {}
+    for registered in world_types:
+        by_slug.setdefault(slug(registered), registered)
+    candidates = [name]
+    if ":" in name:
+        head, tail = name.split(":", 1)
+        candidates += [tail, head]
+    for candidate in candidates:
+        registered = by_slug.get(slug(candidate))
+        if registered:
+            return registered
+    return None
+
+
+# Deliberate copy of Name.slug in core/model/name.py. This file is mounted
+# alone into the container, with no access to this checkout's
+# packages, so it cannot import it - but the two must stay identical
+# or a world folds to one key here and a different one on the host.
+def slug(text):
+    return re.sub(r"[^a-z0-9]", "", unicodedata.normalize("NFKD", (text or "").lower()))
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    # Exactly one of these. --apworld is the normal case: a downloaded
+    # file, identified by which world claims it as its zip_path. --game
+    # names a world that is already part of the Archipelago build, which
+    # has no file to point at and so cannot be identified that way.
+    parser.add_argument("--apworld")
+    parser.add_argument("--game", help="Test a world that ships with Archipelago, by name, "
+                         "instead of an apworld file. Mutually exclusive with --apworld.")
+    parser.add_argument("--output-dir", required=True)
+    parser.add_argument("--timeout", type=int, default=300)
+    parser.add_argument("--companion-pool", default="",
+                         help="Comma-separated core games to draw companions from. Empty "
+                              "means every core world, which risks drawing one that cannot "
+                              "generate here (see container/verify_companions.py).")
+    parser.add_argument("--companion-min", type=int, default=2,
+                         help="Fewest core games to draw as companions for a multi-game seed.")
+    parser.add_argument("--companion-max", type=int, default=5,
+                         help="Most core games to draw as companions for a multi-game seed.")
+    parser.add_argument("--random-repeats", type=int, default=3,
+                         help="How many times the randomized-option modes run. Each attempt "
+                              "rolls a different option combination.")
+    parser.add_argument("--repeats", type=int, default=10,
+                         help="How many times to run the single-game and multi-game tests. "
+                              "Stability is a pass rate, not a single verdict.")
+    parser.add_argument("--spoiler", type=int, default=3,
+                         help="Spoiler level passed to Generate. 3 computes the full "
+                              "playthrough, exercising accessibility logic.")
+    args = parser.parse_args()
+    if bool(args.apworld) == bool(args.game):
+        parser.error("pass exactly one of --apworld or --game")
+
+    # Determine the game name from the loaded World class itself rather
+    # than the apworld's archipelago.json manifest: older apworlds don't
+    # ship a manifest at all, and the core (<0.7.0 here) tolerates that
+    # and loads them anyway - a manifest-only lookup would misreport a
+    # perfectly loadable world as failed. World.zip_path is set only for
+    # worlds loaded from a .apworld zip (worlds/AutoWorld.py), and equals
+    # the apworld's own path, so it identifies our one mounted world
+    # exactly regardless of manifest presence.
+    # worlds/__init__.py logs each failed world's traceback and carries
+    # on; without a handler here that detail is lost and all the pipeline
+    # can say is "found 0 worlds".
+    load_log = io.StringIO()
+    log_handler = logging.StreamHandler(load_log)
+    log_handler.setLevel(logging.ERROR)
+    logging.getLogger().addHandler(log_handler)
+    try:
+        import worlds
+        from worlds.AutoWorld import AutoWorldRegister
+    finally:
+        logging.getLogger().removeHandler(log_handler)
+
+    ensure_tutorials_default()
+
+    if args.game:
+        # A core world is already loaded - there is no file to identify
+        # it by, and none of the load-failure reporting below applies. It
+        # either exists in this build or it does not, and "does not" is a
+        # naming problem on the sheet rather than a defect in a world.
+        resolved = resolve_core_game(args.game, AutoWorldRegister.world_types)
+        if resolved:
+            args.game = resolved
+        else:
+            # "unknown", not "unverified". Nothing about this world is
+            # broken - this Archipelago build simply has no world under
+            # that name, which is a naming mismatch on the sheet (or a
+            # world that has since been removed from core), and calling
+            # it Broken would blame a world for the sheet's wording.
+            print(json.dumps({
+                "game": None,
+                "outcome": "unknown",
+                "detail": (f"this Archipelago build registers no world named {args.game!r}, "
+                            f"so there was nothing to test. The sheet lists this row as Core, "
+                            f"so either its name differs from the world's own or the world is "
+                            f"no longer part of Archipelago."),
+                "elapsed_seconds": 0,
+            }))
+            return 1
+        custom_games = [args.game]
+    else:
+        custom_games = [
+            name for name, world_type in AutoWorldRegister.world_types.items()
+            if str(getattr(world_type, "zip_path", "") or "") == args.apworld
+        ]
+    if len(custom_games) != 1:
+        # Its own outcome, NOT "passed". This covers several distinct
+        # causes (a duplicate game name against a now-built-in world, an
+        # import-time crash, a file that isn't a valid apworld at all,
+        # a genuine collision between two unrelated apworlds) - none of
+        # them necessarily an apworld defect, but none of them a world
+        # this pipeline has verified either. Calling it "passed" promoted
+        # files that never loaded and crash-looped real WebHost startup
+        # Factorio Platformer did exactly that: not a valid zip, zero
+        # worlds loaded, recorded as a pass, promoted, and crash-looping
+        # real WebHost startup - it needed a hand-written blacklist entry
+        # to undo. promote.py takes "passed" alone, so this verdict now
+        # keeps such a file out of worlds_live/ on its own.
+        summary, traceback_text = describe_load_failure(load_log.getvalue(), args.apworld)
+        if len(custom_games) > 1:
+            detail = (f"this file registers {len(custom_games)} worlds, not one: "
+                      f"{', '.join(custom_games)}. Only a single-world apworld can be "
+                      f"promoted, since the pipeline cannot tell which one this row means.")
+        elif summary:
+            detail = f"the world failed to import: {summary}"
+            if traceback_text:
+                detail += f"\n\n{traceback_text.strip()}"
+        else:
+            # No captured traceback: the file is not a loadable apworld at
+            # all (not a zip, wrong layout), so nothing was even attempted.
+            detail = (f"no world loaded from this file and no import was attempted - it is "
+                      f"probably not a valid .apworld (failed_world_loads="
+                      f"{worlds.failed_world_loads})")
+        print(json.dumps({
+            "game": None,
+            "outcome": "unverified",
+            "detail": detail,
+            "elapsed_seconds": 0,
+        }))
+        return 1
+
+    game = custom_games[0]
+
+    Path(args.output_dir).mkdir(parents=True, exist_ok=True)
+
+    def run_mode(games, options_per_game=None, seed=None, single_player=None, randomized=False):
+        if single_player is None:
+            single_player = len(games) == 1
+        mode_results, timed_out, elapsed = run_with_timeout(
+            games, args.output_dir, args.timeout,
+            options_per_game=options_per_game, seed=seed, spoiler=args.spoiler)
+        outcome, detail = classify(mode_results, timed_out, elapsed,
+                                    single_player=single_player, randomized=randomized)
+        return {"outcome": outcome, "detail": detail, "elapsed_seconds": round(elapsed, 1),
+                "timed_out": timed_out}
+
+    # A timeout anywhere abandons the rest of this apworld. One world that
+    # hangs would otherwise cost timeout x repeats x modes - by far the
+    # largest thing in a run - and a world that cannot finish one seed has
+    # nothing useful to say about its stability across ten.
+    state = {"timed_out": False}
+
+    def skipped(reason):
+        return {"outcome": "unknown", "detail": reason, "elapsed_seconds": None}
+
+    def run_repeated(players, repeats, single_player=None, options=None, randomized=False):
+        """The same test several times, reported as a pass rate.
+
+        Stability is not something a single generation can tell you: S, D
+        and M each get a different Archipelago seed, so the fill differs
+        every time, and a world that places items successfully four times
+        in five is genuinely unstable. Running once records whichever
+        result we happened to get.
+
+        Each repeat uses an explicit seed so a failure among ten passes is
+        reproducible - without it the interesting case is the one that
+        cannot be re-examined.
+        """
+        attempts = []
+        for _ in range(max(repeats, 1)):
+            seed = random.randint(0, 2 ** 31 - 1)
+            # players is a callable so the multi-game test can draw a
+            # different set of companions every attempt - ten runs against
+            # one fixed pair only ever tests that pair.
+            games = players() if callable(players) else players
+            # A callable, so the randomized modes can roll a different
+            # point in the option space for every attempt - repeating one
+            # roll would only re-measure the same combination.
+            per_game = options() if callable(options) else options
+            result = run_mode(games, options_per_game=per_game, seed=seed,
+                               single_player=single_player, randomized=randomized)
+            result["seed"] = seed
+            if len(games) > 1:
+                result["companions"] = games[1:]
+            attempts.append(result)
+            if result["timed_out"]:
+                state["timed_out"] = True
+                break
+
+        passed = [a for a in attempts if a["outcome"] == "passed"]
+        failed = [a for a in attempts if a["outcome"] != "passed"]
+        if not failed:
+            outcome, detail = "passed", f"generated {len(passed)}/{len(attempts)} times"
+        elif not passed:
+            # Every attempt agreeing on a non-failure reason keeps that
+            # reason. A world that needs a base ROM reports needs_input
+            # once and "failed all 10 attempts" ten times over, and
+            # flattening those to "failed" blames the world for a file
+            # this pipeline does not have - which matters most for core
+            # worlds, where roughly seven of them need a ROM and would
+            # otherwise all read as Broken.
+            outcomes = {attempt["outcome"] for attempt in attempts}
+            outcome = outcomes.pop() if len(outcomes) == 1 else "failed"
+            detail = f"failed all {len(attempts)} attempts. {failed[0]['detail']}"
+            if outcome != "failed":
+                detail = (f"{outcome} on all {len(attempts)} attempts. "
+                           f"{failed[0]['detail']}")
+        else:
+            # The whole point of repeating: a world that works sometimes
+            # is worse for a player than one that never does, because
+            # nobody knows to avoid it.
+            outcome = "flaky"
+            detail = (f"generated {len(passed)}/{len(attempts)} times. "
+                      f"First failure (seed {failed[0]['seed']}): {failed[0]['detail']}")
+        return {
+            "outcome": outcome,
+            "detail": detail,
+            "elapsed_seconds": round(sum(a["elapsed_seconds"] or 0 for a in attempts), 1),
+            "attempts": len(attempts),
+            # What was asked for, as opposed to what ran - a timeout cuts
+            # a mode short, and the retest cache compares against this so
+            # such a record is not re-run forever.
+            "requested_attempts": max(repeats, 1),
+            "passed_attempts": len(passed),
+            "failed_seeds": [a["seed"] for a in failed],
+            # Which companions each failing attempt drew, so a failure
+            # caused by one particular combination can be reproduced.
+            "failed_companions": [a.get("companions") for a in failed if a.get("companions")],
+        }
+
+    tests = {}
+
+    # 1. the world on its own, every option at its default.
+    tests["single"] = run_repeated([game], args.repeats)
+    tests["single"]["label"] = "Single game"
+
+    # 2. the same world twice. A world that keeps state on the class
+    #    rather than the instance works alone and corrupts itself here -
+    #    a bug class no other mode can see.
+    tests["duplicate"] = (skipped("skipped - an earlier test timed out") if state["timed_out"]
+                           else run_mode([game, game], single_player=False))
+    tests["duplicate"]["label"] = "Two players, same game"
+
+    # 3. the world alongside other games. Each attempt draws a fresh
+    #    random selection of core worlds rather than reusing one fixed
+    #    pair: ten runs against the same two companions only ever tell you
+    #    about those two, while a different combination each time samples
+    #    what a real multiworld looks like.
+    configured_pool = [name.strip() for name in args.companion_pool.split(",") if name.strip()]
+    if configured_pool:
+        # Verified by the host once per run: every one of these generated
+        # a seed on its own in this exact environment, so a multi-game
+        # failure is attributable to the world under test rather than to
+        # a companion that was never going to work.
+        pool = [name for name in configured_pool if name != game]
+    else:
+        pool = core_worlds(exclude=game)
+    low = max(args.companion_min, 1)
+    high = max(args.companion_max, low)
+
+    def draw_companions():
+        count = min(random.randint(low, high), len(pool))
+        return random.sample(pool, count)
+
+    if state["timed_out"]:
+        tests["multi"] = skipped("skipped - an earlier test timed out")
+    elif pool:
+        tests["multi"] = run_repeated(
+            lambda: [game] + draw_companions(), args.repeats, single_player=False)
+        tests["multi"]["companion_pool"] = len(pool)
+        tests["multi"]["companion_range"] = [low, high]
+        tests["multi"]["detail"] += f" (against {low}-{high} core games drawn from {len(pool)})"
+    else:
+        tests["multi"] = skipped("no core worlds available to pair with")
+    tests["multi"]["label"] = "Multi game"
+
+    # 4 and 5. The same two shapes as above - the world alone, and the
+    #    world alongside the companions - but with its options randomized
+    #    instead of left at their defaults. The default option set is a
+    #    single point in a world's option space and most option-dependent
+    #    crashes are nowhere near it.
+    #
+    #    One roll each, and every roll that does not produce a seed is a
+    #    failure, including one the world rejects with OptionError. That
+    #    last point is the whole reason these modes exist: the yaml a
+    #    player writes is accepted as valid - nothing in the template or
+    #    the upload path objects to "random" on two interdependent
+    #    options - so the conflict only surfaces when generation is
+    #    actually attempted, possibly after everyone has submitted, and it
+    #    is intermittent. Re-rolling until the world accepts something
+    #    would hide exactly the failure a real player hits.
+    #
+    #    Each randomized run is gated on its own default-options
+    #    counterpart: SR only when the world generates alone, MR only when
+    #    it generates alongside the companions. A randomized run whose
+    #    plain equivalent already fails would fail for the same underlying
+    #    reason and cost a generation to learn nothing.
+    rolls = None
+    if state["timed_out"]:
+        aborted = "skipped - an earlier test timed out"
+        tests["single_random"] = skipped(aborted)
+        tests["multi_random"] = skipped(aborted)
+    elif tests["single"]["outcome"] == "passed" or tests["multi"]["outcome"] == "passed":
+        try:
+            # One roll per attempt, drawn up front so SR and MR run the
+            # SAME combinations as each other. Pairing them is what makes
+            # "SR passed but MR failed" attributable to the extra players
+            # rather than to two different option sets.
+            rolls = [random_options(game) for _ in range(max(args.random_repeats, 1))]
+        except BaseException as error:
+            rolls = None
+            unreadable = f"could not read this world's options: {type(error).__name__}: {error}"
+            tests["single_random"] = skipped(unreadable)
+            tests["multi_random"] = skipped(unreadable)
+
+    if rolls:
+        def roller():
+            # Walks the pre-drawn rolls in order, so attempt N of SR and
+            # attempt N of MR use the same option values.
+            supply = iter(rolls)
+            def next_roll():
+                return {game: next(supply)}
+            return next_roll
+
+        # "flaky" counts as generating: the world produced a seed at
+        # least once, so a randomized run can still tell us something.
+        if tests["single"]["outcome"] in ("passed", "flaky"):
+            result = run_repeated([game], len(rolls), single_player=True,
+                                   options=roller(), randomized=True)
+            result["randomized_options"] = len(rolls[0])
+            tests["single_random"] = result
+        else:
+            tests["single_random"] = skipped(
+                "skipped - the single-game test did not pass")
+
+        if tests["multi"]["outcome"] in ("passed", "flaky"):
+            result = run_repeated(lambda: [game] + draw_companions(), len(rolls),
+                                   single_player=False, options=roller(), randomized=True)
+            result["randomized_options"] = len(rolls[0])
+            tests["multi_random"] = result
+        else:
+            tests["multi_random"] = skipped(
+                "skipped - the multi-game test did not pass")
+    elif "single_random" not in tests:
+        neither = "skipped - neither the single-game nor the multi-game test passed"
+        tests["single_random"] = skipped(neither)
+        tests["multi_random"] = skipped(neither)
+
+    tests["single_random"]["label"] = "Random options, alone"
+    tests["multi_random"]["label"] = "Random options, multi game"
+
+    # The top-level fields stay exactly what they were - the single-player
+    # verdict - because that is what promotion and every existing consumer
+    # reads. The per-mode results live alongside it.
+    report = {
+        "game": game,
+        "outcome": tests["single"]["outcome"],
+        "detail": tests["single"]["detail"],
+        "elapsed_seconds": tests["single"]["elapsed_seconds"],
+        "multi_outcome": tests["multi"]["outcome"],
+        "multi_detail": tests["multi"]["detail"],
+        "multi_elapsed_seconds": tests["multi"]["elapsed_seconds"],
+        "multi_companions": tests["multi"].get("companions", []),
+        "tests": tests,
+    }
+
+    print(json.dumps(report))
+    # Exit code still reflects the single-player result alone: it is what
+    # gates promotion, and the other modes are reported rather than
+    # blocking.
+    return {"passed": 0, "needs_input": 2}.get(tests["single"]["outcome"], 1)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

container/drivers/verify_companions.py

@@ -0,0 +1,65 @@
+"""Runs inside the archipelago Docker image. Reports which core worlds
+can actually generate a seed in THIS environment, so the multi-game tests
+only ever draw companions that work.
+
+Core worlds are shipped with Archipelago, but that does not make them all
+usable here: some need a base ROM this container has no copy of, and at
+least one (Final Fantasy) needs a settings yaml generated by an external
+website and refuses a generic one. Drawing such a world as a companion
+fails the whole seed, and the failure gets recorded against the apworld
+under test rather than the companion that caused it - Blender was marked
+flaky 7/10 purely because three of its ten draws included worlds like
+these.
+
+Rather than maintain an exclusion list by hand, this generates one solo
+seed per core world and keeps the ones that succeed. A world drops out
+by itself when its ROM is missing, and comes back by itself once the ROM
+is added.
+
+Shares run_test.py's generation machinery rather than repeating it, so
+"can generate" means exactly the same thing in both places.
+"""
+
+import argparse
+import json
+import sys
+import warnings
+
+sys.path.insert(0, "/app")
+
+import run_test  # noqa: E402 - mounted alongside this script
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output-dir", required=True)
+    parser.add_argument("--timeout", type=int, default=120,
+                         help="Per-world timeout. Deliberately shorter than the apworld "
+                              "timeout: a companion that is slow to generate would multiply "
+                              "across every multi-game attempt that draws it.")
+    parser.add_argument("--spoiler", type=int, default=2)
+    args = parser.parse_args()
+
+    warnings.simplefilter("ignore")
+    run_test.ensure_tutorials_default()
+
+    from pathlib import Path
+    Path(args.output_dir).mkdir(parents=True, exist_ok=True)
+
+    verified = []
+    rejected = {}
+    for game in run_test.core_worlds(exclude=None):
+        results, timed_out, elapsed = run_test.run_with_timeout(
+            [game], args.output_dir, args.timeout, spoiler=args.spoiler)
+        outcome, detail = run_test.classify(results, timed_out, elapsed)
+        if outcome == "passed":
+            verified.append(game)
+        else:
+            rejected[game] = f"{outcome}: {detail}"[:300]
+
+    print(json.dumps({"verified": verified, "rejected": rejected}))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

notebooks/tester_playground.ipynb

@@ -0,0 +1,66 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Testing worlds\n",
+    "\n",
+    "`TestSession` prints the paths it uses, downloads the apworld if it isn't\n",
+    "there yet, then runs all five generation modes and prints the verdicts.\n",
+    "\n",
+    "Results go to `scratch_state.json` under the data directory, not the live\n",
+    "`state.json` - pass `live=True` when a result is meant to count.\n",
+    "\n",
+    "Needs the package installed: `pip install -e .` from the repository root."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## One world"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "from archipelago_tester.pipeline.batch.session import TestSession\n",
+    "\n",
+    "TestSession(\"Baba Is You\").run()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Several worlds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "TestSession([\"Baba Is You\", \"Anodyne\", \"Axiom Verge\"]).run()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "pygments_lexer": "ipython3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "APWorldTester"
+version = "0.1.0"
+description = "Tests Archipelago community worlds by generating seeds with them"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.10"
+dependencies = [
+    "requests",
+    "beautifulsoup4",
+    "pyyaml",
+]
+
+[project.scripts]
+apworld-tester = "apworld_tester.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+apworld_tester = [
+    "templates/*.yaml",
+    "container/Dockerfile.test",
+    "container/drivers/*.py",
+]

src/apworld_tester/__init__.py

@@ -0,0 +1,55 @@
+"""Tests Archipelago community worlds by generating seeds with them.
+
+The publishing half - archipelago-world-site - imports this package as a
+dependency. Nothing here knows that it exists.
+
+Everything downstream needs is re-exported here, so callers import from
+``apworld_tester`` and never from the module layout underneath it.
+"""
+
+from apworld_tester.core.config.config import Config
+from apworld_tester.core.config.paths import Paths
+from apworld_tester.core.model.apworld import ApworldFile
+from apworld_tester.core.model.blacklist import Blacklist
+from apworld_tester.core.model.fingerprint import Fingerprint
+from apworld_tester.core.model.game import Game
+from apworld_tester.core.model.instant import Instant
+from apworld_tester.core.model.modes import Modes
+from apworld_tester.core.model.name import Name
+from apworld_tester.core.model.verdict import Verdict
+from apworld_tester.core.state.keys import StateKeys
+from apworld_tester.core.state.lock import LockHeldError, PipelineLock
+from apworld_tester.core.state.store import StateStore
+from apworld_tester.pipeline.batch.run import TestRun
+from apworld_tester.pipeline.build.archipelago import ArchipelagoBuild
+from apworld_tester.pipeline.build.docker import Docker
+from apworld_tester.pipeline.download.update_history import UpdateHistory
+from apworld_tester.pipeline.generation.command import ContainerCommand
+from apworld_tester.pipeline.sheet.reader import SheetReader
+from apworld_tester.pipeline.update.stage_error import StageError
+from apworld_tester.pipeline.update.update_run import UpdateRun
+
+__all__ = [
+    "ApworldFile",
+    "ArchipelagoBuild",
+    "Blacklist",
+    "Config",
+    "ContainerCommand",
+    "Docker",
+    "Fingerprint",
+    "Game",
+    "Instant",
+    "LockHeldError",
+    "Modes",
+    "Name",
+    "Paths",
+    "PipelineLock",
+    "SheetReader",
+    "StageError",
+    "StateKeys",
+    "StateStore",
+    "TestRun",
+    "UpdateHistory",
+    "UpdateRun",
+    "Verdict",
+]

src/apworld_tester/cli.py

@@ -0,0 +1,101 @@
+"""The command line: getting a config file into place.
+
+The package is installed rather than cloned, so the starter config
+cannot be "the example beside this script" - it is shipped as package
+data and copied out on request.
+"""
+
+import argparse
+import os
+import shutil
+import sys
+
+from importlib import resources
+
+from apworld_tester.core.config.config import Config, ConfigNotFound
+
+
+class ConfigTemplate:
+    """The starter config.yaml carried inside the package."""
+
+    PACKAGE = "apworld_tester.templates"
+    NAME = "config.yaml"
+
+    @classmethod
+    def read(cls):
+        return (resources.files(cls.PACKAGE) / cls.NAME).read_text(
+            encoding="utf-8")
+
+
+class InitCommand:
+    """Write a starter config where the search order will find it."""
+
+    def __init__(self, destination=None, force=False):
+        self.destination = destination or Config.user_path()
+        self.force = force
+
+    def run(self):
+        if os.path.exists(self.destination) and not self.force:
+            print(f"config already exists: {self.destination}")
+            print("pass --force to overwrite it")
+            return 1
+        os.makedirs(os.path.dirname(self.destination), exist_ok=True)
+        if os.path.exists(self.destination):
+            shutil.copyfile(self.destination, f"{self.destination}.bak")
+            print(f"kept the old one as {self.destination}.bak")
+        with open(self.destination, "w", encoding="utf-8") as handle:
+            handle.write(ConfigTemplate.read())
+        print(f"wrote {self.destination}")
+        print("Edit general.output_directory and general.roms_directory "
+              "before the first run.")
+        return 0
+
+
+class PathCommand:
+    """Say which config would be used, and what was tried."""
+
+    def run(self):
+        try:
+            print(Config.default_path())
+            return 0
+        except ConfigNotFound as error:
+            print(error, file=sys.stderr)
+            return 1
+
+
+class Cli:
+    """One entry point, so an installed copy is self-describing."""
+
+    @staticmethod
+    def parser():
+        parser = argparse.ArgumentParser(
+            prog="apworld-tester", description=__doc__)
+        commands = parser.add_subparsers(dest="command", required=True)
+        init = commands.add_parser(
+            "init", help="Write a starter config.yaml and say where.")
+        init.add_argument(
+            "--path", default=None,
+            help="Write here instead of the user config directory.")
+        init.add_argument(
+            "--force", action="store_true",
+            help="Overwrite an existing config, keeping a .bak copy.")
+        commands.add_parser(
+            "config-path",
+            help="Print the config that would be used, or where it "
+                 "was looked for.")
+        return parser
+
+    @classmethod
+    def main(cls, argv=None):
+        args = cls.parser().parse_args(argv)
+        if args.command == "init":
+            return InitCommand(args.path, args.force).run()
+        return PathCommand().run()
+
+
+def main(argv=None):
+    return Cli.main(argv)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/apworld_tester/container/Dockerfile.test

@@ -0,0 +1,20 @@
+# Unprivileged wrapper for the generation tests.
+#
+# Upstream's image runs as root and leaves /.local root-owned, so running
+# it as the host user fails before generation starts (PermissionError on
+# /.local) and running it as root leaves root-owned seeds in the mounted
+# output directory. Two lines fix both, and they are the only difference
+# from the base image.
+#
+# Built once per Archipelago tag, not per apworld test, so the cost of
+# the recursive chown is paid once and amortised over a few hundred
+# generations.
+ARG BASE_IMAGE=archipelago:latest
+FROM ${BASE_IMAGE}
+
+ARG RUNNER_UID=1000
+ARG RUNNER_GID=1000
+
+RUN chown -R ${RUNNER_UID}:${RUNNER_GID} /app /.local 2>/dev/null || chown -R ${RUNNER_UID}:${RUNNER_GID} /app
+
+USER ${RUNNER_UID}:${RUNNER_GID}

src/apworld_tester/container/drivers/identify_apworld.py

@@ -0,0 +1,49 @@
+"""Runs inside the archipelago Docker image. Reports which game (if any)
+a single mounted .apworld registers itself as under Archipelago's own
+World registry - the import/registration step only, no seed generation.
+
+Used to verify a freshly downloaded .apworld's actual content matches
+the sheet row that fetched it, rather than trusting release/asset
+name-matching alone (see AssetMatcher): the filename
+and release title are just text an author chose, but AutoWorldRegister
+is filled from the World class's own "game" attribute, which is what
+Archipelago itself will treat this file as - the authoritative source.
+"""
+
+import argparse
+import json
+import sys
+import warnings
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--apworld", required=True)
+    args = parser.parse_args()
+
+    warnings.simplefilter("ignore")
+
+    try:
+        import worlds
+        from worlds.AutoWorld import AutoWorldRegister
+    except BaseException as error:
+        # The apworld (or something it imports) crashed outright - that's
+        # itself a meaningful signal (this file is broken/wrong), not an
+        # infrastructure failure, so it's still reported as a clean JSON
+        # result rather than a non-zero exit / stack trace.
+        print(json.dumps({"games": [], "detail": f"{type(error).__name__}: {error}"}))
+        return 0
+
+    games = [
+        name for name, world_type in AutoWorldRegister.world_types.items()
+        if str(getattr(world_type, "zip_path", "") or "") == args.apworld
+    ]
+    print(json.dumps({
+        "games": games,
+        "detail": None if games else f"failed_world_loads={worlds.failed_world_loads}",
+    }))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/apworld_tester/container/drivers/run_test.py

@@ -0,0 +1,827 @@
+"""Runs inside the archipelago Docker image. Generates one seed for a
+single installed apworld and reports the outcome as one JSON line on
+stdout: passed, needs_input (missing base ROM or similar), or failed.
+
+The logic mirrors Archipelago's own test/hosting/generate.py, but is
+copied here rather than imported from it: the project's .dockerignore
+excludes test/ from the image, so that module isn't present at runtime.
+"""
+
+import argparse
+import io
+import json
+import logging
+import random
+import re
+import shutil
+import sys
+import unicodedata
+import time
+import warnings
+from multiprocessing import Manager, Process, set_start_method
+from pathlib import Path
+
+def ensure_tutorials_default():
+    # Monkeypatch applied AFTER whichever Archipelago version's own real
+    # worlds/AutoWorld.py has already loaded normally - defaults
+    # WebWorld.tutorials to [] if that core doesn't already declare it,
+    # without needing to know or duplicate anything else about that
+    # core's own AutoWorld.py content. Previously this was a full-file
+    # bind-mount replacement (container/AutoWorld.py (absent - see config.py)) built against one
+    # specific version's source; that broke outright when tested against
+    # an older Archipelago release whose real AutoWorld.py imports
+    # differently (e.g. no rule_builder module before it was added
+    # upstream) - replacing the whole file assumes it stays
+    # source-compatible with everything else that release's core
+    # expects, which isn't true across versions. This achieves the same
+    # fix (see the original container/AutoWorld.py (absent - see config.py) comment for why it's
+    # needed - WebHost.py's invalid_worlds filter vs.
+    # network_data_package snapshot timing) without that assumption.
+    from worlds.AutoWorld import WebWorld
+    if "tutorials" not in vars(WebWorld):
+        WebWorld.tutorials = []
+
+
+class WebHostIncompatibleError(RuntimeError):
+    """Generation succeeds, but this world would crash WebHost's own
+    startup - a bug class Generate.py never exercises (see
+    check_webhost_compatibility below)."""
+
+
+def check_webhost_compatibility(game):
+    # Reproduces WebHost's own generate_yaml_templates() (Options.py),
+    # scoped to just this one world, so a bug like Option.visibility
+    # being set to a bare int instead of a Visibility flag - which
+    # crashes generate_yaml_templates() for the ENTIRE deployed site,
+    # not just the offending world, since it iterates every visible
+    # world in one loop - is caught here as a normal test failure
+    # instead of only in production. (A hand-run diagnostic that located
+    # the offending worlds used to live in container/tools/; removed.)
+    # which found this bug class live on the deployed instance.
+    from inspect import cleandoc
+
+    import yaml
+    from jinja2 import Template
+
+    import Options
+    from Utils import local_path, __version__
+    from worlds.AutoWorld import AutoWorldRegister
+
+    world = AutoWorldRegister.world_types[game]
+    if not hasattr(world.web, "tutorials"):
+        # WebHost.py strips any world missing this from AutoWorldRegister
+        # entirely at startup (its invalid_worlds set) - so it can't
+        # crash generate_yaml_templates (never part of that loop), but
+        # the world is now gone from WebHost's registry altogether. That
+        # breaks something worse than templates: MultiServer can't find
+        # the game's data (item_name_groups etc.) when starting a room,
+        # so *hosting* any seed generated with this world crashes with
+        # a KeyError - even though generation itself succeeded fine.
+        raise WebHostIncompatibleError(
+            "passes generation but has no WebWorld.tutorials - WebHost "
+            "strips it from its world registry entirely at startup, so "
+            "hosting a room for a seed generated with it crashes "
+            "MultiServer with a KeyError on the game name"
+        )
+
+    def dictify_range(option, option_val):
+        data = {option_val: 50}
+        for sub_option in ["random", "random-low", "random-high",
+                            f"random-range-{option.range_start}-{option.range_end}"]:
+            if sub_option != option_val:
+                data[sub_option] = 0
+        notes = {}
+        for name, number in getattr(option, "special_range_names", {}).items():
+            notes[name] = f"equivalent to {number}"
+            if number in data:
+                data[name] = data[number]
+                del data[number]
+            elif name in data:
+                pass
+            else:
+                data[name] = 0
+        return data, notes
+
+    def yaml_dump_scalar(scalar):
+        return yaml.dump(scalar).replace("...\n", "").strip()
+
+    with open(local_path("data", "options.yaml")) as f:
+        template = Template(f.read())
+
+    try:
+        option_groups = Options.get_option_groups(world)
+        presets = world.web.options_presets.copy()
+        presets.update({"": {}})
+        for name, preset in presets.items():
+            template.render(
+                option_groups=option_groups,
+                __version__=__version__,
+                game=game,
+                world_version=world.world_version.as_simple_string(),
+                yaml_dump=yaml_dump_scalar,
+                dictify_range=dictify_range,
+                cleandoc=cleandoc,
+                preset_name=name,
+                preset=preset,
+            )
+    except Exception as error:
+        raise WebHostIncompatibleError(
+            f"passes generation but would crash WebHost's own startup "
+            f"(generate_yaml_templates): {type(error).__name__}: {error}"
+        ) from error
+
+
+def core_worlds(exclude):
+    """Every world Archipelago ships, minus the one under test.
+
+    Core worlds are the safe companions: they are part of the build
+    itself, so they are always present and always match the running
+    version - unlike an apworld, which might be missing or broken for
+    reasons that have nothing to do with the world being tested.
+    """
+    from worlds.AutoWorld import AutoWorldRegister
+    return sorted(
+        name for name, world_type in AutoWorldRegister.world_types.items()
+        if not str(getattr(world_type, "zip_path", "") or "") and name != exclude
+    )
+
+
+def random_options(game):
+    """One concrete random value per option of `game`.
+
+    Only the option types with a defined value space are randomized -
+    toggles, choices and ranges. Free text, option lists/sets and item
+    dicts (start inventory, plando, exclusions...) have no meaningful
+    "random", and forcing one would fail generation for reasons that have
+    nothing to do with the world being tested.
+
+    The values are RESOLVED here rather than written into the yaml as the
+    string "random". Both randomized modes then run the exact same option
+    values, so a difference between them is attributable to the extra
+    players and nothing else - writing "random" would have each mode roll
+    its own values from its own seed, and SR-passes-while-MR-fails would
+    say nothing about the multiworld.
+
+    Resolution goes through the option's own from_text("random"), so the
+    values are exactly the ones Archipelago itself would roll: weighted
+    ranges, named-range specials and all.
+    """
+    from worlds.AutoWorld import AutoWorldRegister
+    import Options
+
+    randomizable = (Options.Toggle, Options.Choice, Options.Range)
+    world_type = AutoWorldRegister.world_types[game]
+    chosen = {}
+    for name, option in getattr(world_type.options_dataclass, "type_hints", {}).items():
+        if not issubclass(option, randomizable):
+            continue
+        try:
+            chosen[name] = option.from_text("random").value
+        except Exception:
+            # A world can define an option whose randomization raises;
+            # leaving it out means it keeps its default rather than
+            # failing the whole mode for one unrollable option.
+            continue
+    return chosen
+
+
+def generate_one(games, dest, results, options_per_game=None, seed=None, spoiler=3):
+    # games is the full player list: the world under test first, then any
+    # companions (see RunSettings). One entry is the single-player
+    # test, the same game twice is the duplicate test, and the world plus
+    # companions is the multi-game test - all the same code path, differing
+    # only in who is in the seed and what options they roll.
+    warnings.simplefilter("ignore")
+    try:
+        from tempfile import TemporaryDirectory
+
+        import Generate
+        import Main
+
+        ensure_tutorials_default()
+
+        with TemporaryDirectory() as players_dir, TemporaryDirectory() as output_dir:
+            for number, game in enumerate(games, start=1):
+                player_path = Path(players_dir) / f"{number}.yaml"
+                # {} means every option at its default - the only point in
+                # option space the pipeline used to cover.
+                game_options = (options_per_game or {}).get(game, {})
+                player_path.write_text(json.dumps({
+                    "name": f"Tester{number}",
+                    "game": game,
+                    game: game_options,
+                    "description": f"ArchiUpdater compatibility test: {game}",
+                }), encoding="utf-8")
+
+            sys.argv = [
+                sys.argv[0],
+                "--player_files_path", players_dir,
+                "--outputpath", output_dir,
+                # Spoiler 3 computes the full playthrough, which exercises
+                # accessibility/reachability logic that plain generation
+                # never reaches - "a seed came out" becomes "the seed is
+                # actually completable".
+                "--spoiler", str(spoiler),
+            ]
+            if seed is not None:
+                sys.argv += ["--seed", str(seed)]
+            Main.main(*Generate.main())
+            check_webhost_compatibility(games[0])
+
+            output_files = list(Path(output_dir).glob("*.zip"))
+            if len(output_files) != 1:
+                raise RuntimeError(
+                    f"expected exactly one output file, found {len(output_files)}")
+            final_file = Path(dest) / output_files[0].name
+            # rename() can't cross the tmpdir -> bind-mounted output volume
+            # boundary, so move (copy + delete) instead.
+            shutil.move(str(output_files[0]), str(final_file))
+            results.append(str(final_file))
+    except BaseException as error:
+        results.append(error)
+        raise
+
+
+def describe_load_failure(log_text, apworld_path):
+    """The real reason an apworld did not load, out of the swallowed log.
+
+    worlds/__init__.py catches an import failure per world, formats the
+    traceback, and hands it to logging.exception - so by default it goes
+    nowhere and the pipeline could only report "found 0 worlds", which
+    says nothing an author could act on. Untitled Goose Game, for
+    instance, defines `option_random` on a Choice, which Archipelago
+    reserves; the assert fires at class-definition time during import.
+
+    Returns (summary, traceback) - summary being the exception and the
+    line in the world's own code that raised it.
+    """
+    marker = "Could not load world"
+    TRACEBACK_HEADER = "Traceback (most recent call last):"
+    blocks = [block for block in log_text.split(marker) if apworld_path in block]
+    if not blocks:
+        return None, None
+    block = marker + blocks[-1]
+    lines = [line for line in block.splitlines() if line.strip()]
+
+    # The exception itself is the last line of a traceback.
+    exception = next((line.strip() for line in reversed(lines)
+                      if line.strip() and not line.startswith((" ", "\t"))
+                      and "Traceback" not in line and marker not in line), None)
+
+    # The deepest frame inside the apworld itself - core frames above it
+    # are just the import machinery and tell an author nothing.
+    where = None
+    for line in lines:
+        match = re.search(r'File "([^"]*%s[^"]*)", line (\d+)' % re.escape(apworld_path), line)
+        if match:
+            inside = match.group(1).split(apworld_path + "/", 1)[-1]
+            where = f"{inside}, line {match.group(2)}"
+
+    summary = exception or "import failed"
+    if where:
+        summary += f" (in {where})"
+
+    # logging.exception both formats the traceback into the message AND
+    # appends exc_info's own copy, so the block holds it twice - keep one.
+    first = block.find(TRACEBACK_HEADER)
+    if first != -1:
+        second = block.find(TRACEBACK_HEADER, first + len(TRACEBACK_HEADER))
+        if second != -1:
+            block = block[:second]
+
+    # Drop the import machinery. Every one of these frames is identical
+    # for every failing world and tells an author nothing; what is left is
+    # their own code and the line that actually raised.
+    # A frame is its File line plus the indented source and caret lines
+    # under it, so drop the whole frame - removing only the File line
+    # leaves orphaned source fragments behind.
+    noise = ("<frozen importlib", "importlib/__init__.py", "worlds/__init__.py", "_bootstrap")
+    kept = []
+    skipping = False
+    for line in block.splitlines():
+        stripped = line.lstrip()
+        if stripped.startswith("File "):
+            skipping = any(n in line for n in noise)
+            if skipping:
+                continue
+        elif skipping:
+            # Continuation of a dropped frame: still indented, and not the
+            # start of the next frame.
+            if line.startswith(" ") or not line.strip():
+                continue
+            skipping = False
+        kept.append(line)
+    return summary, "\n".join(kept)
+
+
+def rooted_in(error, exception_type):
+    seen = set()
+    while error is not None and id(error) not in seen:
+        seen.add(id(error))
+        if isinstance(error, exception_type):
+            return True
+        error = error.__cause__ or error.__context__
+    return False
+
+
+def rooted_in_named(error, class_name):
+    # Some apworlds define their own exception class inside their own
+    # zipimported package (e.g. CTJoT's InvalidYamlException) rather
+    # than a stable top-level module - there's no fixed path to import
+    # and isinstance-check it the way Fill.FillError works, so match by
+    # class name instead.
+    seen = set()
+    while error is not None and id(error) not in seen:
+        seen.add(id(error))
+        if type(error).__name__ == class_name:
+            return True
+        error = error.__cause__ or error.__context__
+    return False
+
+
+def run_with_timeout(games, dest, timeout, options_per_game=None, seed=None, spoiler=3):
+    # fork, not spawn. Every mode runs in its own process so a generation
+    # can be killed on timeout and cannot leak state into the next one -
+    # but spawn re-imports the whole of Archipelago in each child, five
+    # times per apworld, which measured as the single largest cost in a
+    # run. Forking inherits the import the parent already did.
+    #
+    # The isolation that matters is unchanged: the child still gets its
+    # own copy-on-write memory, so whatever generation mutates dies with
+    # it and the parent's registry stays pristine for the next mode.
+    #
+    # Safe here specifically because the forking process is
+    # single-threaded: main() starts no threads of its own, and the
+    # Manager below is a separate process rather than a thread in this
+    # one. Forking a multi-threaded parent is the case to avoid, and this
+    # is not one.
+    try:
+        set_start_method("fork")
+    except RuntimeError:
+        pass
+    manager = Manager()
+    results = manager.list()
+    process = Process(target=generate_one,
+                      args=(games, dest, results, options_per_game, seed, spoiler))
+    started = time.monotonic()
+    process.start()
+    process.join(timeout)
+    timed_out = process.is_alive()
+    if timed_out:
+        process.terminate()
+        process.join(5)
+        if process.is_alive():
+            process.kill()
+            process.join()
+    elapsed = time.monotonic() - started
+    return list(results), timed_out, elapsed
+
+
+def classify(results, timed_out, elapsed, single_player=True, randomized=False):
+    if timed_out:
+        return "failed", f"timed out after {elapsed:.0f}s"
+    if not results:
+        return "failed", "generation process exited without a result"
+    result = results[0]
+    if isinstance(result, BaseException):
+        if rooted_in(result, FileNotFoundError):
+            return "needs_input", str(result)
+        if randomized and rooted_in_named(result, "OptionError"):
+            # The world rejected the option combination we rolled - not a
+            # crash and not a fill failure, but the world validating its
+            # own input. Archipelago rolls every option independently, so
+            # a world with interdependent options (Blender's min/max
+            # similarity percent, for example) can always be handed a
+            # combination it considers invalid. A player writing
+            # "random" for both hits exactly this, so it is worth
+            # reporting - but as its own thing, not as a defect.
+            #
+            # Only when the options were randomized: a world that rejects
+            # its OWN DEFAULTS is genuinely broken and stays "failed".
+            return "invalid_options", f"rejected the rolled options: {result}"
+        from Fill import FillError
+        if rooted_in(result, FillError):
+            # A fill failure is a failure in both modes. This used to be
+            # excused in single-player on the theory that some games'
+            # item/location balance "only works out in a real multiworld"
+            # - the multi-game test exists precisely to check that theory,
+            # and it does not hold (Dead Cells fails both ways), so the
+            # excuse is gone: a seed that cannot be filled is a seed that
+            # cannot be generated, whoever else is in it.
+            where = "a multi-game seed" if not single_player else "a single-player seed"
+            return "failed", f"fill failed in {where}: {result}"
+        if rooted_in_named(result, "InvalidYamlException"):
+            # Some worlds (e.g. CTJoT) require a yaml pre-generated by an
+            # external tool and reject our generic single-player yaml
+            # outright - not a defect, just incompatible with this
+            # testing methodology.
+            return "passed", f"requires an externally-generated yaml (not a defect): {result}"
+        return "failed", f"{type(result).__name__}: {result}"
+    return "passed", str(result)
+
+
+def resolve_core_game(name, world_types):
+    """Match a sheet name to a world Archipelago actually registers.
+
+    The sheet and the worlds disagree about wording often enough that an
+    exact comparison finds only 72 of the 81 core rows. The rest differ
+    by an accent ("Pokemon Emerald"), a hyphen ("Choo-Choo Charles"),
+    capitalisation ("EarthBound"), or a series prefix the sheet adds and
+    the world does not ("The Legend of Zelda: Ocarina of Time").
+
+    Tried in order: the name as written, then folded to letters and
+    digits, then either side of a colon folded the same way. Either side,
+    because the sheet puts the distinguishing part before the colon as
+    often as after it - "Super Mario Land 2: The Golden Coins" is
+    registered as "Super Mario Land 2", while "The Legend of Zelda:
+    Ocarina of Time" is registered as "Ocarina of Time".
+    """
+    if name in world_types:
+        return name
+    by_slug = {}
+    for registered in world_types:
+        by_slug.setdefault(slug(registered), registered)
+    candidates = [name]
+    if ":" in name:
+        head, tail = name.split(":", 1)
+        candidates += [tail, head]
+    for candidate in candidates:
+        registered = by_slug.get(slug(candidate))
+        if registered:
+            return registered
+    return None
+
+
+# Deliberate copy of Name.slug in core/model/name.py. This file is mounted
+# alone into the container, with no access to this checkout's
+# packages, so it cannot import it - but the two must stay identical
+# or a world folds to one key here and a different one on the host.
+def slug(text):
+    return re.sub(r"[^a-z0-9]", "", unicodedata.normalize("NFKD", (text or "").lower()))
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    # Exactly one of these. --apworld is the normal case: a downloaded
+    # file, identified by which world claims it as its zip_path. --game
+    # names a world that is already part of the Archipelago build, which
+    # has no file to point at and so cannot be identified that way.
+    parser.add_argument("--apworld")
+    parser.add_argument("--game", help="Test a world that ships with Archipelago, by name, "
+                         "instead of an apworld file. Mutually exclusive with --apworld.")
+    parser.add_argument("--output-dir", required=True)
+    parser.add_argument("--timeout", type=int, default=300)
+    parser.add_argument("--companion-pool", default="",
+                         help="Comma-separated core games to draw companions from. Empty "
+                              "means every core world, which risks drawing one that cannot "
+                              "generate here (see container/verify_companions.py).")
+    parser.add_argument("--companion-min", type=int, default=2,
+                         help="Fewest core games to draw as companions for a multi-game seed.")
+    parser.add_argument("--companion-max", type=int, default=5,
+                         help="Most core games to draw as companions for a multi-game seed.")
+    parser.add_argument("--random-repeats", type=int, default=3,
+                         help="How many times the randomized-option modes run. Each attempt "
+                              "rolls a different option combination.")
+    parser.add_argument("--repeats", type=int, default=10,
+                         help="How many times to run the single-game and multi-game tests. "
+                              "Stability is a pass rate, not a single verdict.")
+    parser.add_argument("--spoiler", type=int, default=3,
+                         help="Spoiler level passed to Generate. 3 computes the full "
+                              "playthrough, exercising accessibility logic.")
+    args = parser.parse_args()
+    if bool(args.apworld) == bool(args.game):
+        parser.error("pass exactly one of --apworld or --game")
+
+    # Determine the game name from the loaded World class itself rather
+    # than the apworld's archipelago.json manifest: older apworlds don't
+    # ship a manifest at all, and the core (<0.7.0 here) tolerates that
+    # and loads them anyway - a manifest-only lookup would misreport a
+    # perfectly loadable world as failed. World.zip_path is set only for
+    # worlds loaded from a .apworld zip (worlds/AutoWorld.py), and equals
+    # the apworld's own path, so it identifies our one mounted world
+    # exactly regardless of manifest presence.
+    # worlds/__init__.py logs each failed world's traceback and carries
+    # on; without a handler here that detail is lost and all the pipeline
+    # can say is "found 0 worlds".
+    load_log = io.StringIO()
+    log_handler = logging.StreamHandler(load_log)
+    log_handler.setLevel(logging.ERROR)
+    logging.getLogger().addHandler(log_handler)
+    try:
+        import worlds
+        from worlds.AutoWorld import AutoWorldRegister
+    finally:
+        logging.getLogger().removeHandler(log_handler)
+
+    ensure_tutorials_default()
+
+    if args.game:
+        # A core world is already loaded - there is no file to identify
+        # it by, and none of the load-failure reporting below applies. It
+        # either exists in this build or it does not, and "does not" is a
+        # naming problem on the sheet rather than a defect in a world.
+        resolved = resolve_core_game(args.game, AutoWorldRegister.world_types)
+        if resolved:
+            args.game = resolved
+        else:
+            # "unknown", not "unverified". Nothing about this world is
+            # broken - this Archipelago build simply has no world under
+            # that name, which is a naming mismatch on the sheet (or a
+            # world that has since been removed from core), and calling
+            # it Broken would blame a world for the sheet's wording.
+            print(json.dumps({
+                "game": None,
+                "outcome": "unknown",
+                "detail": (f"this Archipelago build registers no world named {args.game!r}, "
+                            f"so there was nothing to test. The sheet lists this row as Core, "
+                            f"so either its name differs from the world's own or the world is "
+                            f"no longer part of Archipelago."),
+                "elapsed_seconds": 0,
+            }))
+            return 1
+        custom_games = [args.game]
+    else:
+        custom_games = [
+            name for name, world_type in AutoWorldRegister.world_types.items()
+            if str(getattr(world_type, "zip_path", "") or "") == args.apworld
+        ]
+    if len(custom_games) != 1:
+        # Its own outcome, NOT "passed". This covers several distinct
+        # causes (a duplicate game name against a now-built-in world, an
+        # import-time crash, a file that isn't a valid apworld at all,
+        # a genuine collision between two unrelated apworlds) - none of
+        # them necessarily an apworld defect, but none of them a world
+        # this pipeline has verified either. Calling it "passed" promoted
+        # files that never loaded and crash-looped real WebHost startup
+        # Factorio Platformer did exactly that: not a valid zip, zero
+        # worlds loaded, recorded as a pass, promoted, and crash-looping
+        # real WebHost startup - it needed a hand-written blacklist entry
+        # to undo. promote.py takes "passed" alone, so this verdict now
+        # keeps such a file out of worlds_live/ on its own.
+        summary, traceback_text = describe_load_failure(load_log.getvalue(), args.apworld)
+        if len(custom_games) > 1:
+            detail = (f"this file registers {len(custom_games)} worlds, not one: "
+                      f"{', '.join(custom_games)}. Only a single-world apworld can be "
+                      f"promoted, since the pipeline cannot tell which one this row means.")
+        elif summary:
+            detail = f"the world failed to import: {summary}"
+            if traceback_text:
+                detail += f"\n\n{traceback_text.strip()}"
+        else:
+            # No captured traceback: the file is not a loadable apworld at
+            # all (not a zip, wrong layout), so nothing was even attempted.
+            detail = (f"no world loaded from this file and no import was attempted - it is "
+                      f"probably not a valid .apworld (failed_world_loads="
+                      f"{worlds.failed_world_loads})")
+        print(json.dumps({
+            "game": None,
+            "outcome": "unverified",
+            "detail": detail,
+            "elapsed_seconds": 0,
+        }))
+        return 1
+
+    game = custom_games[0]
+
+    Path(args.output_dir).mkdir(parents=True, exist_ok=True)
+
+    def run_mode(games, options_per_game=None, seed=None, single_player=None, randomized=False):
+        if single_player is None:
+            single_player = len(games) == 1
+        mode_results, timed_out, elapsed = run_with_timeout(
+            games, args.output_dir, args.timeout,
+            options_per_game=options_per_game, seed=seed, spoiler=args.spoiler)
+        outcome, detail = classify(mode_results, timed_out, elapsed,
+                                    single_player=single_player, randomized=randomized)
+        return {"outcome": outcome, "detail": detail, "elapsed_seconds": round(elapsed, 1),
+                "timed_out": timed_out}
+
+    # A timeout anywhere abandons the rest of this apworld. One world that
+    # hangs would otherwise cost timeout x repeats x modes - by far the
+    # largest thing in a run - and a world that cannot finish one seed has
+    # nothing useful to say about its stability across ten.
+    state = {"timed_out": False}
+
+    def skipped(reason):
+        return {"outcome": "unknown", "detail": reason, "elapsed_seconds": None}
+
+    def run_repeated(players, repeats, single_player=None, options=None, randomized=False):
+        """The same test several times, reported as a pass rate.
+
+        Stability is not something a single generation can tell you: S, D
+        and M each get a different Archipelago seed, so the fill differs
+        every time, and a world that places items successfully four times
+        in five is genuinely unstable. Running once records whichever
+        result we happened to get.
+
+        Each repeat uses an explicit seed so a failure among ten passes is
+        reproducible - without it the interesting case is the one that
+        cannot be re-examined.
+        """
+        attempts = []
+        for _ in range(max(repeats, 1)):
+            seed = random.randint(0, 2 ** 31 - 1)
+            # players is a callable so the multi-game test can draw a
+            # different set of companions every attempt - ten runs against
+            # one fixed pair only ever tests that pair.
+            games = players() if callable(players) else players
+            # A callable, so the randomized modes can roll a different
+            # point in the option space for every attempt - repeating one
+            # roll would only re-measure the same combination.
+            per_game = options() if callable(options) else options
+            result = run_mode(games, options_per_game=per_game, seed=seed,
+                               single_player=single_player, randomized=randomized)
+            result["seed"] = seed
+            if len(games) > 1:
+                result["companions"] = games[1:]
+            attempts.append(result)
+            if result["timed_out"]:
+                state["timed_out"] = True
+                break
+
+        passed = [a for a in attempts if a["outcome"] == "passed"]
+        failed = [a for a in attempts if a["outcome"] != "passed"]
+        if not failed:
+            outcome, detail = "passed", f"generated {len(passed)}/{len(attempts)} times"
+        elif not passed:
+            # Every attempt agreeing on a non-failure reason keeps that
+            # reason. A world that needs a base ROM reports needs_input
+            # once and "failed all 10 attempts" ten times over, and
+            # flattening those to "failed" blames the world for a file
+            # this pipeline does not have - which matters most for core
+            # worlds, where roughly seven of them need a ROM and would
+            # otherwise all read as Broken.
+            outcomes = {attempt["outcome"] for attempt in attempts}
+            outcome = outcomes.pop() if len(outcomes) == 1 else "failed"
+            detail = f"failed all {len(attempts)} attempts. {failed[0]['detail']}"
+            if outcome != "failed":
+                detail = (f"{outcome} on all {len(attempts)} attempts. "
+                           f"{failed[0]['detail']}")
+        else:
+            # The whole point of repeating: a world that works sometimes
+            # is worse for a player than one that never does, because
+            # nobody knows to avoid it.
+            outcome = "flaky"
+            detail = (f"generated {len(passed)}/{len(attempts)} times. "
+                      f"First failure (seed {failed[0]['seed']}): {failed[0]['detail']}")
+        return {
+            "outcome": outcome,
+            "detail": detail,
+            "elapsed_seconds": round(sum(a["elapsed_seconds"] or 0 for a in attempts), 1),
+            "attempts": len(attempts),
+            # What was asked for, as opposed to what ran - a timeout cuts
+            # a mode short, and the retest cache compares against this so
+            # such a record is not re-run forever.
+            "requested_attempts": max(repeats, 1),
+            "passed_attempts": len(passed),
+            "failed_seeds": [a["seed"] for a in failed],
+            # Which companions each failing attempt drew, so a failure
+            # caused by one particular combination can be reproduced.
+            "failed_companions": [a.get("companions") for a in failed if a.get("companions")],
+        }
+
+    tests = {}
+
+    # 1. the world on its own, every option at its default.
+    tests["single"] = run_repeated([game], args.repeats)
+    tests["single"]["label"] = "Single game"
+
+    # 2. the same world twice. A world that keeps state on the class
+    #    rather than the instance works alone and corrupts itself here -
+    #    a bug class no other mode can see.
+    tests["duplicate"] = (skipped("skipped - an earlier test timed out") if state["timed_out"]
+                           else run_mode([game, game], single_player=False))
+    tests["duplicate"]["label"] = "Two players, same game"
+
+    # 3. the world alongside other games. Each attempt draws a fresh
+    #    random selection of core worlds rather than reusing one fixed
+    #    pair: ten runs against the same two companions only ever tell you
+    #    about those two, while a different combination each time samples
+    #    what a real multiworld looks like.
+    configured_pool = [name.strip() for name in args.companion_pool.split(",") if name.strip()]
+    if configured_pool:
+        # Verified by the host once per run: every one of these generated
+        # a seed on its own in this exact environment, so a multi-game
+        # failure is attributable to the world under test rather than to
+        # a companion that was never going to work.
+        pool = [name for name in configured_pool if name != game]
+    else:
+        pool = core_worlds(exclude=game)
+    low = max(args.companion_min, 1)
+    high = max(args.companion_max, low)
+
+    def draw_companions():
+        count = min(random.randint(low, high), len(pool))
+        return random.sample(pool, count)
+
+    if state["timed_out"]:
+        tests["multi"] = skipped("skipped - an earlier test timed out")
+    elif pool:
+        tests["multi"] = run_repeated(
+            lambda: [game] + draw_companions(), args.repeats, single_player=False)
+        tests["multi"]["companion_pool"] = len(pool)
+        tests["multi"]["companion_range"] = [low, high]
+        tests["multi"]["detail"] += f" (against {low}-{high} core games drawn from {len(pool)})"
+    else:
+        tests["multi"] = skipped("no core worlds available to pair with")
+    tests["multi"]["label"] = "Multi game"
+
+    # 4 and 5. The same two shapes as above - the world alone, and the
+    #    world alongside the companions - but with its options randomized
+    #    instead of left at their defaults. The default option set is a
+    #    single point in a world's option space and most option-dependent
+    #    crashes are nowhere near it.
+    #
+    #    One roll each, and every roll that does not produce a seed is a
+    #    failure, including one the world rejects with OptionError. That
+    #    last point is the whole reason these modes exist: the yaml a
+    #    player writes is accepted as valid - nothing in the template or
+    #    the upload path objects to "random" on two interdependent
+    #    options - so the conflict only surfaces when generation is
+    #    actually attempted, possibly after everyone has submitted, and it
+    #    is intermittent. Re-rolling until the world accepts something
+    #    would hide exactly the failure a real player hits.
+    #
+    #    Each randomized run is gated on its own default-options
+    #    counterpart: SR only when the world generates alone, MR only when
+    #    it generates alongside the companions. A randomized run whose
+    #    plain equivalent already fails would fail for the same underlying
+    #    reason and cost a generation to learn nothing.
+    rolls = None
+    if state["timed_out"]:
+        aborted = "skipped - an earlier test timed out"
+        tests["single_random"] = skipped(aborted)
+        tests["multi_random"] = skipped(aborted)
+    elif tests["single"]["outcome"] == "passed" or tests["multi"]["outcome"] == "passed":
+        try:
+            # One roll per attempt, drawn up front so SR and MR run the
+            # SAME combinations as each other. Pairing them is what makes
+            # "SR passed but MR failed" attributable to the extra players
+            # rather than to two different option sets.
+            rolls = [random_options(game) for _ in range(max(args.random_repeats, 1))]
+        except BaseException as error:
+            rolls = None
+            unreadable = f"could not read this world's options: {type(error).__name__}: {error}"
+            tests["single_random"] = skipped(unreadable)
+            tests["multi_random"] = skipped(unreadable)
+
+    if rolls:
+        def roller():
+            # Walks the pre-drawn rolls in order, so attempt N of SR and
+            # attempt N of MR use the same option values.
+            supply = iter(rolls)
+            def next_roll():
+                return {game: next(supply)}
+            return next_roll
+
+        # "flaky" counts as generating: the world produced a seed at
+        # least once, so a randomized run can still tell us something.
+        if tests["single"]["outcome"] in ("passed", "flaky"):
+            result = run_repeated([game], len(rolls), single_player=True,
+                                   options=roller(), randomized=True)
+            result["randomized_options"] = len(rolls[0])
+            tests["single_random"] = result
+        else:
+            tests["single_random"] = skipped(
+                "skipped - the single-game test did not pass")
+
+        if tests["multi"]["outcome"] in ("passed", "flaky"):
+            result = run_repeated(lambda: [game] + draw_companions(), len(rolls),
+                                   single_player=False, options=roller(), randomized=True)
+            result["randomized_options"] = len(rolls[0])
+            tests["multi_random"] = result
+        else:
+            tests["multi_random"] = skipped(
+                "skipped - the multi-game test did not pass")
+    elif "single_random" not in tests:
+        neither = "skipped - neither the single-game nor the multi-game test passed"
+        tests["single_random"] = skipped(neither)
+        tests["multi_random"] = skipped(neither)
+
+    tests["single_random"]["label"] = "Random options, alone"
+    tests["multi_random"]["label"] = "Random options, multi game"
+
+    # The top-level fields stay exactly what they were - the single-player
+    # verdict - because that is what promotion and every existing consumer
+    # reads. The per-mode results live alongside it.
+    report = {
+        "game": game,
+        "outcome": tests["single"]["outcome"],
+        "detail": tests["single"]["detail"],
+        "elapsed_seconds": tests["single"]["elapsed_seconds"],
+        "multi_outcome": tests["multi"]["outcome"],
+        "multi_detail": tests["multi"]["detail"],
+        "multi_elapsed_seconds": tests["multi"]["elapsed_seconds"],
+        "multi_companions": tests["multi"].get("companions", []),
+        "tests": tests,
+    }
+
+    print(json.dumps(report))
+    # Exit code still reflects the single-player result alone: it is what
+    # gates promotion, and the other modes are reported rather than
+    # blocking.
+    return {"passed": 0, "needs_input": 2}.get(tests["single"]["outcome"], 1)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/apworld_tester/container/drivers/verify_companions.py

@@ -0,0 +1,65 @@
+"""Runs inside the archipelago Docker image. Reports which core worlds
+can actually generate a seed in THIS environment, so the multi-game tests
+only ever draw companions that work.
+
+Core worlds are shipped with Archipelago, but that does not make them all
+usable here: some need a base ROM this container has no copy of, and at
+least one (Final Fantasy) needs a settings yaml generated by an external
+website and refuses a generic one. Drawing such a world as a companion
+fails the whole seed, and the failure gets recorded against the apworld
+under test rather than the companion that caused it - Blender was marked
+flaky 7/10 purely because three of its ten draws included worlds like
+these.
+
+Rather than maintain an exclusion list by hand, this generates one solo
+seed per core world and keeps the ones that succeed. A world drops out
+by itself when its ROM is missing, and comes back by itself once the ROM
+is added.
+
+Shares run_test.py's generation machinery rather than repeating it, so
+"can generate" means exactly the same thing in both places.
+"""
+
+import argparse
+import json
+import sys
+import warnings
+
+sys.path.insert(0, "/app")
+
+import run_test  # noqa: E402 - mounted alongside this script
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output-dir", required=True)
+    parser.add_argument("--timeout", type=int, default=120,
+                         help="Per-world timeout. Deliberately shorter than the apworld "
+                              "timeout: a companion that is slow to generate would multiply "
+                              "across every multi-game attempt that draws it.")
+    parser.add_argument("--spoiler", type=int, default=2)
+    args = parser.parse_args()
+
+    warnings.simplefilter("ignore")
+    run_test.ensure_tutorials_default()
+
+    from pathlib import Path
+    Path(args.output_dir).mkdir(parents=True, exist_ok=True)
+
+    verified = []
+    rejected = {}
+    for game in run_test.core_worlds(exclude=None):
+        results, timed_out, elapsed = run_test.run_with_timeout(
+            [game], args.output_dir, args.timeout, spoiler=args.spoiler)
+        outcome, detail = run_test.classify(results, timed_out, elapsed)
+        if outcome == "passed":
+            verified.append(game)
+        else:
+            rejected[game] = f"{outcome}: {detail}"[:300]
+
+    print(json.dumps({"verified": verified, "rejected": rejected}))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/apworld_tester/core/config/config.py

@@ -0,0 +1,131 @@
+"""The project's configuration."""
+
+import os
+
+import yaml
+
+
+class ConfigNotFound(FileNotFoundError):
+    """No config.yaml anywhere on the search path.
+
+    Carries the places that were tried, because the fix is always to
+    put a file in one of them and a bare "no such file" names only the
+    last candidate.
+    """
+
+    def __init__(self, candidates):
+        self.candidates = list(candidates)
+        listed = "\n".join(f"  {path}" for path in self.candidates)
+        super().__init__(
+            "no config.yaml found. Looked in:\n"
+            f"{listed}\n"
+            "Run 'apworld-tester init' to write a starter config, "
+            "or pass --config / set APWORLD_TESTER_CONFIG."
+        )
+
+
+class ConfigNotFilledIn(ValueError):
+    """A config that was written by `init` and never edited.
+
+    Its own error rather than the FileNotFoundError the placeholder
+    path eventually causes: that one names a directory nobody chose and
+    reads like a bug in the tool.
+    """
+
+    def __init__(self, path, setting, value):
+        self.path = path
+        self.setting = setting
+        super().__init__(
+            f"{setting} is still the example value {value!r}.\n"
+            f"Edit {path} and set it to a directory on this machine."
+        )
+
+
+class Config:
+    """config.yaml, read once and passed to whatever needs it."""
+
+    #: Overrides every searched location when set.
+    ENV_VAR = "APWORLD_TESTER_CONFIG"
+
+    #: How the shipped template writes a path the user must replace.
+    PLACEHOLDER = "/path/to/your"
+
+    #: Settings that must be real before anything can run.
+    REQUIRED_PATHS = ("output_directory",)
+
+    #: The directory name used under $XDG_CONFIG_HOME and ~/.config.
+    APP_DIR = "apworld-tester"
+
+    def __init__(self, values, path=None):
+        self.values = values
+        self.path = path
+
+    @classmethod
+    def load(cls, path=None):
+        """Read config.yaml from the first place it exists."""
+        path = path or cls.default_path()
+        with open(path, "r", encoding="utf-8") as handle:
+            config = cls(yaml.safe_load(handle) or {}, path)
+        config.check_filled_in()
+        return config
+
+    def check_filled_in(self):
+        """Refuse a config still carrying the template's paths."""
+        for setting in self.REQUIRED_PATHS:
+            value = self.value("general", setting)
+            if value and str(value).startswith(self.PLACEHOLDER):
+                raise ConfigNotFilledIn(self.path, setting, value)
+
+    @staticmethod
+    def user_path():
+        """Where a user's own config.yaml belongs.
+
+        $XDG_CONFIG_HOME when the platform sets it, ~/.config
+        otherwise - the location `init` writes to and the one the
+        documentation names.
+        """
+        base = os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser(
+            "~/.config")
+        return os.path.join(base, Config.APP_DIR, "config.yaml")
+
+    @staticmethod
+    def checkout_path():
+        """config.yaml beside a source checkout, above the package.
+
+        Kept in the search order so running from a clone kept working
+        after installed copies moved to the user config directory. An
+        installed wheel resolves this to somewhere inside site-packages'
+        parent, which simply will not exist.
+        """
+        here = os.path.dirname(os.path.abspath(__file__))
+        root = os.path.dirname(os.path.dirname(os.path.dirname(here)))
+        return os.path.join(os.path.dirname(root), "config.yaml")
+
+    @classmethod
+    def search_paths(cls):
+        """Every location tried, in order."""
+        override = os.environ.get(cls.ENV_VAR)
+        paths = [override] if override else []
+        return paths + [cls.user_path(), cls.checkout_path()]
+
+    @classmethod
+    def default_path(cls):
+        """The first config.yaml that exists, or raise saying where."""
+        candidates = cls.search_paths()
+        for path in candidates:
+            if os.path.isfile(path):
+                return path
+        raise ConfigNotFound(candidates)
+
+    def get(self, key, default=None):
+        """One top-level value."""
+        value = self.values.get(key)
+        return default if value is None else value
+
+    def section(self, name):
+        return self.values.get(name) or {}
+
+    def value(self, section, key, default=None):
+        """One value from a section."""
+        value = self.section(section).get(key)
+        return default if value is None else value

src/apworld_tester/core/config/paths.py

@@ -0,0 +1,125 @@
+"""Every path the pipeline uses."""
+
+import os
+
+from importlib import resources
+
+
+class Paths:
+    """Where everything lives, derived from one Config.
+
+    The checkout holds code and configuration; the data root holds
+    everything a run generates, and defaults outside the checkout
+    because none of that is source. Every path is absolute.
+    """
+
+    def __init__(self, config):
+        self.config = config
+
+    @property
+    def project(self):
+        return os.path.dirname(self.config.path)
+
+    @property
+    def data(self):
+        """Where everything a run generates lives."""
+        configured = self.config.value(
+            "general",
+            "output_directory",
+            "~/.local/share/archipelago-world-tester",
+        )
+        return os.path.abspath(os.path.expanduser(configured))
+
+    @property
+    def state(self):
+        return os.path.join(self.data, "state.json")
+
+    @property
+    def scratch_state(self):
+        return os.path.join(self.data, "scratch_state.json")
+
+    @property
+    def lock(self):
+        return os.path.join(self.data, "pipeline.lock")
+
+    @property
+    def downloads(self):
+        return os.path.join(self.data, "downloads")
+
+    @property
+    def output(self):
+        return os.path.join(self.data, "test_output")
+
+    @property
+    def history(self):
+        """Every apworld version this pipeline has seen, per game.
+
+        Beside the downloads rather than inside them: that directory is
+        a cache and gets cleared to force a clean re-download, and this
+        is the one thing that cannot be rebuilt afterwards.
+        """
+        return os.path.join(self.data, "history")
+
+    @property
+    def checkout(self):
+        return os.path.join(self.data, "archipelago_src")
+
+    @property
+    def core_worlds(self):
+        return os.path.join(self.checkout, "worlds")
+
+    @property
+    def common_client(self):
+        return os.path.join(self.checkout, "CommonClient.py")
+
+    @property
+    def failed_report(self):
+        return os.path.join(self.data, "failed_apworlds.txt")
+
+    @property
+    def sheet_html(self):
+        return os.path.join(self.data, "playable_worlds.html")
+
+    @property
+    def roms(self):
+        """Base ROMs for the worlds that need one."""
+        configured = self.config.value("general", "roms_directory")
+        if not configured:
+            return os.path.join(self.data, "roms")
+        return os.path.abspath(os.path.expanduser(configured))
+
+    @property
+    def env(self):
+        return os.path.join(self.project, ".env")
+
+    @property
+    def container(self):
+        """The Dockerfile and drivers, resolved inside the package.
+
+        Shipped as package data rather than found beside config.yaml:
+        an installed copy has no checkout to look next to, and the
+        config now lives in the user's config directory, which is not
+        where a Dockerfile belongs. Returned as a real filesystem path
+        because every use is a docker build context or a bind mount.
+        """
+        return os.fspath(resources.files("apworld_tester") / "container")
+
+    @property
+    def drivers(self):
+        return os.path.join(self.container, "drivers")
+
+    @property
+    def test_dockerfile(self):
+        return os.path.join(self.container, "Dockerfile.test")
+
+    @property
+    def driver(self):
+        return os.path.join(self.drivers, "run_test.py")
+
+    @property
+    def identify_driver(self):
+        return os.path.join(self.drivers, "identify_apworld.py")
+
+    @property
+    def verify_driver(self):
+        return os.path.join(self.drivers, "verify_companions.py")

src/apworld_tester/core/config/run_settings.py

@@ -0,0 +1,86 @@
+"""How much testing one run asks for.
+
+Companions are drawn at random from the worlds Archipelago ships. Core
+worlds are the safe choice: part of the build itself, so always present
+and always matching the running version, unlike an apworld that might
+be missing or broken for reasons of its own. A fresh draw per attempt
+matters - ten runs against one fixed pair only ever test that pair.
+"""
+
+
+class RunSettings:
+    """The testing section, as the numbers a run is built from."""
+
+    #: Archipelago's spoiler levels, by what each one does. The numbers
+    #: are Generate.py's `--spoiler`, not a choice this project makes.
+    SPOILER_LEVELS = {
+        "none": 0,
+        "log": 1,
+        "playthrough": 2,
+        "paths": 3,
+    }
+
+    def __init__(self, config):
+        self.config = config
+
+    def number(self, key, default):
+        return int(self.config.value("testing", key, default))
+
+    @property
+    def companion_range(self):
+        """How many core worlds join the world under test.
+
+        Redrawn every attempt, so ten repeats sample ten combinations.
+        Clamped so a mistyped config cannot ask for an empty or
+        backwards range.
+        """
+        low = max(self.number("companion_min", 2), 1)
+        return low, max(self.number("companion_max", 5), low)
+
+    @property
+    def repeats(self):
+        """How often the single and multi tests are repeated.
+
+        Stability is a pass rate across runs, not one verdict: each
+        repeat gets a different seed, so the fill differs every time.
+        """
+        return self.number("repeats", 10)
+
+    @property
+    def random_repeats(self):
+        """How often the two randomized modes are repeated.
+
+        Lower on purpose: single and multi re-roll only the seed, so
+        ten runs measure one thing precisely, while each randomized run
+        rolls a different point in the option space and three samples
+        say much more than one did.
+        """
+        return self.number("random_repeats", 3)
+
+    @property
+    def spoiler(self):
+        """How much of the spoiler log each generation computes.
+
+        "playthrough" is the level worth paying for: it exercises the
+        accessibility and reachability logic that plain generation
+        skips, so a world whose logic is wrong fails here rather than
+        in a player's seed. "paths" adds entrance paths on top and
+        "log" writes the file without checking anything.
+        """
+        name = str(self.config.value("testing", "spoiler", "playthrough"))
+        try:
+            return self.SPOILER_LEVELS[name.strip().casefold()]
+        except KeyError:
+            known = ", ".join(self.SPOILER_LEVELS)
+            raise ValueError(
+                f"testing.spoiler is {name!r}; expected one of {known}"
+            ) from None
+
+    @property
+    def jobs(self):
+        """How many apworlds are tested at once.
+
+        Each test is one isolated `docker run` and subprocess.run
+        releases the GIL while it waits, so threads are all this needs.
+        """
+        return int(self.config.value("general", "threads", 4))

src/apworld_tester/core/config/secrets.py

@@ -0,0 +1,41 @@
+"""Credentials, kept out of config.yaml."""
+
+import os
+
+
+class Secrets:
+    """The .env file at the checkout's root.
+
+    Kept out of config.yaml: that file gets opened, edited and shared,
+    and a token in it leaks the first time it is.
+    """
+
+    #: The environment variable a GitHub token is read from.
+    GITHUB_TOKEN = "GITHUB_TOKEN"
+
+    def __init__(self, paths):
+        self.paths = paths
+
+    def values(self):
+        """NAME -> value for every line in the file."""
+        if not os.path.exists(self.paths.env):
+            return {}
+        values = {}
+        with open(self.paths.env, "r", encoding="utf-8") as handle:
+            for line in handle:
+                line = line.strip()
+                if not line or line.startswith("#") or "=" not in line:
+                    continue
+                name, value = line.split("=", 1)
+                values[name.strip()] = value.strip().strip("\"'")
+        return values
+
+    def get(self, name, default=None):
+        """One secret, or `default` when it is unset or empty."""
+        value = self.values().get(name)
+        return default if value in (None, "") else value
+
+    @property
+    def github_token(self):
+        """The GitHub token, or None."""
+        return self.get(self.GITHUB_TOKEN)

src/apworld_tester/core/display/progress.py

@@ -0,0 +1,108 @@
+"""Progress reporting for the long stages."""
+
+import shutil
+import sys
+import time
+
+
+class Progress:
+    """One stage's progress, tallied by outcome as it goes.
+
+    Both the download stage and the generation tests walk ~650 games, so
+    per-line output scrolls past uselessly when a run is watched. Under
+    cron the opposite is true - stdout is a log file, where a bar
+    redrawn with carriage returns is noise - so each caller supplies the
+    line it would have printed, and that is what a non-terminal gets.
+    """
+
+    def __init__(self, total, config=None, labels=None, stream=None):
+        self.total = total
+        self.config = config
+        self.labels = dict(labels or self.configured_labels())
+        self.stream = stream or sys.stdout
+        self.counts = {}
+        self.started = None
+
+    def configured_labels(self):
+        """Short labels for each outcome."""
+        if self.config is None:
+            return {}
+        return self.config.value("display", "outcome_labels", {})
+
+    @property
+    def width(self):
+        """How many characters wide the bar is drawn."""
+        if self.config is None:
+            return 28
+        return int(self.config.value("display", "progress_bar_width", 28))
+
+    @staticmethod
+    def duration(seconds):
+        """Seconds as m:ss, or h:mm:ss past an hour."""
+        seconds = int(seconds)
+        hours, rest = divmod(seconds, 3600)
+        minutes, seconds = divmod(rest, 60)
+        if hours:
+            return f"{hours}:{minutes:02d}:{seconds:02d}"
+        return f"{minutes:02d}:{seconds:02d}"
+
+    @property
+    def tally(self):
+        """The running counts, labelled keys first."""
+        ordered = [key for key in self.labels if self.counts.get(key)]
+        ordered += [key for key in self.counts if key not in self.labels]
+        return " ".join(
+            f"{self.labels.get(key, key)} {self.counts[key]}"
+            for key in ordered
+        )
+
+    def bar(self, fraction):
+        """The bar itself, filled to `fraction`."""
+        filled = int(self.width * fraction)
+        return "#" * filled + "-" * (self.width - filled)
+
+    def eta(self, index, elapsed):
+        """Time remaining at the current rate, blank once finished."""
+        if not index or index >= self.total:
+            return ""
+        return f" eta {self.duration(elapsed / index * (self.total - index))}"
+
+    def line(self, index, text):
+        """The whole status line for this update.
+
+        Position and time only. The running tally and the item's name
+        belong to the summary printed when the stage ends - on the bar
+        they change every redraw and are unreadable while doing it.
+        """
+        fraction = index / self.total if self.total else 1
+        elapsed = time.monotonic() - self.started
+        return (f"[{self.bar(fraction)}] {index}/{self.total} "
+                f"{fraction * 100:3.0f}%  "
+                f"{self.duration(elapsed)}{self.eta(index, elapsed)}")
+
+    def draw(self, index, text):
+        """Redraw the bar in place, ending the line when finished.
+
+        Padded to overwrite a previously longer line, then trimmed to
+        the terminal: a wrapped line cannot be overwritten by a carriage
+        return, and the bar starts spilling down the screen.
+        """
+        columns = shutil.get_terminal_size((100, 24)).columns
+        padded = self.line(index, text)[:columns - 1].ljust(columns - 1)
+        self.stream.write("\r" + padded)
+        self.stream.flush()
+        if index >= self.total:
+            self.stream.write("\n")
+            self.stream.flush()
+
+    def update(self, index, key=None, text="", fallback_line=None):
+        """Record one finished item and show where the stage is."""
+        if self.started is None:
+            self.started = time.monotonic()
+        if key is not None:
+            self.counts[key] = self.counts.get(key, 0) + 1
+        if not self.stream.isatty():
+            if fallback_line is not None:
+                print(fallback_line, file=self.stream)
+            return
+        self.draw(index, text)

src/apworld_tester/core/model/apworld.py

@@ -0,0 +1,120 @@
+"""The filename an apworld needs before Archipelago will import it."""
+
+import json
+import os
+import re
+import zipfile
+
+
+class ApworldFile:
+    """One downloaded apworld, and what its name has to be.
+
+    Archipelago derives the module name from the FILENAME, so a version
+    in the name reads as package separators and a backslash is just a
+    character. The right name is the zip's own package folder.
+    """
+
+    #: The extension an apworld is recognised by.
+    SUFFIX = ".apworld"
+
+    def __init__(self, path):
+        self.path = path
+
+    @property
+    def entries(self):
+        """Every entry name in the zip, empty if it will not open."""
+        try:
+            with zipfile.ZipFile(self.path) as archive:
+                return archive.namelist()
+        except (OSError, zipfile.BadZipFile):
+            return []
+
+    @property
+    def uses_backslash_separators(self):
+        """Whether the zip stores its paths with backslashes.
+
+        Invisible on Windows and fatal on Linux: zipimport swaps "/" for
+        the platform separator when reading, so on Windows both forms
+        match and on Linux the backslash entry never matches the
+        "pkg/__init__.py" it looks for. The author sees a working
+        apworld and every Linux user sees ModuleNotFoundError.
+        """
+        names = self.entries
+        if not names:
+            return False
+        return (not any("/" in name for name in names)
+                and any("\\" in name for name in names))
+
+    @property
+    def manifest(self):
+        """The apworld's own archipelago.json, or None.
+
+        Its self-declared "game" and "world_version", sitting in the
+        file's content regardless of which release was resolved for it -
+        or whether one ever was. Older apworlds predate the convention
+        and ship none, so this is best-effort display information rather
+        than something worth failing over.
+        """
+        try:
+            with zipfile.ZipFile(self.path) as archive:
+                name = next((entry for entry in archive.namelist()
+                             if entry.endswith("/archipelago.json")), None)
+                if name is None:
+                    return None
+                with archive.open(name) as handle:
+                    return json.load(handle)
+        except (OSError, zipfile.BadZipFile, json.JSONDecodeError, KeyError):
+            return None
+
+    @property
+    def package_folder(self):
+        """The single top-level package inside the zip, or None."""
+        names = [name.replace("\\", "/") for name in self.entries]
+        packages = {
+            name.split("/", 1)[0]
+            for name in names
+            if name.count("/") == 1 and name.endswith("/__init__.py")
+        }
+        return packages.pop() if len(packages) == 1 else None
+
+    @property
+    def importable_name(self):
+        """What this file must be called for Archipelago to import it."""
+        base = os.path.basename(self.path.replace("\\", "/"))
+        if base.lower().endswith(ApworldFile.SUFFIX):
+            stem = base[:-len(ApworldFile.SUFFIX)]
+        else:
+            stem = base
+        stem = self.package_folder or self.sanitize(stem)
+        return stem + ApworldFile.SUFFIX
+
+    def sanitize(self, stem):
+        """A filename stem reduced to something importable."""
+        base = os.path.basename(stem.replace("\\", "/"))
+        cleaned = re.sub(r"[^0-9A-Za-z_]", "_", base).strip("_")
+        if cleaned and cleaned[0].isdigit():
+            cleaned = "_" + cleaned
+        return cleaned or "apworld"
+
+    def prepare(self, working_directory):
+        """The path to hand Archipelago, and whether it was repaired."""
+        if not self.uses_backslash_separators:
+            return self.path, False
+        os.makedirs(working_directory, exist_ok=True)
+        destination = os.path.join(working_directory, self.importable_name)
+        return self.write_normalized(destination), True
+
+    def write_normalized(self, destination):
+        """Copy the apworld with its separators repaired."""
+        with zipfile.ZipFile(self.path) as source:
+            with zipfile.ZipFile(destination, "w",
+                                 zipfile.ZIP_DEFLATED) as target:
+                for item in source.infolist():
+                    repaired = zipfile.ZipInfo(
+                        filename=item.filename.replace("\\", "/"),
+                        date_time=item.date_time,
+                    )
+                    repaired.compress_type = item.compress_type
+                    repaired.external_attr = item.external_attr
+                    target.writestr(repaired, source.read(item.filename))
+        return destination

src/apworld_tester/core/model/blacklist.py

@@ -0,0 +1,30 @@
+"""Worlds this pipeline declines to test."""
+
+from apworld_tester.core.model.name import Name
+
+
+class Blacklist:
+    """Worlds not tested.
+
+    A world that times out is the most expensive thing in a run and the
+    least informative. Entries are permanent until removed by hand -
+    "do not retry" is the whole reason one is listed.
+    """
+
+    #: Recorded verdict for a world that was never tested.
+    OUTCOME = "unknown"
+
+    #: What the status page shows in place of a test result.
+    DETAIL = "not tested: blacklisted"
+
+    def __init__(self, config):
+        self.config = config
+
+    @property
+    def entries(self):
+        """Every listed world, by folded name."""
+        listed = self.config.get("blacklist", []) or []
+        return {Name(name).slug for name in listed if Name(name).slug}
+
+    def listed(self, game_name):
+        return Name(game_name).slug in self.entries

src/apworld_tester/core/model/fingerprint.py

@@ -0,0 +1,30 @@
+"""Content hashes, used to decide what has changed."""
+
+import hashlib
+import os
+
+
+class Fingerprint:
+    """The content of a file or a directory, as one hash."""
+
+    @staticmethod
+    def of_file(path):
+        """One file's sha256, read in chunks."""
+        digest = hashlib.sha256()
+        with open(path, "rb") as handle:
+            for chunk in iter(lambda: handle.read(65536), b""):
+                digest.update(chunk)
+        return digest.hexdigest()
+
+    @classmethod
+    def of_directory(cls, directory):
+        """One hash covering every file directly inside a directory."""
+        digest = hashlib.sha256()
+        if not os.path.isdir(directory):
+            return digest.hexdigest()
+        for name in sorted(os.listdir(directory)):
+            path = os.path.join(directory, name)
+            if os.path.isfile(path):
+                digest.update(name.encode("utf-8"))
+                digest.update(cls.of_file(path).encode("utf-8"))
+        return digest.hexdigest()

src/apworld_tester/core/model/game.py

@@ -0,0 +1,47 @@
+"""One game from the worlds sheet."""
+
+import re
+
+
+class Game:
+    """A sheet row: a name, its links, and what the sheet says about it."""
+
+    def __init__(self, name, links=None, release=None, stability=None,
+                 pr_status=None):
+        self.name = name
+        self.links = list(links or [])
+        self.release = release
+        self.stability = stability
+        self.pr_status = pr_status
+        for link in self.links:
+            self.remember_release(link)
+
+    @staticmethod
+    def is_release_link(link):
+        """Whether a link points at a GitHub releases page."""
+        return bool(re.match(
+            r"^https?://(?:www\.)?github\.com/[^/]+/[^/]+/releases"
+            r"(?:[/?#]|$)",
+            link or "",
+            re.IGNORECASE,
+        ))
+
+    @property
+    def is_core(self):
+        return (self.pr_status or "").strip().lower() == "core"
+
+    def remember_release(self, link):
+        """Keep this link as the release URL if none was found yet."""
+        if self.release is None and self.is_release_link(link):
+            self.release = link
+        return self.release
+
+    def add_link(self, link):
+        """Add a link, ignoring one already recorded."""
+        if link not in self.links:
+            self.links.append(link)
+        self.remember_release(link)
+        return self.links
+
+    def __str__(self):
+        return f"{self.name} ({len(self.links)} links)"

src/apworld_tester/core/model/instant.py

@@ -0,0 +1,35 @@
+"""One timestamp, as the moment it names."""
+
+import datetime
+
+
+class Instant:
+    """An ISO 8601 timestamp parsed into a comparable moment.
+
+    Hosts do not agree on how to write the offset: GitHub sends a
+    trailing "Z", this pipeline's own clock writes "+00:00", and GitLab
+    and Gitea send a local one ("...19:55:36-04:00" is really 23:55Z).
+    Compared as text those sort by the numbers on the clock face rather
+    than by when they happened, so anything comparing two timestamps
+    from different sources has to parse them first.
+    """
+
+    @staticmethod
+    def of(text):
+        """One timestamp, or None when it is not a usable one.
+
+        A naive timestamp is read as UTC: everything written here is,
+        and the alternative is a comparison that raises rather than one
+        that is merely approximate. "Z" is spelled out because
+        fromisoformat only accepts it from 3.11.
+        """
+        if not text:
+            return None
+        try:
+            parsed = datetime.datetime.fromisoformat(
+                str(text).replace("Z", "+00:00"))
+        except (TypeError, ValueError):
+            return None
+        if parsed.tzinfo is None:
+            return parsed.replace(tzinfo=datetime.timezone.utc)
+        return parsed

src/apworld_tester/core/model/modes.py

@@ -0,0 +1,22 @@
+"""The generation modes a complete result carries."""
+
+
+class Modes:
+    """What the driver runs against every world.
+
+    Not configuration: each name is a branch in the container driver
+    and in Verdict, so a mode cannot be added or removed by editing a
+    list. Changing this would only desynchronise the cache check and
+    the results table from what actually ran.
+    """
+
+    SINGLE = "single"
+    DUPLICATE = "duplicate"
+    MULTI = "multi"
+    SINGLE_RANDOM = "single_random"
+    MULTI_RANDOM = "multi_random"
+
+    ALL = (SINGLE, DUPLICATE, MULTI, SINGLE_RANDOM, MULTI_RANDOM)
+
+    #: The modes that cannot demote a world on their own.
+    SECONDARY = (DUPLICATE, SINGLE_RANDOM, MULTI_RANDOM)

src/apworld_tester/core/model/name.py

@@ -0,0 +1,42 @@
+"""Folding a game name down to a comparison key."""
+
+import re
+import unicodedata
+
+
+class Name:
+    """One game name, and the two ways it gets folded.
+
+    One implementation, because five copies once disagreed and a world
+    silently lost its blacklist entry.
+    """
+
+    def __init__(self, text):
+        self.text = text or ""
+
+    @property
+    def slug(self):
+        """The key for matching this name against another."""
+        folded = unicodedata.normalize("NFKD", self.text.lower())
+        return re.compile(r"[^a-z0-9]").sub("", folded)
+
+    @property
+    def report_slug(self):
+        return re.compile(r"[^a-z0-9]+").sub("-", self.text.lower()).strip("-")
+
+    @property
+    def directory(self):
+        return re.sub(r'[<>:"/\\|?*]', "_", self.text).strip().rstrip(".")
+
+    def matches(self, other):
+        """Whether an internally registered name means this game.
+
+        One-directional: the internal name is usually the shorter,
+        canonical one and the sheet's is often longer, so checking
+        containment the other way would let a short internal name match
+        an unrelated row that happens to start the same way.
+        """
+        if not self.text or not other:
+            return False
+        internal = Name(other).slug
+        return internal == self.slug or internal in self.slug

src/apworld_tester/core/model/verdict.py

@@ -0,0 +1,71 @@
+"""Where a world's test record lands on the stability ladder."""
+
+from apworld_tester.core.model.modes import Modes
+
+
+class Verdict:
+    """One world's tested stability.
+
+    The ladder: broken, unknown, flaky, solo_only, unstable, stable.
+    Single and multi decide the first four, because those are what a
+    player does; the other modes can only stop a world being stable.
+    """
+
+    #: Outcomes that count as a failure in a secondary mode.
+    FAILURES = ("failed", "invalid_options", "flaky")
+
+    def __init__(self, record, config):
+        self.record = record or {}
+        self.config = config
+
+    @property
+    def modes(self):
+        return self.record.get("tests") or {}
+
+    @property
+    def never_imported(self):
+        return self.record.get("outcome") == "unverified"
+
+    def outcome_of(self, mode):
+        return (self.modes.get(mode) or {}).get("outcome")
+
+    @property
+    def primary(self):
+        """What the single and multi runs decide, or None if they pass."""
+        single = self.outcome_of(Modes.SINGLE)
+        multi = self.outcome_of(Modes.MULTI)
+        if single == "needs_input" or multi == "needs_input":
+            return "unknown"
+        if single == "failed":
+            return "broken"
+        if single == "flaky" or multi == "flaky":
+            return "flaky"
+        if multi == "failed":
+            return "solo_only"
+        if single != "passed":
+            return "unknown"
+        return None
+
+    @property
+    def secondary_failed(self):
+        """Whether a mode that cannot demote alone still failed."""
+        return any(self.outcome_of(mode) in self.FAILURES
+                   for mode in Modes.SECONDARY)
+
+    @property
+    def name(self):
+        """Where this record lands on the ladder."""
+        if self.never_imported:
+            return "broken"
+        if not self.modes or not self.outcome_of(Modes.SINGLE):
+            return "unknown"
+        decided = self.primary
+        if decided is not None:
+            return decided
+        return "unstable" if self.secondary_failed else "stable"
+
+    @property
+    def label(self):
+        labels = self.config.value("stability", "labels", {})
+        return labels.get(self.name, "Unknown")
+

src/apworld_tester/core/state/keys.py

@@ -0,0 +1,23 @@
+"""The keys this pipeline writes into state.json."""
+
+
+class StateKeys:
+    """Names of the entries in the state file.
+
+    Not configuration: state.json is written and read only by this
+    pipeline, so renaming a key here would not migrate the file, it
+    would orphan every record already in it.
+    """
+
+    APWORLD_TESTS = "apworld_tests"
+    UPGRADE_SNAPSHOT = "apworld_tests_at_last_upgrade"
+    ARCHIPELAGO_TAG = "archipelago_tag"
+    ARCHIPELAGO_TAG_PUBLISHED_AT = "archipelago_tag_published_at"
+    PREVIOUS_ARCHIPELAGO_TAG = "previous_archipelago_tag"
+    COMPANION_POOL = "verified_companions"
+    GAME_RELEASES = "game_releases"
+    DISCONTINUED = "discontinued"
+    LAST_CHECKED_AT = "last_checked_at"
+    FAILED_RELEASE = "failed_release"
+    FAILED_REASON = "failed_reason"
+    STABILITY_HISTORY = "stability_history"

src/apworld_tester/core/state/lock.py

@@ -0,0 +1,37 @@
+"""The exclusive lock a whole run holds."""
+
+import fcntl
+
+
+class LockHeldError(RuntimeError):
+    """Another run already holds the lock."""
+
+
+class PipelineLock:
+    """Stops two runs, or a run and a promotion, from overlapping.
+
+    Non-blocking: a second run fails fast rather than queuing. Queuing
+    would be worse - a full pass takes hours, so a queued one would
+    start against an Archipelago version that had moved on.
+    """
+
+    def __init__(self, path):
+        self.path = path
+        self.handle = None
+
+    def __enter__(self):
+        self.handle = open(self.path, "a+")
+        try:
+            fcntl.flock(self.handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError as error:
+            self.handle.close()
+            raise LockHeldError(
+                f"Another run already holds {self.path} - refusing to "
+                "start."
+            ) from error
+        return self
+
+    def __exit__(self, *unused):
+        fcntl.flock(self.handle, fcntl.LOCK_UN)
+        self.handle.close()
+        return False

src/apworld_tester/core/state/scratch.py

@@ -0,0 +1,32 @@
+"""Choosing between the live record and a scratch copy of it."""
+
+import os
+import shutil
+
+
+class ScratchState:
+    """Where a run's results go, live or not.
+
+    A run is an experiment unless it says otherwise: promotion and the
+    status page read the live file, so a hand-run test that wrote there
+    would put a world nobody can download into the deployable record.
+
+    Seeding the scratch copy from the live one is what makes that cheap
+    rather than punishing - without it a scratch run has no recorded
+    Archipelago version to test against and no verified companion pool,
+    so it would stop for the first and re-check every core world for
+    the second.
+    """
+
+    def __init__(self, paths, live=False):
+        self.paths = paths
+        self.live = live
+
+    @property
+    def path(self):
+        if self.live:
+            return self.paths.state
+        scratch = self.paths.scratch_state
+        if not os.path.exists(scratch) and os.path.exists(self.paths.state):
+            shutil.copy(self.paths.state, scratch)
+        return scratch

src/apworld_tester/core/state/store.py

@@ -0,0 +1,55 @@
+"""Reading and writing the pipeline's JSON state."""
+
+import json
+import os
+import tempfile
+
+
+class StateStore:
+    """What every world last did, and the companion pool behind it."""
+
+    def __init__(self, path):
+        self.path = path
+
+    def load(self):
+        return self.read(self.path, {})
+
+    def save(self, data):
+        self.write(self.path, data)
+
+    @staticmethod
+    def read(path, default=None):
+        """A JSON file's contents, or `default` when it is absent."""
+        if not os.path.exists(path):
+            return {} if default is None else default
+        with open(path, "r", encoding="utf-8") as handle:
+            return json.load(handle)
+
+    @staticmethod
+    def write(path, data):
+        """Write JSON so a reader never sees a partial file.
+
+        A kill mid-write used to leave state.json truncated mid-object -
+        valid up to the cut point and invalid after it - silently
+        costing every recorded result on the next load. The batch saves
+        after every single test, so that window was hit often.
+
+        os.replace is atomic: the file is always either the old complete
+        version or the new one.
+        """
+        directory = os.path.dirname(os.path.abspath(path)) or "."
+        handle_id, temporary = tempfile.mkstemp(
+            dir=directory, prefix=".state_", suffix=".tmp")
+        try:
+            with os.fdopen(handle_id, "w", encoding="utf-8") as handle:
+                json.dump(data, handle, indent=2, sort_keys=True)
+                handle.write("\n")
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(temporary, path)
+        except BaseException:
+            try:
+                os.remove(temporary)
+            except OSError:
+                pass
+            raise

src/apworld_tester/pipeline/batch/apworld_finder.py

@@ -0,0 +1,67 @@
+"""Finding the apworlds already downloaded."""
+
+import os
+
+
+class ApworldFinder:
+    """The download directory, as the worlds a run can test."""
+
+    def __init__(self, root):
+        self.root = root
+
+    @staticmethod
+    def matches(game_name, only, exact):
+        if not only:
+            return True
+        if exact:
+            return any(t.casefold() == game_name.casefold() for t in only)
+        return any(t.lower() in game_name.lower() for t in only)
+
+    def in_folder(self, game_dir):
+        """Every apworld directly inside one game's folder.
+
+        A directory where a file should be is skipped rather than
+        crashing the batch: that is almost always docker auto-creating
+        an empty directory at a bind-mount source that did not exist
+        yet, and a fresh download replaces it next time.
+        """
+        found = []
+        for entry in sorted(os.listdir(game_dir)):
+            if not entry.lower().endswith(".apworld"):
+                continue
+            path = os.path.join(game_dir, entry)
+            if not os.path.isfile(path):
+                print(f"Skipping {path} - not a regular file (directory?)")
+                continue
+            found.append(path)
+        return found
+
+    def find(self, only=None, exact=False):
+        """Every downloaded apworld, optionally narrowed to `only`.
+
+        `only` matches on substring, which is what --only wants. That
+        is ambiguous when one game's name contains another's - "Hollow
+        Knight" also selects "Hollow Knight_ Silksong" - so a caller
+        that already resolved a term to a real folder passes exact.
+        """
+        found = []
+        if not os.path.isdir(self.root):
+            return found
+        for game_name in sorted(os.listdir(self.root)):
+            if not self.matches(game_name, only, exact):
+                continue
+            game_dir = os.path.join(self.root, game_name)
+            if not os.path.isdir(game_dir):
+                continue
+            found += [(game_name, path) for path in self.in_folder(game_dir)]
+        return found
+
+    @property
+    def available(self):
+        return sorted({name for name, _ in self.find()})
+
+    def resolve(self, term):
+        """The folders a name selects, an exact match winning."""
+        matches = sorted({name for name, _ in self.find(only=[term])})
+        exact = [n for n in matches if n.casefold() == term.casefold()]
+        return exact or matches

src/apworld_tester/pipeline/batch/batch.py

@@ -0,0 +1,119 @@
+"""Testing every downloaded apworld and recording what each did."""
+
+from apworld_tester.core.model.fingerprint import Fingerprint
+from apworld_tester.core.state.keys import StateKeys
+from apworld_tester.core.state.store import StateStore
+from apworld_tester.pipeline.batch.companion_pool import CompanionPool
+from apworld_tester.pipeline.batch.failed_report import FailedReport
+from apworld_tester.pipeline.batch.options import Options
+from apworld_tester.pipeline.batch.planner import Planner
+from apworld_tester.pipeline.batch.runner import Runner
+from apworld_tester.pipeline.batch.work_list import WorkList
+
+
+class Batch:
+    """One run: plan it, serve what is cached, test the rest.
+
+    An apworld whose file content and Archipelago version both match
+    its last recorded test is served from cache. The caller forces a
+    full re-test when the version changes, because last week's passes
+    no longer apply to anything.
+    """
+
+    def __init__(self, config, paths, settings, image, tag, force=False,
+                 on_progress=None, **asked):
+        self.config = config
+        self.paths = paths
+        self.settings = settings
+        self.image = image
+        self.tag = tag
+        self.force = force or bool(asked.get("only"))
+        self.on_progress = on_progress
+        self.options = Options(paths, settings, **asked).resolve()
+        self.store = StateStore(self.options["state_path"])
+
+    @property
+    def writes_live_report(self):
+        """Whether this run owns the "what is broken" report.
+
+        The report is about the deployable, current-version state: a
+        previous-version comparison writing here would conflate "not
+        promotable now" with "does not work on a release nobody runs",
+        and a run pointed at a scratch state file is not the live
+        record either.
+        """
+        return (self.options["state_key"]
+                == StateKeys.APWORLD_TESTS
+                and self.options["state_path"] == self.paths.state)
+
+    def pool(self):
+        return CompanionPool(
+            config=self.config,
+            paths=self.paths,
+            store=self.store,
+            image=self.image,
+            tag=self.tag,
+            roms_directory=self.options["roms_directory"],
+        ).resolve()
+
+    def plan(self, tests, fingerprint, state):
+        """Everything to consider, what is known, what to run.
+
+        Blacklisted rows and the confirmation date on a cached one are
+        the entries written outside the runner, which is what normally
+        persists state - without the save here they would exist only
+        in memory, and a run that tested nothing would write nothing.
+        """
+        work = WorkList(
+            root=self.options["root_directory"],
+            only=self.options["only"],
+            exact=self.options["exact_only"],
+            core_games=self.options["core_games"],
+        ).items()
+        planner = Planner(
+            config=self.config,
+            settings=self.settings,
+            tests=tests,
+            tag=self.tag,
+            roms_fingerprint=fingerprint,
+            force=self.force,
+            repeats=self.options["repeats"],
+            random_repeats=self.options["random_repeats"],
+            companion_range=self.options["companion_range"],
+            # A release published after a world's last result
+            # invalidates it, whatever the file on disk hashes to.
+            releases=state.get(StateKeys.GAME_RELEASES, {}),
+        )
+        results, to_test = planner.plan(work)
+        if planner.wrote_state:
+            self.store.save(state)
+        return work, results, to_test
+
+    def report_cached(self, results, total):
+        """Free to produce, so reported without waiting on the pool."""
+        for index, report in enumerate(results, start=1):
+            if self.on_progress:
+                self.on_progress(index, total, report)
+
+    def run(self):
+        self.options["pool"] = self.pool()
+        state = self.store.load()
+        tests = state.setdefault(self.options["state_key"], {})
+        fingerprint = Fingerprint.of_directory(self.options["roms_directory"])
+        work, results, to_test = self.plan(tests, fingerprint, state)
+        self.report_cached(results, len(work))
+        if to_test:
+            results += Runner(
+                config=self.config,
+                paths=self.paths,
+                store=self.store,
+                options=self.options,
+                state=state,
+                tests=tests,
+                tag=self.tag,
+                image=self.image,
+                roms_fingerprint=fingerprint,
+            ).run(to_test, len(work), len(results), self.on_progress)
+        if self.writes_live_report:
+            FailedReport(state, self.paths.failed_report).write()
+        return results

src/apworld_tester/pipeline/batch/cache_check.py

@@ -0,0 +1,120 @@
+"""Whether a previous result still answers this run's question."""
+
+import re
+
+from apworld_tester.core.model.instant import Instant
+from apworld_tester.core.model.modes import Modes
+
+
+class CacheCheck:
+    """One world's recorded result, against what is being asked now."""
+
+    def __init__(self, settings, previous):
+        self.settings = settings
+        self.previous = previous
+
+    #: Failure text pointing at a base ROM, not the world's logic. A
+    #: wrong or missing ROM surfaces under many exception types, so the
+    #: type cannot recognise one and the message is what is left.
+    #: Word-bounded so "from" does not read as "rom".
+    ROM_FAILURE = re.compile(
+        r"\b(rom|md5|sha1|checksum)\b"
+        r"|\.(sfc|smc|z64|n64|nes|gba|gbc|gb|gen|md|iso|bin)\b",
+        re.IGNORECASE,
+    )
+
+    def mentions_rom(self, detail):
+        return bool(detail and self.ROM_FAILURE.search(str(detail)))
+
+    def needs_rom_retry(self, roms_fingerprint):
+        """Whether a changed ROM folder invalidates this result.
+
+        A missing ROM raises FileNotFoundError and lands in
+        "needs_input", but a WRONG one raises whatever the world throws
+        after reading it and lands in "failed" - so retrying only
+        needs_input meant replacing a bad dump could never clear the
+        result. A "failed" result additionally has to look like a ROM
+        problem: most failures carrying a stale fingerprint were fill
+        failures and deprecated APIs that no ROM will fix.
+        """
+        if self.previous is None:
+            return False
+        outcome = self.previous.get("outcome")
+        rom_related = (outcome == "needs_input"
+                       or (outcome == "failed"
+                           and self.mentions_rom(self.previous.get("detail"))))
+        return (rom_related and self.previous.get("roms_fingerprint")
+                != roms_fingerprint)
+
+    def released_since(self, released_at):
+        """Whether the row's apworld was published after it was tested.
+
+        The hash catches a file whose content changed, but not a host
+        that offered a new one and had it turn out identical - and
+        "identical" is only ever this pipeline's word for it, resting
+        on the copy it happened to hold. A publication later than the
+        answer means the answer was reached before whatever is being
+        offered now, and testing again is the only thing that settles
+        it. It settles it for good, too: the new result is dated after
+        the release, so this stops firing until the release moves
+        again.
+        """
+        if self.previous is None:
+            return False
+        released = Instant.of(released_at)
+        tested = Instant.of(self.previous.get("tested_at"))
+        if released is None or tested is None:
+            return False
+        return tested < released
+
+    @staticmethod
+    def attempts(recorded_modes):
+        """How many attempts a cached record was reached with.
+
+        The randomized modes used to run exactly once and carried no
+        count at all, so a missing value there means "one".
+        """
+        if not recorded_modes:
+            return None, None
+        return (
+            (recorded_modes.get("single") or {}).get("requested_attempts"),
+            (recorded_modes.get("single_random") or {})
+            .get("requested_attempts"),
+        )
+
+    def answers(self, repeats, random_repeats, companion_range):
+        """Whether the record answers what this run is asking.
+
+        It does not when the record predates per-mode results, when the
+        set of modes has changed, when it was reached with fewer
+        repeats - "it generated once" is not "it generated ten times
+        out of ten" - or when the companion range differs. A world that
+        never imported is exempt: "unverified" is a complete answer
+        about this exact file, and the file has to change for the
+        answer to.
+        """
+        if self.previous is None:
+            return True
+        if self.previous.get("outcome") == "unverified":
+            return True
+        recorded = self.previous.get("tests")
+        if recorded is None or set(recorded) != set(Modes.ALL):
+            return False
+        single, randomized = self.attempts(recorded)
+        if single != repeats:
+            return False
+        if randomized is not None and randomized != random_repeats:
+            return False
+        recorded_range = (recorded.get("multi") or {}).get(
+            "companion_range") or []
+        return list(recorded_range) == list(companion_range)
+
+    def serves(self, current_hash, tag, roms_fingerprint, repeats,
+               random_repeats, companion_range, released_at=None):
+        """Whether this world can be served from its previous result."""
+        return (self.previous is not None
+                and self.previous.get("hash") == current_hash
+                and self.previous.get("tag") == tag
+                and not self.needs_rom_retry(roms_fingerprint)
+                and not self.released_since(released_at)
+                and self.answers(repeats, random_repeats, companion_range))

src/apworld_tester/pipeline/batch/companion_pool.py

@@ -0,0 +1,65 @@
+"""The core worlds a run may use as companions."""
+
+from datetime import datetime, timezone
+
+from apworld_tester.core.model.fingerprint import Fingerprint
+from apworld_tester.core.state.keys import StateKeys
+from apworld_tester.pipeline.generation.companion_check import (
+    CompanionCheck,
+)
+
+
+class CompanionPool:
+    """Verified once per run and cached against version and ROMs.
+
+    The answer only changes when Archipelago or the ROMs folder does,
+    and what it rejects would otherwise fail seeds belonging to the
+    world under test.
+    """
+
+    def __init__(self, config, paths, store, image, tag, roms_directory):
+        self.config = config
+        self.paths = paths
+        self.store = store
+        self.image = image
+        self.tag = tag
+        self.fingerprint = Fingerprint.of_directory(roms_directory)
+
+    @property
+    def key(self):
+        return StateKeys.COMPANION_POOL
+
+    def cached(self, state):
+        cached = state.get(self.key) or {}
+        if (cached.get("tag") == self.tag
+                and cached.get("roms_fingerprint") == self.fingerprint):
+            return cached.get("games") or []
+        return None
+
+    def store_result(self, state, result):
+        state[self.key] = {
+            "tag": self.tag,
+            "roms_fingerprint": self.fingerprint,
+            "games": result["verified"],
+            "rejected": result["rejected"],
+            "checked_at": datetime.now(timezone.utc).isoformat(),
+        }
+        self.store.save(state)
+        print(f"{len(result['verified'])} core worlds usable as companions, "
+              f"{len(result['rejected'])} rejected")
+        for game, reason in sorted(result["rejected"].items()):
+            print(f"    {game}: {reason[:90]}")
+        return result["verified"]
+
+    def resolve(self):
+        state = self.store.load()
+        cached = self.cached(state)
+        if cached is not None:
+            return cached
+        result = CompanionCheck(self.config, self.paths, self.image).run(
+            self.paths.output)
+        if not result.get("ran"):
+            print(f"companion verification failed ({result.get('detail')}) - "
+                  "multi-game tests will draw from every core world")
+            return []
+        return self.store_result(state, result)

src/apworld_tester/pipeline/batch/discontinued.py

@@ -0,0 +1,69 @@
+"""Flagging test records whose game left the sheet."""
+
+from apworld_tester.core.model.name import Name
+from apworld_tester.core.state.keys import StateKeys
+
+
+class Discontinued:
+    """A world taken off the sheet stops being one this pipeline
+    tracks: it should leave the status page and stop being deployed.
+    The record is kept and labelled rather than deleted, so its history
+    survives and a row that comes back - or that vanished because of a
+    bad sheet parse - is one flag away from being live again.
+    """
+
+    def __init__(self, store):
+        self.store = store
+
+    @staticmethod
+    def sheet_name(key, record):
+        """The sheet row a record's key came from.
+
+        A download- or core-stage record is keyed on the bare sheet
+        name, since nothing was ever fetched for it. Everything else is
+        keyed "<folder>/<file>.apworld". The record's own "game" is
+        deliberately not used: that is the name Archipelago registered
+        the world under, which can differ from the sheet's wording.
+        """
+        if (record or {}).get("stage") in ("download", "core"):
+            return key
+        return key.split("/", 1)[0]
+
+    def apply(self, tests, live):
+        """Set or clear the flag, reporting only what moved.
+
+        Comparison is on the folded directory name of both sides, which
+        is what turned the sheet name into the download folder in the
+        first place, so punctuation and case cannot make a still-listed
+        game look discontinued. An unchanged run leaves the state file
+        byte-identical.
+        """
+        flag = StateKeys.DISCONTINUED
+        changed = {}
+        for key, record in tests.items():
+            name = self.sheet_name(key, record)
+            gone = Name(name).directory not in live
+            if gone != bool(record.get(flag)):
+                changed[key] = gone
+            if gone:
+                record[flag] = True
+            else:
+                record.pop(flag, None)
+        return changed
+
+    def mark(self, sheet_names):
+        """Flag every record whose game is no longer listed.
+
+        Does nothing when handed no sheet names: an empty list means
+        the sheet was not read, not that every world on earth was
+        retired, and acting on it would empty the page.
+        """
+        if not sheet_names:
+            return {}
+        state = self.store.load()
+        tests = state.get(StateKeys.APWORLD_TESTS, {})
+        changed = self.apply(
+            tests, {Name(name).directory for name in sheet_names})
+        if changed:
+            self.store.save(state)
+        return changed

src/apworld_tester/pipeline/batch/download_failures.py

@@ -0,0 +1,88 @@
+"""Recording games whose apworld could never be fetched."""
+
+from datetime import datetime, timezone
+
+from apworld_tester.core.model.name import Name
+from apworld_tester.core.state.keys import StateKeys
+from apworld_tester.pipeline.batch.record import TestRecord
+
+
+class DownloadFailures:
+    """A game with no file has nothing to key a result on the way a
+    tested world does, so it is keyed on its own sheet name - and still
+    shows up as an unknown row rather than vanishing between the sheet
+    and the results. Real result keys always contain "/", so a bare
+    name cannot collide with one.
+    """
+
+    def __init__(self, config, store):
+        self.config = config
+        self.store = store
+
+    @staticmethod
+    def tested_already(tests, name):
+        """Whether a real result already answers for this game.
+
+        A download row exists so a game with no file still shows up
+        rather than vanishing between the sheet and the results. When
+        an apworld is on disk and was tested, nothing is vanishing -
+        the link broke, which the download report says, while the file
+        that world was last tested from is still there and still the
+        answer. Writing a second row then reports one game twice, and
+        the two rows collide at promotion because they claim the same
+        game name.
+        """
+        folded = Name(name).directory
+        return any(
+            "/" in key and Name(key.split("/", 1)[0]).directory == folded
+            for key in tests
+        )
+
+    @staticmethod
+    def detail_for(reason):
+        return ("apworld could not be downloaded, so it was never tested: "
+                f"{reason}")
+
+    def clear_stale(self, tests, games, skipped):
+        """Drop rows for games that downloaded this time.
+
+        Only ever touches entries this class wrote, never a real
+        generation-test result.
+        """
+        failed = {name for name, _ in skipped}
+        for game in games:
+            previous = tests.get(game.name)
+            if (game.name not in failed and previous is not None
+                    and previous.get("stage") == "download"):
+                del tests[game.name]
+
+    def note(self, tests, name, reason, tag, now):
+        """Record one failure, keeping an unchanged row's date.
+
+        A row failing the same way as last time keeps the date of the
+        attempt that reached that answer and only has its last-checked
+        stamp moved: a test date that advanced every run would claim a
+        world was looked at when nothing about it was.
+        """
+        detail = self.detail_for(reason)
+        previous = tests.get(name)
+        if (previous is not None and previous.get("stage") == "download"
+                and previous.get("detail") == detail):
+            previous["tag"] = tag
+            previous[StateKeys.LAST_CHECKED_AT] = now
+            return
+        tests[name] = TestRecord(name).download_failure(
+            detail, tag)
+
+    def record(self, games, skipped, tag):
+        state = self.store.load()
+        tests = state.setdefault(StateKeys.APWORLD_TESTS, {})
+        self.clear_stale(tests, games, skipped)
+        now = datetime.now(timezone.utc).isoformat()
+        for name, reason in skipped:
+            if self.tested_already(tests, name):
+                tests.pop(name, None)
+                continue
+            self.note(tests, name, reason, tag, now)
+        self.store.save(state)
+        return state

src/apworld_tester/pipeline/batch/failed_report.py

@@ -0,0 +1,36 @@
+"""The list of worlds that were tested and did not pass."""
+
+from apworld_tester.core.state.keys import StateKeys
+
+
+class FailedReport:
+    """Written after every run, from the whole accumulated state.
+
+    Not just this run's results: a targeted run touches a couple of
+    entries and the report must still reflect every other known
+    failure. Kept out of state.json, which is bookkeeping rather than a
+    report, so answering the question does not mean writing a query.
+    """
+
+    def __init__(self, state, path):
+        self.state = state
+        self.path = path
+
+    def failed(self):
+        tests = self.state.get(StateKeys.APWORLD_TESTS, {})
+        return sorted(
+            ((key, record) for key, record in tests.items()
+             if record.get("outcome") == "failed"),
+            key=lambda item: item[1].get("game") or item[0],
+        )
+
+    def write(self):
+        with open(self.path, "w", encoding="utf-8") as handle:
+            failed = self.failed()
+            if not failed:
+                handle.write("No failed apworlds.\n")
+                return
+            for key, record in failed:
+                detail = (record.get("detail") or "").splitlines()[0]
+                handle.write(
+                    f"{record.get('game') or key} ({key}): {detail}\n")

src/apworld_tester/pipeline/batch/fetcher.py

@@ -0,0 +1,92 @@
+"""Downloading the worlds a run was asked for but does not have."""
+
+import os
+
+import requests
+
+from apworld_tester.core.config.secrets import Secrets
+from apworld_tester.pipeline.batch.apworld_finder import ApworldFinder
+from apworld_tester.pipeline.download.download_run import DownloadRun
+from apworld_tester.pipeline.sheet.fetcher import SheetFetcher
+from apworld_tester.pipeline.sheet.reader import SheetReader
+
+
+class Fetcher:
+    """Fills the download directory from the sheet, on demand.
+
+    That directory is a working directory this pipeline fills itself,
+    from the same sheet a full pass reads - not somewhere a user
+    points at their own copy. A world never fetched is work not done
+    yet rather than an error.
+    """
+
+    def __init__(self, config, paths, root=None):
+        self.config = config
+        self.paths = paths
+        self.root = root or paths.downloads
+        self.finder = ApworldFinder(self.root)
+
+    def session(self):
+        session = requests.Session()
+        token = Secrets(self.paths).get("GITHUB_TOKEN")
+        if token:
+            session.headers["Authorization"] = f"token {token}"
+        return session
+
+    def listed(self):
+        """The sheet's rows, fetching the sheet if this box has none."""
+        fetcher = SheetFetcher(self.config, self.paths)
+        if not os.path.exists(self.paths.sheet_html):
+            print(f"fetching the {fetcher.tab_name} sheet "
+                  f"-> {self.paths.sheet_html}")
+        fetcher.ensure()
+        return SheetReader(self.config, self.paths.sheet_html).games()
+
+    @staticmethod
+    def row_for(term, listed):
+        """The one sheet row a name selects.
+
+        An exact name wins over a substring, and a term still matching
+        several rows is refused rather than guessed at - downloading
+        and testing two worlds because a name was ambiguous is worse
+        than saying so.
+        """
+        rows = [g for g in listed if term.lower() in g.name.lower()]
+        exact = [g for g in rows if g.name.casefold() == term.casefold()]
+        rows = exact or rows
+        if not rows:
+            raise LookupError(
+                f"{term!r} is not on the worlds sheet - nothing to "
+                "download and nothing to test")
+        if len(rows) > 1:
+            names = ", ".join(g.name for g in rows)
+            raise LookupError(f"{term!r} matches {len(rows)} sheet rows: "
+                              f"{names}. Name one of them exactly.")
+        return rows[0]
+
+    def report(self, terms):
+        """Say where each requested world landed, or raise."""
+        for term in terms:
+            found = self.finder.find(only=[term])
+            if not found:
+                raise LookupError(
+                    f"no apworld could be obtained for {term!r}")
+            for name, path in found:
+                print(f"  {name}: {path}")
+
+    def ensure(self, terms):
+        """Download whichever of `terms` is not already here."""
+        missing = [t for t in terms if not self.finder.find(only=[t])]
+        if not missing:
+            return []
+        listed = self.listed()
+        wanted = [self.row_for(term, listed) for term in missing]
+        print(f"downloading {', '.join(g.name for g in wanted)} "
+              f"-> {self.root}")
+        run = DownloadRun(self.config, self.paths, self.session(),
+                          root_directory=self.root)
+        _, skipped, _ = run.all(wanted)
+        for name, reason in skipped:
+            print(f"  {name} could not be downloaded: {reason}")
+        self.report(missing)
+        return [g.name for g in wanted]

src/apworld_tester/pipeline/batch/options.py

@@ -0,0 +1,48 @@
+"""What one batch run was asked for, with the gaps filled in."""
+
+from apworld_tester.core.state.keys import StateKeys
+
+
+class Options:
+    """The caller's choices, over config.yaml's defaults.
+
+    `repeats` of None means "whatever config.yaml says". An explicit
+    count trades confidence for time, and the cache check compares
+    against it, so a cheap run cannot masquerade as a full one.
+    """
+
+    def __init__(self, paths, settings, **asked):
+        self.paths = paths
+        self.settings = settings
+        self.asked = asked
+
+    def chosen(self, key, default):
+        value = self.asked.get(key)
+        return default if value is None else value
+
+    def counted(self, key, default):
+        value = self.asked.get(key)
+        return default if value is None else int(value)
+
+    def resolve(self):
+        return {
+            "root_directory": self.chosen("root_directory",
+                                          self.paths.downloads),
+            "output_directory": self.chosen("output_directory",
+                                            self.paths.output),
+            "state_path": self.chosen("state_path", self.paths.state),
+            "roms_directory": self.chosen("roms_directory", self.paths.roms),
+            "jobs": self.chosen("jobs", self.settings.jobs),
+            "state_key": self.chosen(
+                "state_key", StateKeys.APWORLD_TESTS),
+            "companion_range": self.chosen("companion_range",
+                                           self.settings.companion_range),
+            "repeats": self.counted("repeats", self.settings.repeats),
+            "random_repeats": self.counted("random_repeats",
+                                           self.settings.random_repeats),
+            "spoiler": self.settings.spoiler,
+            "timeout": self.asked.get("timeout"),
+            "only": self.asked.get("only"),
+            "exact_only": bool(self.asked.get("exact_only")),
+            "core_games": self.asked.get("core_games") or (),
+        }

src/apworld_tester/pipeline/batch/orphaned_records.py

@@ -0,0 +1,57 @@
+"""Dropping test records whose apworld is no longer on disk."""
+
+import os
+
+from apworld_tester.core.state.keys import StateKeys
+
+
+class OrphanedRecords:
+    """Results for files that have since gone.
+
+    A record is keyed on the path its apworld had under the downloads
+    root, so a file that is removed - a release that renamed its asset,
+    a folder cleared by hand - leaves its result behind. Nothing reads
+    that record as stale: it still names a tag and a verdict, so the
+    status page goes on showing the world, and where the replacement
+    file sits beside it the game is reported twice.
+
+    Only records with a file behind them are considered. A download- or
+    core-stage record is keyed on the bare sheet name and never had one,
+    and Discontinued is what retires those.
+    """
+
+    def __init__(self, store, downloads):
+        self.store = store
+        self.downloads = downloads
+
+    def orphaned(self, tests):
+        """Every keyed-on-a-path record whose file is missing."""
+        return [key for key, record in tests.items()
+                if "/" in key
+                and (record or {}).get("stage") not in ("download", "core")
+                and not os.path.isfile(os.path.join(self.downloads, key))]
+
+    def safe(self):
+        """Whether the downloads folder looks like itself.
+
+        An unreadable or empty root would make every record an orphan
+        and empty the page, which is the one outcome worth refusing:
+        a genuinely empty downloads directory has nothing to prune for
+        anyway, so declining costs nothing and guessing costs the lot.
+        """
+        if not os.path.isdir(self.downloads):
+            return False
+        return any(os.scandir(self.downloads))
+
+    def prune(self):
+        """Remove them, returning the keys that went."""
+        if not self.safe():
+            return []
+        state = self.store.load()
+        tests = state.get(StateKeys.APWORLD_TESTS, {})
+        gone = self.orphaned(tests)
+        for key in gone:
+            del tests[key]
+        if gone:
+            self.store.save(state)
+        return gone

src/apworld_tester/pipeline/batch/planner.py

@@ -0,0 +1,104 @@
+"""Splitting the work into known results and tests still to run."""
+
+from apworld_tester.core.model.blacklist import Blacklist
+from apworld_tester.core.model.name import Name
+from apworld_tester.core.state.keys import StateKeys
+from apworld_tester.pipeline.batch.cache_check import CacheCheck
+from apworld_tester.pipeline.batch.record import TestRecord
+
+
+class Planner:
+    """What is already answered, and what has to be generated.
+
+    The blacklist is checked before the cache and is unaffected by a
+    force flag: "do not retry" is the entire reason a world is listed,
+    so a flag that retried it would defeat the point.
+    """
+
+    def __init__(self, config, settings, tests, tag, roms_fingerprint,
+                 force, repeats, random_repeats, companion_range,
+                 releases=None):
+        self.config = config
+        self.settings = settings
+        self.tests = tests
+        self.tag = tag
+        self.roms_fingerprint = roms_fingerprint
+        self.force = force
+        self.repeats = repeats
+        self.random_repeats = random_repeats
+        self.companion_range = companion_range
+        self.blacklist = Blacklist(config)
+        # Keyed on the folded name: the work list names a row by
+        # its download folder, while game_releases keys on the
+        # sheet's own wording, and the two differ wherever the
+        # sheet uses a colon or its own capitalisation.
+        self.releases = {
+            Name(name).directory: (release or {}).get("published_at")
+            for name, release in (releases or {}).items()
+        }
+        self.wrote_state = False
+
+    def record_for(self, item):
+        game_name, apworld_path, key, _ = item
+        return TestRecord(game_name, key,
+                          is_core=apworld_path is None)
+
+    def decline(self, item):
+        """Write and return the result for a declined world."""
+        _, _, key, current_hash = item
+        record = self.record_for(item)
+        self.tests[key] = record.blacklisted(
+            blacklist=self.blacklist,
+            current_hash=current_hash,
+            tag=self.tag,
+            roms_fingerprint=self.roms_fingerprint,
+        )
+        self.wrote_state = True
+        return record.as_result(self.tests[key])
+
+    def confirm(self, previous):
+        """Note that this answer was looked at and still holds.
+
+        A cached result is not re-run, so its tested_at stays at the
+        run that reached it - which on a page refreshed today reads as
+        a world nobody has looked at in weeks rather than as a result
+        nothing has been able to change. This is the date that says
+        which of the two it is, and only a download failure was
+        recording it, so it was missing from every row that passed.
+        """
+        previous[StateKeys.LAST_CHECKED_AT] = TestRecord.now()
+        self.wrote_state = True
+
+    def known(self, item):
+        """This world's answer already, or None if it must be tested."""
+        game_name, _, key, current_hash = item
+        if self.blacklist.listed(game_name):
+            return self.decline(item)
+        previous = self.tests.get(key)
+        cache = CacheCheck(self.settings, previous)
+        served = not self.force and cache.serves(
+            current_hash=current_hash,
+            tag=self.tag,
+            roms_fingerprint=self.roms_fingerprint,
+            repeats=self.repeats,
+            random_repeats=self.random_repeats,
+            companion_range=self.companion_range,
+            # A core world has no release of its own here, so this
+            # is None for one and the rule cannot fire: what dates
+            # it is the Archipelago build, already compared above.
+            released_at=self.releases.get(game_name),
+        )
+        if not served:
+            return None
+        self.confirm(previous)
+        return self.record_for(item).as_result(previous)
+
+    def plan(self, work):
+        results, to_test = [], []
+        for item in work:
+            answer = self.known(item)
+            if answer is None:
+                to_test.append(item)
+            else:
+                results.append(answer)
+        return results, to_test

src/apworld_tester/pipeline/batch/record.py

@@ -0,0 +1,111 @@
+"""The state entry written for one world."""
+
+from datetime import datetime, timezone
+
+from apworld_tester.core.model.blacklist import Blacklist
+from apworld_tester.core.state.keys import StateKeys
+
+
+class TestRecord:
+    """What a finished, declined or undownloadable world records.
+
+    A core world has no file behind it, so it is marked as such and
+    carries no apworld path.
+    """
+
+    def __init__(self, game_name, key=None, is_core=False):
+        self.game_name = game_name
+        self.key = key
+        self.is_core = is_core
+
+    @staticmethod
+    def now():
+        return datetime.now(timezone.utc).isoformat()
+
+    def cored(self, record):
+        if self.is_core:
+            record["stage"] = "core"
+            record["apworld"] = None
+        return record
+
+    def from_report(self, report, current_hash, tag, internal_game,
+                    roms_fingerprint):
+        """One finished test.
+
+        Single and multi verdicts stay separate: different questions.
+        Every mode's own verdict is kept too, so adding one needs no
+        change here. "repaired" is true when the file had to be fixed
+        before it would load - a real result, but for a corrected copy
+        rather than the file the author published.
+        """
+        return self.cored({
+            "hash": current_hash,
+            "tag": tag,
+            "game": internal_game,
+            "outcome": report.get("outcome"),
+            "detail": report.get("detail"),
+            "elapsed_seconds": report.get("elapsed_seconds"),
+            "multi_outcome": report.get("multi_outcome"),
+            "multi_detail": report.get("multi_detail"),
+            "multi_elapsed_seconds": report.get("multi_elapsed_seconds"),
+            "multi_companions": report.get("multi_companions"),
+            "tests": report.get("tests"),
+            "roms_fingerprint": roms_fingerprint,
+            "tested_at": self.now(),
+            "repaired": bool(report.get("repaired")),
+        })
+
+    def blacklisted(self, blacklist, current_hash, tag,
+                    roms_fingerprint):
+        """A world this pipeline declines to test."""
+        return self.cored({
+            "hash": current_hash,
+            "tag": tag,
+            "game": self.game_name,
+            "outcome": Blacklist.OUTCOME,
+            "detail": blacklist.DETAIL,
+            "elapsed_seconds": None,
+            "multi_outcome": None,
+            "multi_detail": None,
+            "multi_elapsed_seconds": None,
+            "multi_companions": None,
+            "tests": None,
+            "roms_fingerprint": roms_fingerprint,
+            "tested_at": self.now(),
+            "blacklisted": True,
+        })
+
+    def download_failure(self, detail, tag):
+        """A game whose apworld could not be fetched.
+
+        "unknown", never "failed": nothing was tested, so nothing is
+        known to be broken. Whether the link was a Discord message, an
+        unsupported host, or a release with no apworld in it, the
+        conclusion is the same - it could not be checked.
+        """
+        return {
+            "game": self.game_name,
+            "apworld": None,
+            "outcome": "unknown",
+            "detail": detail,
+            "elapsed_seconds": None,
+            "stage": "download",
+            "tag": tag,
+            "hash": None,
+            "tested_at": self.now(),
+            StateKeys.LAST_CHECKED_AT: self.now(),
+        }
+
+    def as_result(self, record):
+        """One entry of the caller-facing result list."""
+        return {
+            "game": self.game_name,
+            "apworld": None if self.is_core else self.key,
+            "outcome": record.get("outcome"),
+            "detail": record.get("detail"),
+            "elapsed_seconds": record.get("elapsed_seconds"),
+            "multi_outcome": record.get("multi_outcome"),
+            "multi_detail": record.get("multi_detail"),
+            "tests": record.get("tests"),
+            "skipped": True,
+        }

src/apworld_tester/pipeline/batch/reporter.py

@@ -0,0 +1,49 @@
+"""Saying what each finished world did."""
+
+from apworld_tester.core.display.progress import Progress
+from apworld_tester.core.model.verdict import Verdict
+
+
+class Reporter:
+    """One run's progress callback.
+
+    Holds its own bar rather than a module variable: two runs in one
+    process would otherwise share a bar and a tally.
+    """
+
+    def __init__(self, config, bar=False):
+        self.config = config
+        self.wants_bar = bar
+        self.bar = None
+
+    @staticmethod
+    def marker(report):
+        return ("cached" if report.get("skipped")
+                else f"{report.get('elapsed_seconds')}s")
+
+    def verdicts(self, report):
+        text = f"{report['outcome']}"
+        multi = report.get("multi_outcome")
+        return f"{text} / multi:{multi}" if multi else text
+
+    def drawn(self, index, total, report):
+        if self.bar is None or self.bar.total != total:
+            self.bar = Progress(total=total, config=self.config)
+        self.bar.update(
+            index=index,
+            key=report.get("outcome") or "?",
+            text=report.get("game") or "",
+            fallback_line=(f"[{index}/{total}] {report['game']}: "
+                           f"{self.verdicts(report)} ({self.marker(report)})"),
+        )
+
+    def printed(self, index, total, report):
+        print(f"[{index}/{total}] {report.get('game')}: "
+              f"{Verdict(report, self.config).label} ({self.marker(report)})",
+              flush=True)
+
+    def __call__(self, index, total, report):
+        if self.wants_bar:
+            self.drawn(index, total, report)
+        else:
+            self.printed(index, total, report)

src/apworld_tester/pipeline/batch/results.py

@@ -0,0 +1,95 @@
+"""The reports one run produced."""
+
+from apworld_tester.core.model.modes import Modes
+from apworld_tester.core.model.verdict import Verdict
+
+
+class Results(list):
+    """A run's reports, printable as a table.
+
+    A plain list of the same dicts the batch returns, so anything that
+    worked on those still works. The repr is the addition, because the
+    first thing anyone does with these is look at them.
+    """
+
+    def __init__(self, reports, config):
+        super().__init__(reports)
+        self.config = config
+
+    def verdict(self, report):
+        return Verdict(report, self.config).name
+
+    @property
+    def counts(self):
+        tally = {}
+        for report in self:
+            name = self.verdict(report)
+            tally[name] = tally.get(name, 0) + 1
+        return dict(sorted(tally.items(), key=lambda item: -item[1]))
+
+    @property
+    def summary(self):
+        """Outcome counts, multi verdicts under their own prefix.
+
+        They are a separate question, so they are not merged into one
+        total.
+        """
+        counts = {}
+        for report in self:
+            outcome = report.get("outcome")
+            counts[outcome] = counts.get(outcome, 0) + 1
+            multi = report.get("multi_outcome")
+            if multi:
+                counts[f"multi:{multi}"] = counts.get(f"multi:{multi}", 0) + 1
+        return counts
+
+    #: Best to worst, so the shape of a run reads top to bottom.
+    LADDER = ("stable", "unstable", "flaky", "solo_only", "broken",
+              "unknown")
+
+    @property
+    def overview(self):
+        """The run's verdicts, in the words the sheet uses.
+
+        The counts alone answer "what happened", but only in this
+        pipeline's vocabulary; a reader comparing against the sheet
+        needs the labels, and a reader watching a run needs them in a
+        fixed order rather than sorted by however this run landed.
+        """
+        labels = self.config.value("stability", "labels", {})
+        counts = self.counts
+        names = [name for name in self.LADDER if name in counts]
+        names += [name for name in counts if name not in self.LADDER]
+        width = max((len(labels.get(name, name)) for name in names),
+                    default=0)
+        lines = [f"{len(self)} worlds tested", ""]
+        lines += [
+            f"  {labels.get(name, name):{width}s}  {counts[name]:>4d}"
+            for name in names
+        ]
+        return "\n".join(lines)
+
+    def row(self, report, width):
+        """One world's line: name, verdict, every mode's outcome."""
+        mode_results = report.get("tests") or {}
+        per_mode = " ".join(
+            f"{mode}={(mode_results.get(mode) or {}).get('outcome') or '-'}"
+            for mode in Modes.ALL
+        )
+        marker = " (cached)" if report.get("skipped") else ""
+        return (f"{report.get('game') or '?':{width}s}  "
+                f"{Verdict(report, self.config).label:12s}  "
+                f"{per_mode}{marker}")
+
+    def __repr__(self):
+        if not self:
+            return "no worlds tested"
+        width = max(len(report.get("game") or "") for report in self)
+        lines = [
+            self.row(report, width)
+            for report in sorted(self, key=lambda r: r.get("game") or "")
+        ]
+        if len(self) > 1:
+            lines += ["", "  ".join(f"{name}: {count}"
+                                    for name, count in self.counts.items())]
+        return "\n".join(lines)

src/apworld_tester/pipeline/batch/run.py

@@ -0,0 +1,92 @@
+"""Testing worlds by hand, from the notebook or a shell."""
+
+from apworld_tester.core.config.config import Config
+from apworld_tester.core.config.paths import Paths
+from apworld_tester.core.config.run_settings import RunSettings
+from apworld_tester.core.state.lock import PipelineLock
+from apworld_tester.core.state.scratch import ScratchState
+from apworld_tester.core.state.store import StateStore
+from apworld_tester.pipeline.batch.apworld_finder import ApworldFinder
+from apworld_tester.pipeline.batch.batch import Batch
+from apworld_tester.pipeline.batch.fetcher import Fetcher
+from apworld_tester.pipeline.batch.reporter import Reporter
+from apworld_tester.pipeline.batch.results import Results
+from apworld_tester.pipeline.batch.version import Version
+from apworld_tester.pipeline.build.archipelago import ArchipelagoBuild
+
+
+class TestRun:
+    """Test one world, several, or every downloaded one.
+
+    `live` records into the real state file - off by default, because a
+    hand-run test is an experiment. `cached` only affects a whole-set
+    run; a named world is always retested.
+    """
+
+    def __init__(self, game=None, repeats=None, random_repeats=None, jobs=2,
+                 live=False, cached=False, root_directory=None, quiet=False,
+                 update=False):
+        self.config = Config.load()
+        self.paths = Paths(self.config)
+        self.settings = RunSettings(self.config)
+        self.game = game
+        self.repeats = repeats
+        self.random_repeats = random_repeats
+        self.jobs = jobs
+        self.live = live
+        self.cached = cached
+        self.root = root_directory or self.paths.downloads
+        self.quiet = quiet
+        self.update = update
+
+    @property
+    def state_path(self):
+        return ScratchState(self.paths, live=self.live).path
+
+    def worlds(self):
+        """The folders to test, downloading any that are missing.
+
+        Resolved to real folder names and matched exactly from here on,
+        so the set tested is the set that was just named.
+        """
+        if self.game is None:
+            return None
+        terms = [self.game] if isinstance(self.game, str) else list(self.game)
+        Fetcher(self.config, self.paths, self.root).ensure(terms)
+        finder = ApworldFinder(self.root)
+        return [name for term in terms for name in finder.resolve(term)]
+
+    def announce(self, tag, only):
+        """What is about to run, and nothing else."""
+        named = ", ".join(only) if only else "every downloaded apworld"
+        print(f"archipelago {tag}")
+        print(f"testing     {named}")
+
+    def run(self):
+        """The reports, printable as a table."""
+        store = StateStore(self.paths.state)
+        build = ArchipelagoBuild(self.config, self.paths)
+        tag, _ = Version(self.config, self.paths, store).prepare(
+            update=self.update)
+        only = self.worlds()
+        build.ensure_test_image(tag)
+        state_path = self.state_path
+        if not self.quiet:
+            self.announce(tag, only)
+        with PipelineLock(self.paths.lock):
+            return Results(Batch(
+                config=self.config,
+                paths=self.paths,
+                settings=self.settings,
+                image=build.test_image_tag(tag),
+                tag=tag,
+                force=not self.cached,
+                on_progress=None if self.quiet else Reporter(self.config),
+                root_directory=self.root,
+                state_path=state_path,
+                only=only,
+                exact_only=only is not None,
+                jobs=self.jobs,
+                repeats=self.repeats,
+                random_repeats=self.random_repeats,
+            ).run(), self.config)

src/apworld_tester/pipeline/batch/runner.py

@@ -0,0 +1,86 @@
+"""Running the tests the cache could not answer."""
+
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from apworld_tester.pipeline.batch.record import TestRecord
+from apworld_tester.pipeline.generation.tester import WorldTester
+
+
+class Runner:
+    """Tests each outstanding world and persists what it did."""
+
+    def __init__(self, config, paths, store, options, state, tests, tag,
+                 image, roms_fingerprint):
+        self.config = config
+        self.paths = paths
+        self.store = store
+        self.options = options
+        self.state = state
+        self.tests = tests
+        self.tag = tag
+        self.tester = WorldTester(config, paths, image)
+        self.roms_fingerprint = roms_fingerprint
+        self.lock = threading.Lock()
+
+    @staticmethod
+    def promotion_name(report, game_name):
+        """The name Archipelago itself registered this world under.
+
+        Recorded for promotion's collision check, while the report's
+        own "game" becomes the sheet folder name for display. A world
+        that never loaded has no real name, so it falls back to a
+        marked sheet name - otherwise every such entry would collide
+        under one None key and none would promote.
+        """
+        return report.get("game") or f"unverified:{game_name}"
+
+    def store_record(self, key, record):
+        """Persist after every test, not just at the end, so a killed
+        run does not lose the tests it already completed.
+        """
+        with self.lock:
+            self.tests[key] = record
+            self.store.save(self.state)
+
+    def test_one(self, game_name, apworld_path, key, current_hash):
+        options = self.options
+        report = self.tester.test(
+            apworld_path=apworld_path,
+            game=None if apworld_path else game_name,
+            output_dir=options["output_directory"],
+            timeout=options["timeout"],
+            companion_range=options["companion_range"],
+            spoiler=options["spoiler"],
+            repeats=options["repeats"],
+            companion_pool=options["pool"],
+            random_repeats=options["random_repeats"],
+        )
+        record = TestRecord(game_name, key,
+                            is_core=apworld_path is None)
+        self.store_record(key, record.from_report(
+            report=report,
+            current_hash=current_hash,
+            tag=self.tag,
+            internal_game=self.promotion_name(report, game_name),
+            roms_fingerprint=self.roms_fingerprint,
+        ))
+        report["game"] = game_name
+        report["apworld"] = key
+        report["skipped"] = False
+        return report
+
+    def run(self, to_test, total, completed, on_progress):
+        """Run the outstanding tests, reporting as they land."""
+        results = []
+        reporting = threading.Lock()
+        with ThreadPoolExecutor(max_workers=self.options["jobs"]) as pool:
+            futures = [pool.submit(self.test_one, *args) for args in to_test]
+            for future in as_completed(futures):
+                report = future.result()
+                results.append(report)
+                with reporting:
+                    completed += 1
+                    if on_progress:
+                        on_progress(completed, total, report)
+        return results

src/apworld_tester/pipeline/batch/session.py

@@ -0,0 +1,92 @@
+"""Testing worlds by hand, from the notebook or a shell."""
+
+from apworld_tester.core.config.config import Config
+from apworld_tester.core.config.paths import Paths
+from apworld_tester.core.config.run_settings import RunSettings
+from apworld_tester.core.state.lock import PipelineLock
+from apworld_tester.core.state.scratch import ScratchState
+from apworld_tester.core.state.store import StateStore
+from apworld_tester.pipeline.batch.apworld_finder import ApworldFinder
+from apworld_tester.pipeline.batch.batch import Batch
+from apworld_tester.pipeline.batch.fetcher import Fetcher
+from apworld_tester.pipeline.batch.reporter import Reporter
+from apworld_tester.pipeline.batch.results import Results
+from apworld_tester.pipeline.batch.version import Version
+from apworld_tester.pipeline.build.archipelago import ArchipelagoBuild
+
+
+class TestSession:
+    """Test one world, several, or every downloaded one.
+
+    `live` records into the real state file - off by default, because a
+    hand-run test is an experiment. `cached` only affects a whole-set
+    run; a named world is always retested.
+    """
+
+    def __init__(self, game=None, repeats=None, random_repeats=None, jobs=2,
+                 live=False, cached=False, root_directory=None, quiet=False,
+                 update=False):
+        self.config = Config.load()
+        self.paths = Paths(self.config)
+        self.settings = RunSettings(self.config)
+        self.game = game
+        self.repeats = repeats
+        self.random_repeats = random_repeats
+        self.jobs = jobs
+        self.live = live
+        self.cached = cached
+        self.root = root_directory or self.paths.downloads
+        self.quiet = quiet
+        self.update = update
+
+    @property
+    def state_path(self):
+        return ScratchState(self.paths, live=self.live).path
+
+    def worlds(self):
+        """The folders to test, downloading any that are missing.
+
+        Resolved to real folder names and matched exactly from here on,
+        so the set tested is the set that was just named.
+        """
+        if self.game is None:
+            return None
+        terms = [self.game] if isinstance(self.game, str) else list(self.game)
+        Fetcher(self.config, self.paths, self.root).ensure(terms)
+        finder = ApworldFinder(self.root)
+        return [name for term in terms for name in finder.resolve(term)]
+
+    def announce(self, tag, only):
+        """What is about to run, and nothing else."""
+        named = ", ".join(only) if only else "every downloaded apworld"
+        print(f"archipelago {tag}")
+        print(f"testing     {named}")
+
+    def run(self):
+        """The reports, printable as a table."""
+        store = StateStore(self.paths.state)
+        build = ArchipelagoBuild(self.config, self.paths)
+        tag, _ = Version(self.config, self.paths, store).prepare(
+            update=self.update)
+        only = self.worlds()
+        build.ensure_test_image(tag)
+        state_path = self.state_path
+        if not self.quiet:
+            self.announce(tag, only)
+        with PipelineLock(self.paths.lock):
+            return Results(Batch(
+                config=self.config,
+                paths=self.paths,
+                settings=self.settings,
+                image=build.test_image_tag(tag),
+                tag=tag,
+                force=not self.cached,
+                on_progress=None if self.quiet else Reporter(self.config),
+                root_directory=self.root,
+                state_path=state_path,
+                only=only,
+                exact_only=only is not None,
+                jobs=self.jobs,
+                repeats=self.repeats,
+                random_repeats=self.random_repeats,
+            ).run(), self.config)

src/apworld_tester/pipeline/batch/version.py

@@ -0,0 +1,87 @@
+"""Which Archipelago version a run tests against."""
+
+from apworld_tester.core.state.keys import StateKeys
+from apworld_tester.pipeline.build.archipelago import ArchipelagoBuild
+from apworld_tester.pipeline.build.docker import Docker
+
+
+class Version:
+    """Resolve the version, and build it if this machine lacks it."""
+
+    def __init__(self, config, paths, store):
+        self.paths = paths
+        self.store = store
+        self.build = ArchipelagoBuild(config, paths)
+
+    def resolve(self):
+        """The version to test, and where that choice came from.
+
+        A pin is a decision already made, so nothing else revisits it.
+        Failing that the recorded version wins, then whichever image is
+        on this machine - a test runs entirely inside the container, so
+        an image already here is everything it needs.
+        """
+        pinned = self.build.pinned_version
+        if pinned:
+            return pinned, "pinned in config.yaml"
+        recorded = self.store.load().get(StateKeys.ARCHIPELAGO_TAG)
+        if recorded:
+            return recorded, "from state.json"
+        local = next(iter(self.build.local_tags()), None)
+        return (local, "newest image on this machine") if local else (None,
+                                                                     None)
+
+    def ensure_built(self, tag):
+        """Build what this version needs, if anything is missing.
+
+        Building needs the Archipelago source, which is the one job the
+        checkout has - so this is also the only path that clones.
+        """
+        if tag is None:
+            print("no Archipelago image on this machine - cloning the "
+                  "newest release to build one (several minutes) ...")
+            built, _, _, _ = self.build.update_and_build(self.store)
+            return built, "just built"
+        if not Docker.image_exists(self.build.image_tag(tag)):
+            print(f"Archipelago {tag} is not built on this machine - "
+                  "cloning and building it (several minutes) ...")
+            self.build.build_version(tag)
+        return tag, None
+
+    def note_others(self, tag, source):
+        """Say so when an unpinned run is not using the newest image.
+
+        An unpinned tag can come from an image that happens to be lying
+        around, so a run can quietly answer for a version that is not
+        the current one. A pinned tag was chosen deliberately.
+        """
+        local = self.build.local_tags()
+        if self.build.pinned_version or not local or tag == local[0]:
+            return source
+        return f"{source}, but {local[0]} is also built here"
+
+    def update(self):
+        """Resolve the newest release, build it, and record it.
+
+        Never automatic. A new version invalidates every cached result,
+        so the full re-test it forces is the caller's decision, not a
+        side effect of running. A pinned version stays pinned: the
+        resolution behind this honours the pin, so an update then only
+        ensures that version is built.
+        """
+        tag, previous, changed, _ = self.build.update_and_build(self.store)
+        if self.build.pinned_version:
+            return tag, "pinned in config.yaml"
+        if not changed:
+            return tag, "already the newest release"
+        if previous is None:
+            return tag, "newest release, recorded for the first time"
+        return tag, f"updated from {previous}"
+
+    def prepare(self, update=False):
+        """The version to test against, built if it is missing."""
+        if update:
+            return self.update()
+        tag, source = self.resolve()
+        tag, built = self.ensure_built(tag)
+        return tag, built or self.note_others(tag, source)

src/apworld_tester/pipeline/batch/work_list.py

@@ -0,0 +1,41 @@
+"""Everything one run will consider testing."""
+
+import os
+
+from apworld_tester.core.model.fingerprint import Fingerprint
+from apworld_tester.pipeline.batch.apworld_finder import ApworldFinder
+
+
+class WorkList:
+    """Each world to consider, with its cache key and hash.
+
+    A core world is tested by name with no file behind it, so it is
+    keyed on the bare sheet name - the same "no slash means no apworld"
+    convention download failures follow - and gets a stand-in hash
+    whose real content is the Archipelago build beside it.
+    """
+
+    #: Stands in for a core world's file hash. Its real content is
+    #: the Archipelago build, so the version compared beside it in the
+    #: cache check is what decides staleness.
+    CORE_HASH = "core"
+
+    def __init__(self, root, only=None, exact=False, core_games=()):
+        self.root = root
+        self.only = only
+        self.exact = exact
+        self.core_games = core_games
+
+    def items(self):
+        found = ApworldFinder(self.root).find(only=self.only,
+                                              exact=self.exact)
+        work = [
+            (name, path, os.path.relpath(path, self.root),
+             Fingerprint.of_file(path))
+            for name, path in found
+        ]
+        return work + [
+            (name, None, name, WorkList.CORE_HASH)
+            for name in self.core_games
+            if self.only is None or name in self.only
+        ]

src/apworld_tester/pipeline/download/asset_matcher.py

@@ -0,0 +1,82 @@
+"""Choosing which asset in a release belongs to this game."""
+
+import re
+
+from apworld_tester.core.model.name import Name
+
+
+class AssetMatcher:
+    """Matches release assets and release titles to one sheet row."""
+
+    #: Title slugs that name no game at all, so they are never read as
+    #: evidence that a release belongs to a different sheet row. Not
+    #: configuration: they are the words every release title uses about
+    #: itself, and a release titled "Apworld Release" claims nobody.
+    UNINFORMATIVE = frozenset({
+        "archipelago",
+        "apworld",
+        "release",
+        "latest",
+        "update",
+    })
+
+    def __init__(self, config, game_name):
+        self.config = config
+        self.game = Name(game_name)
+
+    def named(self, assets, exact):
+        """The assets whose filename names this game."""
+        matched = []
+        for asset in assets:
+            name = Name(asset["name"].rsplit(".", 1)[0]).slug
+            if (name == self.game.slug) if exact else (name in self.game.slug):
+                matched.append(asset)
+        return matched
+
+    def select(self, assets):
+        """The assets to download for this game."""
+        if len(assets) <= 1:
+            return assets
+        for matches in (self.named(assets, True), self.named(assets, False)):
+            if len(matches) == 1:
+                return matches
+        return assets
+
+    def title_slug(self, release):
+        """A release's own title, reduced to the game it is for."""
+        title = release.get("name") or release.get("tag_name") or ""
+        title = re.sub(r"\bv?\d+(?:\.\d+)*\b", " ", title)
+        title = re.sub(r"\bapworld\b", " ", title, flags=re.IGNORECASE)
+        return Name(title).slug
+
+    def claimed_by(self, title, sheet_names):
+        """The other sheet row this title names, or None.
+
+        Exact equality only. This is not "the title looks a bit off" -
+        it is positive evidence that a DIFFERENT row is named, which is
+        the only thing strong enough to skip a release the link points
+        at. "Mega Man X2" is exactly a row and is skipped when resolving
+        "Mega Man X"; "Mega Man X1" is nobody's row and is accepted.
+        """
+        if not sheet_names or not self.game.text:
+            return None
+        if not title or title in self.UNINFORMATIVE:
+            return None
+        if title == self.game.slug:
+            return None
+        for other in sheet_names:
+            slug = Name(other).slug
+            if slug != self.game.slug and slug == title:
+                return other
+        return None
+
+    def release_claimed(self, release, sheet_names):
+        return self.claimed_by(self.title_slug(release), sheet_names)
+
+    def identity_claimed(self, found_games, sheet_names):
+        """Whether a downloaded file identifies as another row's game."""
+        for found in found_games or []:
+            other = self.claimed_by(Name(found).slug, sheet_names)
+            if other:
+                return other
+        return None

src/apworld_tester/pipeline/download/auth_error.py

@@ -0,0 +1,23 @@
+"""A token failure, which is the whole run's problem."""
+
+import requests
+
+
+class GitHubAuthError(RuntimeError):
+    """A bad or expired token affects every remaining request."""
+
+    @classmethod
+    def matching(cls, game, error):
+        """This error as an auth failure, or None if it is not one.
+
+        Raised loudly so the whole pool stops, rather than quietly
+        failing game after game against a token that will not work.
+        """
+        if not isinstance(error, requests.HTTPError):
+            return None
+        response = error.response
+        status = response.status_code if response is not None else None
+        if status not in (401, 403):
+            return None
+        return cls(f"GitHub rejected the request for {game.name} with "
+                   f"{status} - check .env / token expiry.")

src/apworld_tester/pipeline/download/context.py

@@ -0,0 +1,58 @@
+"""What every download shares."""
+
+import os
+
+from apworld_tester.core.model.name import Name
+from apworld_tester.pipeline.download.asset_matcher import AssetMatcher
+
+
+class DownloadContext:
+    """Who is downloading what, and where it goes.
+
+    One object rather than the same six arguments threaded through
+    every handler and helper.
+    """
+
+    def __init__(self, config, paths, session, game, identifier=None,
+                 sheet_names=None, root_directory=None):
+        self.config = config
+        self.paths = paths
+        self.session = session
+        self.game = game
+        self.identifier = identifier
+        self.sheet_names = sheet_names
+        self.root = root_directory or paths.downloads
+        self.matcher = AssetMatcher(config, game.name)
+
+    @property
+    def target_directory(self):
+        return os.path.join(self.root, Name(self.game.name).directory)
+
+    def target_path(self, filename):
+        return os.path.join(self.target_directory, filename)
+
+    @property
+    def plain_headers(self):
+        """Headers for a host that is not GitHub.
+
+        The session carries a GitHub token, which must never be sent to
+        an unrelated site.
+        """
+        return {"Authorization": None, "Accept": "*/*"}
+
+    @property
+    def json_headers(self):
+        """Headers for another forge's JSON API, without the token."""
+        return {"Authorization": None, "Accept": "application/json"}
+
+    def wrong_game(self, path):
+        """Why this file is another game's, or None to keep it."""
+        if self.identifier is None:
+            return None
+        identity = self.identifier.identify(path)
+        if not identity.get("ran"):
+            return None
+        found = identity.get("games") or []
+        if any(Name(self.game.name).matches(name) for name in found):
+            return None
+        return found or identity.get("detail")

src/apworld_tester/pipeline/download/download_run.py

@@ -0,0 +1,156 @@
+"""Fetching every game's apworld, concurrently."""
+
+import os
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import requests
+
+from apworld_tester.core.state.keys import StateKeys
+from apworld_tester.core.state.store import StateStore
+from apworld_tester.pipeline.download.auth_error import GitHubAuthError
+from apworld_tester.pipeline.download.context import DownloadContext
+from apworld_tester.pipeline.download.downloader import Downloader
+from apworld_tester.pipeline.download.superseded_assets import (
+    SupersededAssets,
+)
+
+
+class DownloadRun:
+    """One pass over the sheet's rows.
+
+    `sheet_names` should be the FULL sheet: a file can identify as a
+    core row, which never reaches here.
+    """
+
+    def __init__(self, config, paths, session, root_directory=None,
+                 identifier=None, jobs=None, sheet_names=None,
+                 previous_releases=None):
+        self.config = config
+        self.paths = paths
+        self.session = session
+        self.root = root_directory or paths.downloads
+        self.identifier = identifier
+        self.jobs = jobs or int(config.value("general", "threads", 4))
+        self.sheet_names = sheet_names
+        self.previous_releases = previous_releases or {}
+
+    @staticmethod
+    def split(games):
+        """The games worth a network call, and those answered now.
+
+        A row with no link has nothing to look up, so it never takes a
+        slot in the thread pool.
+        """
+        downloadable, skipped = [], []
+        for game in games:
+            if not game.release and not game.links:
+                skipped.append((game.name, "no release link in the sheet"))
+            else:
+                downloadable.append(game)
+        return downloadable, skipped
+
+    def context_for(self, game):
+        return DownloadContext(
+            config=self.config,
+            paths=self.paths,
+            session=self.session,
+            game=game,
+            identifier=self.identifier,
+            sheet_names=self.sheet_names,
+            root_directory=self.root,
+        )
+
+    def one(self, game):
+        """Download one game, which never raises for that game alone.
+
+        A network failure is that game's own result, not the run's -
+        except an auth failure, which is everyone's.
+        """
+        try:
+            downloaded, reason, info = Downloader(
+                context=self.context_for(game),
+                previous=self.previous_releases.get(game.name),
+            ).run()
+        except requests.RequestException as error:
+            auth = GitHubAuthError.matching(game, error)
+            if auth is not None:
+                raise auth from error
+            return game, None, str(error), None
+        return game, downloaded, reason, info
+
+    @staticmethod
+    def source_url(game):
+        """Where this apworld came from, for the status page.
+
+        game.release is only ever a real releases link, so for anything
+        resolved through the plain links the first one stands in.
+        """
+        return game.release or (game.links[0] if game.links else None)
+
+    def record(self, finished, releases, skipped, completed, total,
+               on_asset, on_game):
+        """Record one finished game and tell the caller about it.
+
+        The release is recorded even on failure - it was still found,
+        so its version is worth showing. on_game is called for failures
+        too, unlike on_asset: a progress bar has to count every game
+        that finished, not only the ones that produced a file.
+        """
+        game, downloaded, reason, info = finished
+        releases[game.name] = {"url": self.source_url(game), **(info or {})}
+        if downloaded is None:
+            skipped.append((game.name, reason))
+            if on_game:
+                on_game(completed, total, game.name, False)
+            return 0
+        paths = [path for path, _ in downloaded]
+        for path in SupersededAssets(os.path.dirname(paths[0]),
+                                     paths).prune():
+            print(f"{game.name}: removed superseded {os.path.basename(path)}"
+                  f" - the release no longer offers it")
+        for path, status in downloaded:
+            if on_asset:
+                on_asset(game.name, path, status)
+        if on_game:
+            on_game(completed, total, game.name, True)
+        return len(downloaded)
+
+    @staticmethod
+    def result_of(future, executor):
+        """One finished download, stopping the pool on auth failure."""
+        try:
+            return future.result()
+        except GitHubAuthError:
+            executor.shutdown(cancel_futures=True)
+            raise
+
+    def all(self, games, on_asset=None, on_game=None):
+        """Every game's apworld: how many files, what was skipped, and
+        which release each row resolved to.
+
+        Recording holds a lock, so on_asset always runs serialized and
+        a caller's own progress counter needs none of its own.
+        """
+        self.sheet_names = self.sheet_names or [g.name for g in games]
+        downloadable, skipped = self.split(games)
+        releases, lock = {}, threading.Lock()
+        total = completed = 0
+        with ThreadPoolExecutor(max_workers=self.jobs) as executor:
+            futures = [executor.submit(self.one, g) for g in downloadable]
+            for future in as_completed(futures):
+                finished = self.result_of(future, executor)
+                with lock:
+                    completed += 1
+                    total += self.record(
+                        finished, releases, skipped, completed,
+                        len(downloadable), on_asset, on_game)
+        return total, skipped, releases
+
+    def record_releases(self, releases, state_path=None):
+        """Write each game's resolved release into the state file."""
+        store = StateStore(state_path or self.paths.state)
+        state = store.load()
+        state.setdefault(StateKeys.GAME_RELEASES, {}).update(releases)
+        store.save(state)
+        return state

src/apworld_tester/pipeline/download/downloader.py

@@ -0,0 +1,75 @@
+"""Fetching one sheet row's apworld."""
+
+from apworld_tester.pipeline.download.handlers.gitea_release import (
+    GiteaRelease,
+)
+from apworld_tester.pipeline.download.handlers.github_file import (
+    GitHubFile,
+)
+from apworld_tester.pipeline.download.handlers.github_release import (
+    GitHubRelease,
+)
+from apworld_tester.pipeline.download.handlers.github_repo import (
+    GitHubRepo,
+)
+from apworld_tester.pipeline.download.handlers.gitlab_file import (
+    GitLabFile,
+)
+from apworld_tester.pipeline.download.handlers.gitlab_release import (
+    GitLabRelease,
+)
+from apworld_tester.pipeline.download.handlers.plain_file import PlainFile
+from apworld_tester.pipeline.download.unusable_links import UnusableLinks
+from apworld_tester.pipeline.download.url_shapes import UrlShapes
+
+
+class Downloader:
+    """Whatever shape this game's link is, get the apworld.
+
+    A releases link is resolved directly; anything else is tried
+    against every handler in turn, most specific first, so an exact
+    file beats browsing a whole repo and the plain-URL handler can
+    never shadow a forge-specific one.
+    """
+
+    def __init__(self, context, previous=None):
+        self.context = context
+        self.shapes = UrlShapes(context.config)
+        self.previous = previous
+
+    def handlers(self):
+        return (
+            GitHubFile(self.shapes),
+            GitHubRepo(self.shapes),
+            GitLabFile(self.shapes),
+            GitLabRelease(self.shapes),
+            GiteaRelease(self.shapes, previous=self.previous),
+            PlainFile(self.shapes),
+        )
+
+    def from_release_link(self):
+        handler = GitHubRelease(self.shapes, previous=self.previous)
+        parsed = handler.parse(self.context.game.release)
+        if parsed is None:
+            return None, "not a recognized releases URL", None
+        return handler.download(parsed, self.context)
+
+    def from_link(self, link):
+        """This link's result, or None if no handler recognised it."""
+        for handler in self.handlers():
+            parsed = handler.parse(link)
+            if parsed is not None:
+                return handler.download(parsed, self.context)
+        return None
+
+    def run(self):
+        """The files downloaded, why none were, and the release info."""
+        game = self.context.game
+        if game.release:
+            return self.from_release_link()
+        for link in game.links:
+            result = self.from_link(link)
+            if result is not None:
+                return result
+        return None, UnusableLinks(self.context.config, game.links).detail(), \
+            None

src/apworld_tester/pipeline/download/handlers/base.py

@@ -0,0 +1,45 @@
+"""What every link handler shares."""
+
+import os
+
+from apworld_tester.pipeline.download.local_file import LocalFile
+
+
+class LinkHandler:
+    """One recognised shape of sheet link, and how to fetch from it.
+
+    parse() returning None means "not my shape", which is different from
+    "mine, and it failed": the caller only moves on to the next link in
+    the first case.
+    """
+
+    def __init__(self, shapes):
+        self.shapes = shapes
+
+    def parse(self, link):
+        raise NotImplementedError
+
+    def download(self, parsed, context):
+        raise NotImplementedError
+
+    def file(self, context, filename):
+        return LocalFile(context.session, context.target_path(filename))
+
+    def collect(self, candidates, context, fetch, verify):
+        """Download each candidate, dropping any that is another game."""
+        downloaded, kept, rejected = [], [], []
+        for candidate in candidates:
+            path, status = fetch(candidate)
+            if status in LocalFile.NEEDS_DOWNLOAD and verify:
+                mismatch = context.wrong_game(path)
+                if mismatch is not None:
+                    os.remove(path)
+                    rejected.append((candidate["name"], mismatch))
+                    continue
+            downloaded.append((path, status))
+            kept.append(candidate)
+        return downloaded, kept, rejected
+
+    def rejected_detail(self, rejected, subject):
+        details = "; ".join(f"{name} -> {found}" for name, found in rejected)
+        return f"downloaded {subject} didn't match this game: {details}"

src/apworld_tester/pipeline/download/handlers/gitea_release.py

@@ -0,0 +1,44 @@
+"""A releases page on a self-hosted Gitea or Forgejo instance."""
+
+from apworld_tester.pipeline.download.handlers.base import LinkHandler
+from apworld_tester.pipeline.download.release_downloader import (
+    ReleaseDownloader,
+)
+
+
+class GiteaRelease(LinkHandler):
+    """Codeberg and friends.
+
+    The release JSON is close enough to GitHub's that ReleaseDownloader
+    handles it unchanged, given headers that omit the GitHub token.
+    """
+
+    def __init__(self, shapes, previous=None):
+        super().__init__(shapes)
+        self.previous = previous
+
+    def parse(self, link):
+        return self.shapes.gitea_releases(link)
+
+    def latest(self, context, host, owner, repository):
+        response = context.session.get(
+            f"https://{host}/api/v1/repos/{owner}/{repository}"
+            "/releases/latest",
+            headers=context.json_headers,
+        )
+        if response.status_code == 404:
+            return None
+        response.raise_for_status()
+        return response.json()
+
+    def download(self, parsed, context):
+        host, owner, repository = parsed
+        release = self.latest(context, host, owner, repository)
+        if release is None:
+            return None, "no release found on this host", None
+        return ReleaseDownloader(
+            context=context,
+            release=release,
+            headers=context.plain_headers,
+            previous_published_at=(self.previous or {}).get("published_at"),
+        ).run()

src/apworld_tester/pipeline/download/handlers/github_file.py

@@ -0,0 +1,58 @@
+"""A link straight at an apworld committed in a GitHub repo."""
+
+import os
+import urllib.parse
+
+import requests
+
+from apworld_tester.pipeline.build.archipelago import (
+    ArchipelagoBuild,
+)
+from apworld_tester.pipeline.download.local_file import LocalFile
+from apworld_tester.pipeline.download.handlers.base import LinkHandler
+
+
+class GitHubFile(LinkHandler):
+    """A committed file, named exactly by the sheet.
+
+    No identity check: the link names one file, so there is no second
+    candidate a name comparison could choose between.
+    """
+
+    def parse(self, link):
+        return self.shapes.github_file(link)
+
+    def download(self, parsed, context):
+        """Fetch the file as it currently stands on that branch."""
+        owner, repository, branch, path = parsed
+        filename = os.path.basename(urllib.parse.unquote(path))
+        local = self.file(context, filename)
+        url = ("https://raw.githubusercontent.com/"
+               f"{owner}/{repository}/{branch}/{path}")
+        status = local.status(local.remote_size(url))
+        if status in LocalFile.NEEDS_DOWNLOAD:
+            local.download(url)
+        published = self.published_at(context, owner, repository, path, branch)
+        return [(local.path, status)], None, {"published_at": published}
+
+    def published_at(self, context, owner, repository, path, branch):
+        """When this file was last committed, or None.
+
+        Best effort: a blob link has no release to read a date from, and
+        a download should not fail over display information.
+        """
+        try:
+            response = context.session.get(
+                f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
+                "/commits",
+                params={"path": path, "sha": branch, "per_page": 1},
+            )
+            response.raise_for_status()
+            commits = response.json()
+        except requests.RequestException:
+            return None
+        if not commits:
+            return None
+        commit = commits[0].get("commit", {})
+        return ((commit.get("committer") or {}).get("date")
+                or (commit.get("author") or {}).get("date"))

src/apworld_tester/pipeline/download/handlers/github_release.py

@@ -0,0 +1,81 @@
+"""A GitHub releases page, the sheet's usual link."""
+
+from apworld_tester.pipeline.download.handlers.base import LinkHandler
+from apworld_tester.pipeline.download.release_downloader import (
+    ReleaseDownloader,
+)
+from apworld_tester.pipeline.download.release_identity import (
+    ReleaseIdentity,
+)
+from apworld_tester.pipeline.download.release_picker import ReleasePicker
+from apworld_tester.pipeline.download.walk_back import WalkBack
+
+
+class GitHubRelease(LinkHandler):
+    """Resolve which release the link means, then fetch its apworld."""
+
+    def __init__(self, shapes, previous=None):
+        super().__init__(shapes)
+        self.previous = previous
+
+    def parse(self, link):
+        return self.shapes.releases(link)
+
+    def resolve(self, parsed, context):
+        """The release this link resolves to, or None.
+
+        A hand-pinned tag prefix wins, then a link naming one exact
+        tag, then the search term. Runs every pass: it is the one call
+        that can report a new release, which is the only thing able to
+        change a previously failed answer.
+        """
+        owner, repository, term, tag = parsed
+        picker = ReleasePicker(context, owner, repository, term)
+        overrides = context.config.get("release_overrides") or {}
+        prefix = overrides.get(context.game.name)
+        if prefix:
+            return picker.by_tag_prefix(prefix)
+        if tag:
+            return picker.by_tag(tag)
+        return picker.pick()
+
+    def fetch(self, parsed, context, release):
+        """This release's apworld, walking back if it ships none.
+
+        Only that one cause is walked back from. "Ships another game's
+        apworld" must not be, since an older release of a multi-world
+        repo is precisely the wrong answer there.
+        """
+        owner, repository = parsed[0], parsed[1]
+        downloader = ReleaseDownloader(
+            context=context,
+            release=release,
+            # What this row's date already says, so a re-upload of
+            # unchanged bytes keeps it instead of moving it forward.
+            previous_published_at=(self.previous or {}).get("published_at"),
+        )
+        downloaded, reason, info = downloader.run()
+        if (downloaded is not None or context.identifier is None
+                or downloader.cause != ReleaseDownloader.NO_APWORLD):
+            return downloaded, reason, info
+        fallback, reason = WalkBack(
+            context=context,
+            owner=owner,
+            repository=repository,
+            release=release,
+        ).run(reason)
+        return fallback if fallback is not None else (None, reason, info)
+
+    def download(self, parsed, context):
+        release = self.resolve(parsed, context)
+        if release is None:
+            return None, "no release matched the search term in the link", None
+        identity = ReleaseIdentity(release)
+        repeat = identity.repeated(self.previous)
+        if repeat is not None:
+            return None, repeat, identity.failure(
+                {"published_at": None}, repeat)
+        downloaded, reason, info = self.fetch(parsed, context, release)
+        if downloaded is None:
+            return None, reason, identity.failure(info, reason)
+        return downloaded, reason, info

src/apworld_tester/pipeline/download/handlers/github_repo.py

@@ -0,0 +1,115 @@
+"""A bare repository link, for a world never cut into a release."""
+
+import os
+
+import requests
+
+from apworld_tester.pipeline.build.archipelago import (
+    ArchipelagoBuild,
+)
+from apworld_tester.pipeline.download.local_file import LocalFile
+from apworld_tester.core.model.apworld import ApworldFile
+from apworld_tester.pipeline.download.handlers.base import LinkHandler
+
+
+class GitHubRepo(LinkHandler):
+    """Browses the repo's own file tree for an apworld."""
+
+    def parse(self, link):
+        return self.shapes.github_repo(link)
+
+    def resolve_branch(self, context, owner, repository, branch):
+        """The real default branch, so a version reads as a name.
+
+        "HEAD" works as a ref, but resolving it means the status page
+        shows a branch rather than the literal string.
+        """
+        if branch != "HEAD":
+            return branch
+        try:
+            response = context.session.get(
+                f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}")
+            response.raise_for_status()
+            return response.json().get("default_branch") or "HEAD"
+        except requests.RequestException:
+            return "HEAD"
+
+    def tree_entries(self, context, owner, repository, branch):
+        """Every apworld in the tree, with its size.
+
+        One call: the trees API takes recursive=1 itself, and returns
+        each blob's size for free.
+        """
+        try:
+            response = context.session.get(
+                f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
+                f"/git/trees/{branch}",
+                params={"recursive": "1"},
+            )
+            response.raise_for_status()
+        except requests.RequestException:
+            return []
+        return [
+            (entry["path"], entry.get("size"))
+            for entry in response.json().get("tree", [])
+            if entry.get("type") == "blob"
+            and entry["path"].lower().endswith(ApworldFile.SUFFIX)
+        ]
+
+    def candidates(self, entries):
+        return [
+            {"name": os.path.basename(path), "path": path, "size": size}
+            for path, size in entries
+        ]
+
+    def download(self, parsed, context):
+        owner, repository, branch = parsed
+        branch = self.resolve_branch(context, owner, repository, branch)
+        entries = self.tree_entries(context, owner, repository, branch)
+        if not entries:
+            return None, ("repo link has no release and no .apworld file "
+                          "found in it"), None
+        chosen = context.matcher.select(self.candidates(entries))
+        downloaded, kept, rejected = self.collect(
+            candidates=chosen,
+            context=context,
+            fetch=lambda candidate: self.fetch(
+                candidate, context, owner, repository, branch),
+            verify=len(chosen) > 1,
+        )
+        if not downloaded:
+            detail = self.rejected_detail(rejected, "file(s) from repo")
+            return None, detail, None
+        published = self.published_at(
+            context, owner, repository, kept[0]["path"], branch)
+        return downloaded, None, {"published_at": published}
+
+    def fetch(self, candidate, context, owner, repository, branch):
+        local = self.file(context, candidate["name"])
+        status = local.status(candidate.get("size"))
+        if status in LocalFile.NEEDS_DOWNLOAD:
+            local.download(
+                f"https://raw.githubusercontent.com/{owner}/{repository}"
+                f"/{branch}/{candidate['path']}")
+        return local.path, status
+
+    def published_at(self, context, owner, repository, path, branch):
+        """When the first kept file was last committed.
+
+        Only the first: one API call per candidate otherwise.
+        """
+        try:
+            response = context.session.get(
+                f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
+                "/commits",
+                params={"path": path, "sha": branch, "per_page": 1},
+            )
+            response.raise_for_status()
+            commits = response.json()
+        except requests.RequestException:
+            return None
+        if not commits:
+            return None
+        commit = commits[0].get("commit", {})
+        return ((commit.get("committer") or {}).get("date")
+                or (commit.get("author") or {}).get("date"))

src/apworld_tester/pipeline/download/handlers/gitlab_file.py

@@ -0,0 +1,44 @@
+"""A link straight at an apworld committed in a GitLab repo."""
+
+import os
+import urllib.parse
+
+import requests
+
+from apworld_tester.pipeline.download.local_file import LocalFile
+from apworld_tester.pipeline.download.handlers.base import LinkHandler
+
+
+class GitLabFile(LinkHandler):
+    """A committed file on GitLab, named exactly by the sheet."""
+
+    def parse(self, link):
+        return self.shapes.gitlab_file(link)
+
+    def download(self, parsed, context):
+        project_path, branch, path = parsed
+        filename = os.path.basename(urllib.parse.unquote(path))
+        local = self.file(context, filename)
+        url = f"https://gitlab.com/{project_path}/-/raw/{branch}/{path}"
+        headers = context.json_headers
+        status = local.status(local.remote_size(url, headers=headers))
+        if status in LocalFile.NEEDS_DOWNLOAD:
+            local.download(url, headers=headers)
+        published = self.published_at(context, project_path, path, branch)
+        return [(local.path, status)], None, {"published_at": published}
+
+    def published_at(self, context, project_path, path, branch):
+        """When this file was last committed, or None."""
+        encoded = urllib.parse.quote(project_path, safe="")
+        try:
+            response = context.session.get(
+                f"https://gitlab.com/api/v4/projects/{encoded}"
+                "/repository/commits",
+                params={"path": path, "ref_name": branch, "per_page": 1},
+                headers=context.json_headers,
+            )
+            response.raise_for_status()
+            commits = response.json()
+        except requests.RequestException:
+            return None
+        return commits[0].get("committed_date") if commits else None

src/apworld_tester/pipeline/download/handlers/gitlab_release.py

@@ -0,0 +1,74 @@
+"""A GitLab releases page."""
+
+import urllib.parse
+
+from apworld_tester.core.model.apworld import ApworldFile
+from apworld_tester.pipeline.download.handlers.base import LinkHandler
+from apworld_tester.pipeline.download.local_file import LocalFile
+
+
+class GitLabRelease(LinkHandler):
+    """The newest release of a GitLab project.
+
+    GitLab's assets are shaped nothing like GitHub's - a release-level
+    list of {name, url, direct_asset_url} with no per-asset timestamp -
+    so they are reshaped here rather than reusing the release path.
+    """
+
+    def parse(self, link):
+        return self.shapes.gitlab_releases(link)
+
+    def latest(self, context, project_path):
+        """GitLab's own newest-release shortcut, or None."""
+        encoded = urllib.parse.quote(project_path, safe="")
+        response = context.session.get(
+            f"https://gitlab.com/api/v4/projects/{encoded}"
+            "/releases/permalink/latest",
+            headers=context.json_headers,
+        )
+        if response.status_code == 404:
+            return None
+        response.raise_for_status()
+        return response.json()
+
+    def assets(self, release, context):
+        """The release's apworld links, in GitHub's asset shape."""
+        links = (release.get("assets") or {}).get("links", [])
+        return [
+            {
+                "name": link["name"],
+                "browser_download_url": (link.get("direct_asset_url")
+                                         or link["url"]),
+            }
+            for link in links
+            if (link.get("name") or "").lower().endswith(ApworldFile.SUFFIX)
+        ]
+
+    def fetch(self, asset, context):
+        local = self.file(context, asset["name"])
+        url = asset["browser_download_url"]
+        headers = context.plain_headers
+        status = local.status(local.remote_size(url, headers=headers))
+        if status in LocalFile.NEEDS_DOWNLOAD:
+            local.download(url, headers=headers)
+        return local.path, status
+
+    def download(self, parsed, context):
+        release = self.latest(context, parsed)
+        if release is None:
+            return None, "no GitLab release found for this project", None
+        info = {"published_at": (release.get("released_at")
+                                 or release.get("created_at"))}
+        assets = self.assets(release, context)
+        if not assets:
+            return None, "latest GitLab release has no .apworld asset", info
+        chosen = context.matcher.select(assets)
+        downloaded, _, rejected = self.collect(
+            candidates=chosen,
+            context=context,
+            fetch=lambda asset: self.fetch(asset, context),
+            verify=len(chosen) > 1,
+        )
+        if not downloaded:
+            return None, self.rejected_detail(rejected, "asset(s)"), info
+        return downloaded, None, info

src/apworld_tester/pipeline/download/handlers/plain_file.py

@@ -0,0 +1,48 @@
+"""A plain URL ending in .apworld, on any host."""
+
+import email.utils
+
+import requests
+
+from apworld_tester.pipeline.download.local_file import LocalFile
+from apworld_tester.pipeline.download.handlers.base import LinkHandler
+
+
+class PlainFile(LinkHandler):
+    """Any other host. Tried last, so a forge handler always wins."""
+
+    def parse(self, link):
+        return self.shapes.plain_file(link)
+
+    def download(self, parsed, context):
+        url, filename = parsed
+        local = self.file(context, filename)
+        size, published = self.remote_facts(context, url)
+        status = local.status(size)
+        if status in LocalFile.NEEDS_DOWNLOAD:
+            local.download(url, headers=context.plain_headers)
+        return [(local.path, status)], None, {"published_at": published}
+
+    def remote_facts(self, context, url):
+        """The size and date one HEAD request can offer.
+
+        A plain URL has no version to compare, so size is the whole
+        staleness check. Neither is required.
+        """
+        try:
+            response = context.session.head(
+                url, headers=context.plain_headers, allow_redirects=True)
+            response.raise_for_status()
+        except requests.RequestException:
+            return None, None
+        length = response.headers.get("Content-Length")
+        size = int(length) if length and length.isdigit() else None
+        return size, self.modified_at(response.headers.get("Last-Modified"))
+
+    def modified_at(self, modified):
+        if not modified:
+            return None
+        try:
+            return email.utils.parsedate_to_datetime(modified).isoformat()
+        except (TypeError, ValueError):
+            return None

src/apworld_tester/pipeline/download/local_file.py

@@ -0,0 +1,83 @@
+"""Deciding whether a local copy is still the current one."""
+
+import datetime
+import os
+
+import requests
+
+from apworld_tester.core.model.instant import Instant
+
+
+class LocalFile:
+    """One file on disk, against what the remote currently offers.
+
+    "It exists" is not "it is current": an author can replace an
+    asset's content in place, same filename, same version. Two signals
+    catch that - the asset's own timestamp, and its size - and both are
+    needed. A replacement that happens to be the same length is
+    invisible to size alone, and a remote that reports no length at all
+    makes size no signal whatsoever.
+    """
+
+    #: The statuses that mean the file has to be fetched. The other
+    #: two - "verified" and "assumed" - mean the copy on disk stands.
+    NEEDS_DOWNLOAD = ("missing", "stale")
+
+    def __init__(self, session, path):
+        self.session = session
+        self.path = path
+
+    def outdated(self, remote_updated_at):
+        """Whether the remote was touched after this copy arrived.
+
+        mtime is when download() wrote this file, so a remote timestamp
+        later than it means the copy on disk predates what is being
+        offered now. Without this, a release that replaced its asset
+        with one of the same length was never fetched, and the recorded
+        test result went on describing content that no longer exists -
+        while the page showed the new release's date beside it.
+        """
+        stamp = Instant.of(remote_updated_at)
+        if stamp is None:
+            return False
+        arrived = datetime.datetime.fromtimestamp(
+            os.path.getmtime(self.path), datetime.timezone.utc)
+        return arrived < stamp
+
+    def status(self, remote_size, remote_updated_at=None):
+        """Whether this file is missing, stale, verified or assumed.
+
+        The timestamp is checked before the size, and before a missing
+        size falls through to "assumed": it is the stronger signal, and
+        the two cases it catches are exactly the ones size cannot.
+        """
+        if not os.path.exists(self.path):
+            return "missing"
+        if self.outdated(remote_updated_at):
+            return "stale"
+        if remote_size is None:
+            return "assumed"
+        if os.path.getsize(self.path) == remote_size:
+            return "verified"
+        return "stale"
+
+    def remote_size(self, url, headers=None):
+        """The size a HEAD request reports, or None."""
+        try:
+            response = self.session.head(url, headers=headers,
+                                         allow_redirects=True)
+            response.raise_for_status()
+        except requests.RequestException:
+            return None
+        length = response.headers.get("Content-Length")
+        return int(length) if length and length.isdigit() else None
+
+    def download(self, url, headers=None):
+        """Stream a URL into this path, creating its directory."""
+        response = self.session.get(url, stream=True, headers=headers)
+        response.raise_for_status()
+        os.makedirs(os.path.dirname(self.path), exist_ok=True)
+        with open(self.path, "wb") as handle:
+            for chunk in response.iter_content(chunk_size=65536):
+                handle.write(chunk)
+        return self.path

src/apworld_tester/pipeline/download/release_downloader.py

@@ -0,0 +1,179 @@
+"""Fetching the apworld a release carries."""
+
+import os
+
+from apworld_tester.core.model.apworld import ApworldFile
+from apworld_tester.core.model.fingerprint import Fingerprint
+from apworld_tester.pipeline.download.local_file import LocalFile
+from apworld_tester.pipeline.download.zip_search import ZipSearch
+
+
+class ReleaseDownloader:
+    """A GitHub-shaped release, and the apworld it ships.
+
+    Shared with Gitea and Forgejo, whose release JSON is close enough
+    that the same logic applies unchanged.
+    """
+
+    #: Why run() came back empty. The caller decides what to do about
+    #: it from this, never from the message beside it: a release that
+    #: ships no apworld can be walked back from, and one that ships
+    #: another game's must not be.
+    NO_APWORLD = "no_apworld"
+    WRONG_GAME = "wrong_game"
+
+    NO_APWORLD_DETAIL = "latest release has no .apworld asset"
+
+    def __init__(self, context, release, headers=None,
+                 require_identity=False, previous_published_at=None):
+        self.context = context
+        self.release = release
+        self.headers = headers
+        self.require_identity = require_identity
+        self.previous_published_at = previous_published_at
+        self.cause = None
+
+    def named_assets(self):
+        return [
+            asset for asset in self.release.get("assets", [])
+            if asset["name"].lower().endswith(ApworldFile.SUFFIX)
+        ]
+
+    def missing_detail(self):
+        detail = self.NO_APWORLD_DETAIL
+        if any(asset["name"].lower().endswith(ZipSearch.SUFFIX)
+               for asset in self.release.get("assets", [])):
+            detail += " (checked inside its .zip asset(s) too, none found)"
+        return detail
+
+    def assets(self):
+        """Every apworld this release offers, zips included."""
+        found = self.named_assets()
+        if found:
+            return found
+        return ZipSearch(self.context, self.release, self.headers).assets()
+
+    def verifies(self, assets):
+        """Whether a file has to prove which game it is.
+
+        With one candidate there is nothing to choose between, and the
+        sheet's link points at this release - so that file is the row's
+        apworld whatever its World class calls itself. A walk-back
+        release overrides that, since the link no longer points at it.
+        """
+        return len(assets) > 1 or self.require_identity
+
+    @staticmethod
+    def touched_at(asset):
+        """When this asset was last written, as the remote reports it.
+
+        The same value published_at() reports for the row, read here so
+        the two cannot disagree: a file whose recorded release date is
+        newer than the copy on disk is a file that was never fetched.
+        """
+        return asset.get("updated_at") or asset.get("created_at")
+
+    def fetch(self, asset):
+        """The asset's path, how it got there, and whether it changed.
+
+        An asset whose timestamp moved is fetched again - the copy on
+        disk can no longer be trusted to be current - and only then can
+        the two be compared. Identical bytes mean the author re-uploaded
+        the same file, so nothing about this row actually changed. A
+        file lifted out of a zip is taken as changed: ZipSearch does its
+        own fetching, and there is no earlier copy here to compare to.
+        """
+        if "extracted_path" in asset:
+            return asset["extracted_path"], asset["status"], True
+        local = LocalFile(self.context.session,
+                          self.context.target_path(asset["name"]))
+        url = asset["browser_download_url"]
+        status = local.status(
+            local.remote_size(url, headers=self.headers),
+            remote_updated_at=self.touched_at(asset),
+        )
+        if status not in LocalFile.NEEDS_DOWNLOAD:
+            return local.path, status, False
+        before = (Fingerprint.of_file(local.path)
+                  if os.path.exists(local.path) else None)
+        local.download(url, headers=self.headers)
+        return local.path, status, Fingerprint.of_file(local.path) != before
+
+    def rejection(self, path, assets):
+        """Why this file is not ours, or None to keep it.
+
+        A walk-back file that registered no world at all is kept:
+        nothing was proven either way, and generation testing reports
+        the import failure far more usefully than a silent drop.
+        """
+        identity = self.context.identifier.identify(path)
+        if not identity.get("ran"):
+            return None
+        found = identity.get("games") or []
+        if self.require_identity and not found and len(assets) == 1:
+            return None
+        other = self.context.matcher.identity_claimed(
+            found, self.context.sheet_names)
+        matches = bool(found) and self.context.matcher.game.matches(
+            found[0])
+        if other:
+            return f'{found} - belongs to the "{other}" row'
+        if self.verifies(assets) and not matches:
+            return found or identity.get("detail")
+        return None
+
+    def checks(self, status, assets):
+        return (status in LocalFile.NEEDS_DOWNLOAD
+                and self.context.identifier is not None
+                and (self.verifies(assets) or self.context.sheet_names))
+
+    def published_at(self, kept):
+        """When the content this row now holds was published.
+
+        Each asset carries its own timestamp; a release's own stays
+        fixed at creation even when its files are replaced later, so
+        the assets' is the one to read. But that timestamp moves for a
+        re-upload of identical bytes too, and then it dates the upload
+        rather than the world: the page would show a release newer than
+        the test beside it while the tested file never changed, and the
+        Compatibility column would call a world current on the strength
+        of an upload that added nothing. So only an asset whose content
+        actually changed sets this date, and otherwise the one already
+        recorded stands. With nothing recorded there is nothing to
+        hold, and the asset's own timestamp is the best available.
+        """
+        stamps = [(self.touched_at(asset), changed)
+                  for asset, changed in kept]
+        stamps = [pair for pair in stamps if pair[0]]
+        replaced = [stamp for stamp, changed in stamps if changed]
+        if replaced:
+            return max(replaced)
+        if self.previous_published_at:
+            return self.previous_published_at
+        return max((stamp for stamp, _ in stamps), default=None)
+
+    def run(self):
+        """The files downloaded, why none were, and the release info."""
+        assets = self.assets()
+        if not assets:
+            self.cause = self.NO_APWORLD
+            return None, self.missing_detail(), {"published_at": None}
+        assets = self.context.matcher.select(assets)
+        downloaded, kept, rejected = [], [], []
+        for asset in assets:
+            path, status, changed = self.fetch(asset)
+            reason = (self.rejection(path, assets)
+                      if self.checks(status, assets) else None)
+            if reason is not None:
+                os.remove(path)
+                rejected.append((asset["name"], reason))
+                continue
+            downloaded.append((path, status))
+            kept.append((asset, changed))
+        if not downloaded:
+            self.cause = self.WRONG_GAME
+            details = "; ".join(f"{name} -> {found}"
+                                for name, found in rejected)
+            return None, (f"downloaded asset(s) didn't match this game: "
+                          f"{details}"), {"published_at": None}
+        return downloaded, None, {"published_at": self.published_at(kept)}

src/apworld_tester/pipeline/download/release_identity.py

@@ -0,0 +1,55 @@
+"""Remembering which release a failure was decided on."""
+
+from apworld_tester.core.state.keys import StateKeys
+
+
+class ReleaseIdentity:
+    """What makes a resolved release the same one as last run.
+
+    Its tag and publication date, plus every asset's name and size. The
+    sizes matter because an author can replace an asset in place
+    without cutting a new tag.
+    """
+
+    def __init__(self, release):
+        self.release = release
+
+    @property
+    def value(self):
+        assets = sorted(
+            (asset.get("name"), asset.get("size"))
+            for asset in self.release.get("assets") or []
+        )
+        return {
+            "tag": self.release.get("tag_name"),
+            "published_at": self.release.get("published_at"),
+            "assets": [{"name": name, "size": size} for name, size in assets],
+        }
+
+    def repeated(self, previous):
+        """The recorded reason, if this exact release failed before.
+
+        A release that ships no apworld, or only another row's, fails
+        the same way every run - and finding that out costs a download
+        of every zip asset plus a container run to identify what came
+        back. None of that can reach a different answer while the
+        release is unchanged. Keyed on the release rather than the
+        link, because the link is usually /releases/latest and a newly
+        published release is exactly what would fix a world in this
+        state. Only ever holds a verdict about content: a network error
+        escapes as an exception and a container that could not start
+        keeps the file, so neither is cached here.
+        """
+        if not previous:
+            return None
+        if previous.get(StateKeys.FAILED_RELEASE) != self.value:
+            return None
+        return previous.get(StateKeys.FAILED_REASON) or None
+
+    def failure(self, info, reason):
+        """A failure recorded against the release it was decided on."""
+        return {
+            **(info or {}),
+            StateKeys.FAILED_RELEASE: self.value,
+            StateKeys.FAILED_REASON: reason,
+        }

src/apworld_tester/pipeline/download/release_picker.py

@@ -0,0 +1,151 @@
+"""Which release a sheet link means."""
+
+import re
+import urllib.parse
+
+from apworld_tester.pipeline.build.archipelago import (
+    ArchipelagoBuild,
+)
+from apworld_tester.core.model.apworld import ApworldFile
+from apworld_tester.core.model.name import Name
+
+
+class ReleasePicker:
+    """One repository's releases, narrowed to this game's.
+
+    A repo dedicated to one world can answer with its latest release. A
+    repo hosting several has to be matched on the link's search term and
+    the release's own title, and must never answer with a release that
+    another sheet row already claims.
+    """
+
+    def __init__(self, context, owner, repository, term=None):
+        self.context = context
+        self.owner = owner
+        self.repository = repository
+        self.term = term
+        self.matcher = context.matcher
+
+    @property
+    def repo_url(self):
+        return (f"{ArchipelagoBuild.API_ROOT}/repos/"
+                f"{self.owner}/{self.repository}")
+
+    def all(self, max_pages=10):
+        """Every release, newest first, across every page.
+
+        GitHub pages at 30 by default and several source repos hold
+        more than that, which would silently hide real releases from
+        every lookup below.
+        """
+        releases = []
+        for page in range(1, max_pages + 1):
+            response = self.context.session.get(
+                f"{self.repo_url}/releases",
+                params={"per_page": 100, "page": page},
+            )
+            response.raise_for_status()
+            batch = response.json()
+            releases.extend(batch)
+            if len(batch) < 100:
+                break
+        return releases
+
+    def by_tag(self, tag):
+        """The release for exactly this tag, or None."""
+        encoded = urllib.parse.quote(tag, safe="")
+        response = self.context.session.get(
+            f"{self.repo_url}/releases/tags/{encoded}")
+        if response.status_code == 404:
+            return None
+        response.raise_for_status()
+        return response.json()
+
+    def by_tag_prefix(self, prefix):
+        """The newest release whose tag starts with this prefix."""
+        for release in self.all():
+            if (release.get("tag_name") or "").startswith(prefix):
+                return release
+        return None
+
+    def claimed(self, release):
+        return self.matcher.release_claimed(release, self.context.sheet_names)
+
+    def newest(self):
+        """GitHub's own newest full release.
+
+        Right whenever it resolves, since it skips in-progress
+        prereleases. The list's first entry is the fallback for a repo
+        where every release is flagged prerelease, which /latest
+        excludes rather than relaxing.
+        """
+        response = self.context.session.get(f"{self.repo_url}/releases/latest")
+        if response.status_code != 404:
+            response.raise_for_status()
+            return response.json()
+        releases = self.all()
+        return releases[0] if releases else None
+
+    def titled_for_this_row(self):
+        for release in self.all():
+            if self.matcher.title_slug(release) == self.matcher.game.slug:
+                return release
+        return None
+
+    def without_a_term(self):
+        """The release to use when the link named none.
+
+        "Newest in the repo" is a fine answer for a repo holding one
+        world and a coin flip for one holding twelve, so a newest that
+        is titled for somebody else's row gives way to the one titled
+        for this row.
+        """
+        latest = self.newest()
+        if latest is None or not self.claimed(latest):
+            return latest
+        return self.titled_for_this_row() or latest
+
+    def by_title(self, releases):
+        """The first release whose title holds every word of the term.
+
+        The sheet's term is built for GitHub's own search box, which
+        matches per word: requiring the whole phrase rejects real terms
+        like "Jurassic Park Randomizer (SNES) Jurassic Park", where the
+        link generator repeats the game name. That looseness is what
+        lets a term land on a sibling world, so a release titled for
+        another row is skipped and the search continues.
+        """
+        words = re.findall(r"[a-z0-9]+", self.term.lower())
+        if not words:
+            return None
+        for release in releases:
+            title = (f"{release.get('name') or ''} "
+                     f"{release.get('tag_name') or ''}").lower()
+            if all(word in title for word in words) \
+                    and not self.claimed(release):
+                return release
+        return None
+
+    def by_asset_name(self, releases):
+        """The release carrying an asset named for this game.
+
+        A fork mirroring many worlds often tags every release by build
+        date alone, so the game's name appears only in which apworlds
+        the release bundles.
+        """
+        term = Name(self.term).slug
+        for release in releases:
+            for asset in release.get("assets", []):
+                name = asset["name"]
+                if not name.lower().endswith(ApworldFile.SUFFIX):
+                    continue
+                if Name(name.rsplit(".", 1)[0]).slug == term:
+                    return release
+        return None
+
+    def pick(self):
+        """The release this link means, or None."""
+        if self.term is None:
+            return self.without_a_term()
+        releases = self.all()
+        return self.by_title(releases) or self.by_asset_name(releases)

src/apworld_tester/pipeline/download/superseded_assets.py

@@ -0,0 +1,58 @@
+"""Clearing apworlds a game's release no longer offers."""
+
+import os
+
+
+class SupersededAssets:
+    """The files left behind when an author renames their asset.
+
+    A download writes each asset under its own filename, so a release
+    that renamed one leaves the previous name sitting beside it. Both
+    are then real files in the game's folder, and everything downstream
+    keys on the path rather than the game - two work items, two test
+    records, two rows on the status page, and a promotion that refuses
+    both because they claim one game name between them.
+
+    Only ever run for a game whose download succeeded: with nothing
+    fetched there is no current set to compare against, and pruning on
+    a failed network call would delete the copy that still works.
+    """
+
+    #: What this class considers. A game's folder holds nothing else,
+    #: but naming it here keeps a stray file from being deleted on the
+    #: strength of living in the wrong directory.
+    SUFFIX = ".apworld"
+
+    def __init__(self, directory, kept):
+        self.directory = directory
+        self.kept = {os.path.abspath(path) for path in kept}
+
+    def stale(self):
+        """Every apworld in the folder that this download did not write."""
+        if not self.kept or not os.path.isdir(self.directory):
+            return []
+        found = []
+        for entry in sorted(os.listdir(self.directory)):
+            if not entry.lower().endswith(self.SUFFIX):
+                continue
+            path = os.path.join(self.directory, entry)
+            if os.path.isfile(path) and os.path.abspath(path) not in self.kept:
+                found.append(path)
+        return found
+
+    def prune(self):
+        """Delete them, returning what went.
+
+        A file that cannot be removed is not worth failing a download
+        over - it means the next run sees a duplicate, which is what
+        was already happening.
+        """
+        removed = []
+        for path in self.stale():
+            try:
+                os.remove(path)
+            except OSError as error:
+                print(f"could not remove superseded {path}: {error}")
+                continue
+            removed.append(path)
+        return removed

src/apworld_tester/pipeline/download/unusable_links.py

@@ -0,0 +1,56 @@
+"""Why none of a game's links could be downloaded from."""
+
+import urllib.parse
+
+
+class UnusableLinks:
+    """Categorise links no handler recognised.
+
+    Reaching here means every recognised shape was tried against every
+    link, so the status page can say something concrete instead of a
+    generic "failed".
+    """
+
+    #: Hosts that can never serve a downloadable file, whatever handler
+    #: is written for them: a Discord link is a channel or message, not
+    #: an asset. Not configuration - no setting makes Discord fetchable.
+    UNREACHABLE = {
+        "discord.com": "Discord link only (channel/message link, not a "
+                       "downloadable file) - can't be fetched automatically",
+        "discordapp.com": "Discord link only (channel/message link, not a "
+                          "downloadable file) - can't be fetched "
+                          "automatically",
+    }
+
+    def __init__(self, config, links):
+        self.config = config
+        self.links = links
+
+    def domains(self):
+        """Each link's host, or a reason it has none."""
+        found = []
+        for link in self.links:
+            parsed = urllib.parse.urlparse(link)
+            if parsed.scheme not in ("http", "https"):
+                protocol = parsed.scheme or "(none)"
+                return None, f"unsupported link protocol {protocol!r}: {link}"
+            if not parsed.netloc:
+                return None, f"not a valid URL: {link}"
+            netloc = parsed.netloc.lower()
+            found.append(netloc[4:] if netloc.startswith("www.") else netloc)
+        return found, None
+
+    def detail(self):
+        domains, problem = self.domains()
+        if problem is not None:
+            return problem
+        for domain in domains:
+            if domain in self.UNREACHABLE:
+                return self.UNREACHABLE[domain]
+        if "github.com" in domains:
+            return ("GitHub link found, but not a recognized "
+                    "releases/repo/file URL")
+        if "gitlab.com" in domains:
+            return "GitLab link found, but not a recognized releases/blob URL"
+        hosts = ", ".join(sorted(set(domains)))
+        return f"no supported download source (custom host: {hosts})"

src/apworld_tester/pipeline/download/update_history.py

@@ -0,0 +1,168 @@
+"""Every distinct apworld file this pipeline has seen, per game.
+
+One file per game, written only when that game gains content it has not
+seen before - so a run touches the handful of games that actually
+changed rather than rewriting one shared blob. Deliberately NOT under
+downloads/: that directory is a cache and gets cleared to force a clean
+re-download, which is harmless for everything else in it. History is the
+one thing that cannot be rebuilt afterwards - which version existed on
+which date has no other source once it is gone.
+"""
+
+import os
+from datetime import datetime, timezone
+
+from apworld_tester.core.model.apworld import ApworldFile
+from apworld_tester.core.model.fingerprint import Fingerprint
+from apworld_tester.core.model.name import Name
+from apworld_tester.core.state.keys import StateKeys
+from apworld_tester.core.state.store import StateStore
+
+
+class UpdateHistory:
+    """What each game's apworld was, and when.
+
+    Identity is the file's sha256, so a re-download of unchanged content
+    adds nothing and recording is safe to repeat within a run. The
+    game's sheet name is stored inside the file as well as encoded in
+    its name, because the filename is a folded directory name and a
+    sheet rename would otherwise leave a file nothing could identify.
+    """
+
+    def __init__(self, paths, directory=None):
+        self.directory = directory or paths.history
+
+    def path_for(self, game_name):
+        folded = Name(game_name).directory
+        return os.path.join(self.directory, f"{folded}.json")
+
+    def load(self, game_name):
+        return StateStore.read(
+            self.path_for(game_name),
+            {"game": game_name, "watching_since": None,
+             "baseline_at": None, "files": []})
+
+    def save(self, game_name, history):
+        os.makedirs(self.directory, exist_ok=True)
+        StateStore(self.path_for(game_name)).save(history)
+
+    #: Everything recorded before this field existed was gathered from
+    #: whatever the hosts happened to say, at whatever moment the file
+    #: was first downloaded, and none of it measures how often a world
+    #: changes under observation. The first pass to touch a game draws
+    #: a line: that instant is the game's one starting point, and only
+    #: changes seen after it count. See UpdateCadence, which reads it.
+    @staticmethod
+    def watching_since(history, stamp):
+        """When this pipeline began watching this game.
+
+        The earliest content it has already recorded, so a file written
+        before this field existed still gets an honest answer rather
+        than a window that starts the day the field was added. Only a
+        game with no history at all starts watching now.
+        """
+        seen = [item.get("first_seen_at") for item in history["files"]
+                if item.get("first_seen_at")]
+        return min(seen) if seen else stamp
+
+    def entry_for(self, apworld_path, sha256, released_at, source_url, now):
+        """One file's record.
+
+        `released_at` is the upload date the download stage saw;
+        first_seen_at is when this pipeline observed it, which is the
+        only honest timestamp for a file whose host reports no date.
+        """
+        manifest = ApworldFile(apworld_path).manifest or {}
+        return {
+            "version": manifest.get("world_version"),
+            "released_at": released_at,
+            "first_seen_at": (now or datetime.now(timezone.utc)).isoformat(),
+            "sha256": sha256,
+            "filename": os.path.basename(apworld_path),
+            "source_url": source_url,
+        }
+
+    def record_file(self, game_name, apworld_path, sha256, released_at,
+                    source_url=None, now=None):
+        """Add this file to the game's history if its content is new.
+
+        A game whose content has not changed still records that it was
+        looked at. How often a world updates is only meaningful against
+        how long it has been watched, and without that a game seen once
+        and a game seen all year are indistinguishable.
+        """
+        history = self.load(game_name)
+        now = now or datetime.now(timezone.utc)
+        opened = not history.get("watching_since")
+        if opened:
+            history["watching_since"] = self.watching_since(
+                history, now.isoformat())
+        if not history.get("baseline_at"):
+            opened = True
+            history["baseline_at"] = now.isoformat()
+        if any(item.get("sha256") == sha256 for item in history["files"]):
+            if opened:
+                self.save(game_name, history)
+            return None
+        entry = self.entry_for(apworld_path, sha256, released_at,
+                               source_url, now)
+        history["game"] = game_name
+        history["files"].append(entry)
+        # Oldest first. released_at is missing often enough that it
+        # cannot be the only key - first_seen_at always exists and
+        # breaks those ties in the order the pipeline saw them.
+        history["files"].sort(
+            key=lambda item: (item.get("released_at") or "",
+                              item.get("first_seen_at") or ""))
+        self.save(game_name, history)
+        return entry
+
+    def files_in(self, game_directory):
+        return [
+            name for name in sorted(os.listdir(game_directory))
+            if name.lower().endswith(ApworldFile.SUFFIX)
+            and os.path.isfile(os.path.join(game_directory, name))
+        ]
+
+    def record_all(self, state, root_directory, now=None):
+        """Record every apworld on disk that is not yet in history.
+
+        Hashes the files directly rather than reading apworld_tests:
+        that dict only gains a hash once a world has been TESTED, so a
+        file downloaded this run would not appear until the next one -
+        and this runs right after downloading, which is the point at
+        which a new version is known.
+        """
+        releases = state.get(StateKeys.GAME_RELEASES) or {}
+        # game_releases is keyed by the sheet name while the download
+        # folder is that name folded - map one to the other rather than
+        # assuming they match as written.
+        by_folder = {Name(name).directory: info
+                     for name, info in releases.items()}
+        added = []
+        if not os.path.isdir(root_directory):
+            return added
+        for folder in sorted(os.listdir(root_directory)):
+            game_directory = os.path.join(root_directory, folder)
+            if not os.path.isdir(game_directory):
+                continue
+            added += self.record_folder(
+                folder, game_directory, by_folder.get(folder) or {}, now)
+        return added
+
+    def record_folder(self, folder, game_directory, release, now):
+        """Every new apworld in one game's download folder."""
+        added = []
+        for filename in self.files_in(game_directory):
+            path = os.path.join(game_directory, filename)
+            entry = self.record_file(
+                game_name=folder,
+                apworld_path=path,
+                sha256=Fingerprint.of_file(path),
+                released_at=release.get("published_at"),
+                source_url=release.get("url"),
+                now=now,
+            )
+            if entry:
+                added.append((folder, entry))
+        return added

src/apworld_tester/pipeline/download/url_shapes.py

@@ -0,0 +1,98 @@
+"""Recognising the shapes a sheet link can take."""
+
+import os
+import re
+import urllib.parse
+
+from apworld_tester.core.model.apworld import ApworldFile
+
+
+class UrlShapes:
+    """The link shapes this pipeline knows how to fetch from."""
+
+    def __init__(self, config):
+        self.config = config
+
+    def releases(self, url):
+        """Owner, repo, search term and tag from a releases link.
+
+        A link naming one exact tag is read as such: without it, a repo
+        hosting several worlds would fall back to "latest", which drifts
+        to whichever world was tagged most recently.
+        """
+        match = re.match(
+            r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+)/releases",
+            url or "", re.IGNORECASE)
+        if match is None:
+            return None
+        parsed = urllib.parse.urlparse(url)
+        terms = urllib.parse.parse_qs(parsed.query).get("q", [])
+        tag_match = re.match(
+            r"^/[^/]+/[^/]+/releases/tag/([^/?#]+)", parsed.path,
+            re.IGNORECASE)
+        return (
+            match.group(1),
+            match.group(2),
+            terms[0].strip('"') if terms else None,
+            urllib.parse.unquote(tag_match.group(1)) if tag_match else None,
+        )
+
+    def github_file(self, url):
+        """Owner, repo, branch and path from a committed-file link."""
+        match = re.match(
+            r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+)"
+            r"/(?:blob|raw)/([^/]+)/(.+\.apworld)$",
+            url or "", re.IGNORECASE)
+        return match.groups() if match else None
+
+    def github_repo(self, url):
+        """Owner, repo and branch from a bare repository link.
+
+        Deliberately strict, so it does not swallow issues or wiki
+        links: those should keep falling through to the generic "not a
+        releases URL" message rather than triggering a tree search that
+        could never succeed.
+        """
+        match = re.match(
+            r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+?)"
+            r"(?:/tree/([^/?#]+))?/?$",
+            url or "", re.IGNORECASE)
+        if match is None:
+            return None
+        owner, repository, branch = match.groups()
+        return owner, repository, branch or "HEAD"
+
+    def gitlab_file(self, url):
+        """Project path, branch and file path from a GitLab blob link."""
+        match = re.match(
+            r"^https?://(?:www\.)?gitlab\.com/([^/]+(?:/[^/]+)*?)"
+            r"/-/blob/([^/]+)/(.+\.apworld)$",
+            url or "", re.IGNORECASE)
+        return match.groups() if match else None
+
+    def gitlab_releases(self, url):
+        """The project path from a GitLab releases link."""
+        match = re.match(
+            r"^https?://(?:www\.)?gitlab\.com/([^/]+(?:/[^/]+)*?)"
+            r"/-/releases(?:[/?#]|$)",
+            url or "", re.IGNORECASE)
+        return match.group(1) if match else None
+
+    def gitea_releases(self, url):
+        """Host, owner and repo from a Gitea-style releases link."""
+        match = re.match(
+            r"^https?://(?!(?:www\.)?(?:github|gitlab)\.com)"
+            r"([^/]+)/([^/]+)/([^/]+)/releases(?:[/?#]|$)",
+            url or "", re.IGNORECASE)
+        return match.groups() if match else None
+
+    def plain_file(self, url):
+        """A URL ending in .apworld on any host, and its filename."""
+        parsed = urllib.parse.urlparse(url or "")
+        if parsed.scheme not in ("http", "https") or not parsed.netloc:
+            return None
+        path = urllib.parse.unquote(parsed.path)
+        if not path.lower().endswith(ApworldFile.SUFFIX):
+            return None
+        filename = os.path.basename(path)
+        return (url, filename) if filename != ApworldFile.SUFFIX else None

src/apworld_tester/pipeline/download/walk_back.py

@@ -0,0 +1,77 @@
+"""Falling back to an older release that still ships an apworld."""
+
+import requests
+
+from apworld_tester.core.model.apworld import ApworldFile
+from apworld_tester.pipeline.download.release_downloader import (
+    ReleaseDownloader,
+)
+from apworld_tester.pipeline.download.release_picker import ReleasePicker
+
+
+class WalkBack:
+    """One narrow rescue: the newest release forgot the apworld.
+
+    An author cuts a release and re-attaches only a mod zip, leaving
+    the world perfectly downloadable one release back. Every other
+    failure is left alone on purpose - in particular "did not match
+    this game", where an older release holding a different game's
+    apworld is precisely the wrong answer.
+    """
+
+    def __init__(self, context, owner, repository, release):
+        self.context = context
+        self.owner = owner
+        self.repository = repository
+        self.release = release
+
+    def candidate(self, release):
+        """Whether this is an older release that ships an apworld.
+
+        Matches on asset filename alone: this runs over a repo's whole
+        history, and opening every zip in it would cost far more than
+        the problem is worth.
+        """
+        cutoff = self.release.get("published_at")
+        published_at = release.get("published_at")
+        return bool(
+            not release.get("draft")
+            and release.get("tag_name") != self.release.get("tag_name")
+            and published_at
+            and (not cutoff or published_at < cutoff)
+            and any(asset["name"].lower().endswith(ApworldFile.SUFFIX)
+                    for asset in release.get("assets", []))
+        )
+
+    def older(self):
+        """The newest earlier release with an apworld, by date.
+
+        Ordering comes from published_at rather than the order GitHub
+        returns, which follows the underlying tag and is not the same
+        thing.
+        """
+        picker = ReleasePicker(self.context, self.owner, self.repository)
+        try:
+            releases = picker.all()
+        except requests.RequestException:
+            return None
+        candidates = [r for r in releases if self.candidate(r)]
+        if not candidates:
+            return None
+        return max(candidates, key=lambda release: release["published_at"])
+
+    def run(self, reason):
+        """The fallback's result, or None and a combined reason."""
+        older = self.older()
+        if older is None:
+            return None, reason
+        fallback = ReleaseDownloader(
+            context=self.context,
+            release=older,
+            require_identity=True,
+        ).run()
+        if fallback[0] is not None:
+            fallback[2]["fallback_release_tag"] = older.get("tag_name")
+            return fallback, None
+        return None, (f"{reason}; older release {older.get('tag_name')!r} "
+                      f"was tried too: {fallback[1]}")

src/apworld_tester/pipeline/download/zip_search.py

@@ -0,0 +1,96 @@
+"""Looking for an apworld inside a release's zip assets."""
+
+import os
+import tempfile
+import zipfile
+
+import requests
+
+from apworld_tester.core.model.apworld import ApworldFile
+from apworld_tester.pipeline.download.local_file import LocalFile
+
+
+class ZipSearch:
+    """For a release that publishes a zip instead of an apworld.
+
+    Some repos attach a source snapshot with the built apworld nested
+    inside. Every zip is re-downloaded to scratch space each time, since
+    the member's own filename - what a normal download caches on - is
+    not known until the zip is opened.
+    """
+
+    #: The extension a release's archive asset is recognised by.
+    SUFFIX = ".zip"
+
+    def __init__(self, context, release, headers=None):
+        self.context = context
+        self.release = release
+        self.headers = headers
+
+    def zip_assets(self):
+        return [
+            asset for asset in self.release.get("assets", [])
+            if asset["name"].lower().endswith(ZipSearch.SUFFIX)
+        ]
+
+    def members(self, zip_path):
+        """Every apworld inside the zip, with its size."""
+        try:
+            with zipfile.ZipFile(zip_path) as archive:
+                return [
+                    (info.filename, info.file_size)
+                    for info in archive.infolist()
+                    if not info.filename.endswith("/")
+                    and info.filename.lower().endswith(ApworldFile.SUFFIX)
+                ]
+        except zipfile.BadZipFile:
+            return []
+
+    def extract(self, zip_path, member):
+        """Unpack one member into the game's folder."""
+        target = self.context.target_path(os.path.basename(member))
+        os.makedirs(os.path.dirname(target), exist_ok=True)
+        with zipfile.ZipFile(zip_path) as archive:
+            with archive.open(member) as source:
+                with open(target, "wb") as handle:
+                    handle.write(source.read())
+        return target
+
+    def unpack(self, zip_path, members, zip_asset):
+        """The members as assets, already on disk."""
+        results = []
+        for member, size in members:
+            target = self.context.target_path(os.path.basename(member))
+            status = LocalFile(self.context.session, target).status(size)
+            if status in LocalFile.NEEDS_DOWNLOAD:
+                self.extract(zip_path, member)
+            results.append({
+                "name": os.path.basename(member),
+                "updated_at": zip_asset.get("updated_at"),
+                "created_at": zip_asset.get("created_at"),
+                "extracted_path": target,
+                "status": status,
+            })
+        return results
+
+    def search_one(self, zip_asset):
+        with tempfile.TemporaryDirectory() as scratch:
+            zip_path = os.path.join(scratch, zip_asset["name"])
+            local = LocalFile(self.context.session, zip_path)
+            try:
+                local.download(zip_asset["browser_download_url"],
+                               headers=self.headers)
+            except requests.RequestException:
+                return []
+            members = self.members(zip_path)
+            if not members:
+                return []
+            return self.unpack(zip_path, members, zip_asset)
+
+    def assets(self):
+        """The first zip that holds an apworld wins."""
+        for zip_asset in self.zip_assets():
+            found = self.search_one(zip_asset)
+            if found:
+                return found
+        return []

src/apworld_tester/pipeline/generation/command.py

@@ -0,0 +1,99 @@
+"""Building the docker invocations the drivers run in."""
+
+import os
+
+from apworld_tester.core.model.apworld import ApworldFile
+
+
+class ContainerCommand:
+    """The `docker run` for one driver, and the mounts it needs.
+
+    The paths below are where this pipeline mounts things inside its
+    own container. Both ends of that contract live in this repository -
+    these mount arguments and the drivers that receive them - so they
+    are part of the code rather than something to configure.
+    """
+
+    #: Where each driver script is mounted.
+    DRIVER_PATH = "/app/run_test.py"
+    IDENTIFY_DRIVER_PATH = "/app/identify_apworld.py"
+    VERIFY_DRIVER_PATH = "/app/verify_companions.py"
+
+    #: Where the apworld under test, the seeds and the ROMs go.
+    APWORLD_DIRECTORY = "/app/custom_worlds"
+    OUTPUT_DIRECTORY = "/data/output"
+    DATA_DIRECTORY = "/app/data"
+
+    #: A writable cache directory. The image runs as uid 1000 but
+    #: leaves HOME as "/", which is root-owned, so anything resolving
+    #: "~/.cache" fails with PermissionError before generation starts.
+    CACHE_ENVIRONMENT = {"HOME": "/tmp", "XDG_CACHE_HOME": "/tmp/.cache"}
+
+    def __init__(self, config, paths, image):
+        self.config = config
+        self.paths = paths
+        self.image = image
+
+    @property
+    def prefix(self):
+        """The part of every invocation that never varies."""
+        return ["docker", "run", "--rm", "--network", "none",
+                "--entrypoint", "python", *self.cache_environment]
+
+    @property
+    def cache_environment(self):
+        args = []
+        for name, value in self.CACHE_ENVIRONMENT.items():
+            args += ["-e", f"{name}={value}"]
+        return args
+
+    @property
+    def common_client_mount(self):
+        """CommonClient.py, mounted back in from the checkout.
+
+        The image excludes every client file, correctly - but a few
+        apworlds import their own during generation and fail with "No
+        module named 'CommonClient'" purely because it is absent.
+        Absent source means no mount: it exists only after a clone.
+        """
+        if not os.path.isfile(self.paths.common_client):
+            return []
+        return ["-v", f"{self.paths.common_client}:/app/CommonClient.py:ro"]
+
+    @property
+    def rom_mounts(self):
+        """Every base ROM, mounted individually.
+
+        Per file rather than mounting the directory over /app/data,
+        which would shadow the image's own shipped data. Worlds declare
+        rom_file inconsistently - some embed "data/", some do not - so
+        each is mounted at both locations.
+        """
+        directory = self.paths.roms
+        if not os.path.isdir(directory):
+            return []
+        data_in = self.DATA_DIRECTORY
+        args = []
+        for name in sorted(os.listdir(directory)):
+            path = os.path.join(directory, name)
+            if os.path.isfile(path):
+                args += ["-v", f"{path}:{data_in}/{name}:ro"]
+                args += ["-v", f"{path}:/app/{name}:ro"]
+        return args
+
+    def apworld_mount(self, apworld_path):
+        """The mount and container path for the world under test."""
+        if not apworld_path:
+            return [], None
+        name = ApworldFile(apworld_path).importable_name
+        inside = f"{self.APWORLD_DIRECTORY}/{name}"
+        return ["-v", f"{apworld_path}:{inside}:ro"], inside
+
+    def driver_mounts(self, output_dir):
+        """The driver, the output directory, the ROMs and the client."""
+        return [
+            "-v", f"{self.paths.driver}:{self.DRIVER_PATH}:ro",
+            "-v", f"{output_dir}:{self.OUTPUT_DIRECTORY}",
+            *self.rom_mounts,
+            *self.common_client_mount,
+        ]

src/apworld_tester/pipeline/generation/companion_check.py

@@ -0,0 +1,83 @@
+"""Which core worlds can actually generate in this container."""
+
+import os
+import subprocess
+
+from apworld_tester.core.config.run_settings import RunSettings
+from apworld_tester.pipeline.generation.command import ContainerCommand
+from apworld_tester.pipeline.generation.report import DriverReport
+
+
+class CompanionCheck:
+    """Tries every core world once, so the rest can trust the pool.
+
+    Several core worlds need a base ROM this container has no copy of,
+    and drawing one fails the seed - with the failure landing on the
+    apworld under test rather than on the companion that caused it.
+    """
+
+    #: The whole verification's ceiling, and each world's share of it.
+    #: Not configuration: the per-world limit is deliberately shorter
+    #: than a generation timeout, because a companion slow to generate
+    #: would multiply that cost across every seed that draws it.
+    TIMEOUT = 1800
+    PER_WORLD_TIMEOUT = 120
+
+    def __init__(self, config, paths, image):
+        self.config = config
+        self.paths = paths
+        self.image = image
+        self.command = ContainerCommand(config, paths, image)
+
+    def build(self, output_dir, per_world_timeout, spoiler):
+        verify_in = ContainerCommand.VERIFY_DRIVER_PATH
+        return [
+            *self.command.prefix,
+            "-v", f"{self.paths.verify_driver}:{verify_in}:ro",
+            "-v", (f"{self.paths.driver}:"
+                   f"{ContainerCommand.DRIVER_PATH}:ro"),
+            "-v", (f"{output_dir}:"
+                   f"{ContainerCommand.OUTPUT_DIRECTORY}"),
+            *self.command.rom_mounts,
+            *self.command.common_client_mount,
+            self.image,
+            verify_in,
+            "--output-dir", ContainerCommand.OUTPUT_DIRECTORY,
+            "--timeout", str(per_world_timeout),
+            "--spoiler", str(spoiler),
+        ]
+
+    @staticmethod
+    def not_verified(detail):
+        return {"ran": False, "verified": [], "rejected": {},
+                "detail": detail}
+
+    def parse(self, result):
+        data = DriverReport(result).data
+        if data is not None:
+            data.setdefault("ran", True)
+            return data
+        return self.not_verified(
+            f"no JSON output (exit {result.returncode}): "
+            f"{DriverReport.tail(result.stderr)}")
+
+    def run(self, output_dir, timeout=None, per_world_timeout=None,
+            spoiler=None):
+        """The pool of usable companions, or why there is none."""
+        timeout = timeout or self.TIMEOUT
+        per_world_timeout = per_world_timeout or self.PER_WORLD_TIMEOUT
+        if spoiler is None:
+            spoiler = RunSettings(self.config).spoiler
+        output_dir = os.path.abspath(output_dir)
+        os.makedirs(output_dir, exist_ok=True)
+        try:
+            result = subprocess.run(
+                self.build(output_dir, per_world_timeout, spoiler),
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+            )
+        except subprocess.TimeoutExpired:
+            return self.not_verified("companion verification did not "
+                                     f"finish within {timeout}s")
+        return self.parse(result)

src/apworld_tester/pipeline/generation/identifier.py

@@ -0,0 +1,75 @@
+"""Asking an apworld which game it registers as."""
+
+import os
+import subprocess
+import tempfile
+
+from apworld_tester.core.model.apworld import ApworldFile
+from apworld_tester.pipeline.generation.command import ContainerCommand
+from apworld_tester.pipeline.generation.report import DriverReport
+
+
+class ApworldIdentifier:
+    """What a file's own World class says it is.
+
+    Lighter than a test: no ROMs, no output volume, no generation. Used
+    right after download to check the content matches the row that
+    fetched it, before the file is trusted at all.
+    """
+
+    #: Seconds an import check may take. Not configuration: this only
+    #: loads the apworld and reads which games it registers, so a file
+    #: still going after a minute is stuck, not slow.
+    TIMEOUT = 60
+
+    def __init__(self, config, paths, image):
+        self.paths = paths
+        self.image = image
+        self.command = ContainerCommand(config, paths, image)
+
+    def build(self, apworld_path):
+        mount, inside = self.command.apworld_mount(apworld_path)
+        identify_in = ContainerCommand.IDENTIFY_DRIVER_PATH
+        return [
+            *self.command.prefix,
+            *mount,
+            "-v", f"{self.paths.identify_driver}:{identify_in}:ro",
+            *self.command.common_client_mount,
+            self.image,
+            identify_in,
+            "--apworld", inside,
+        ]
+
+    def failed(self, detail):
+        return {"ran": False, "games": [], "detail": detail}
+
+    def identify(self, apworld_path, timeout=None):
+        """Which games this file registers, and whether the check ran."""
+        timeout = timeout or self.TIMEOUT
+        apworld_path = os.path.abspath(apworld_path)
+        if not os.path.isfile(apworld_path):
+            return self.failed(f"{apworld_path} does not exist (not a file)")
+        try:
+            result = self.run(apworld_path, timeout)
+        except subprocess.TimeoutExpired:
+            return self.failed(
+                f"identify check did not finish within {timeout}s")
+        data = DriverReport(result).data
+        if data is not None:
+            data.setdefault("ran", True)
+            return data
+        return self.failed(
+            f"no JSON output (exit {result.returncode}): "
+            f"{DriverReport.tail(result.stderr)}")
+
+    def run(self, apworld_path, timeout):
+        """Run the driver against a prepared copy of the file.
+
+        Repaired here as well as at test time, so a backslash-separated
+        world is not rejected at download for registering no game and
+        never reaching the test that could repair it.
+        """
+        with tempfile.TemporaryDirectory(prefix="apworld-id-") as workdir:
+            checked, _ = ApworldFile(apworld_path).prepare(workdir)
+            return subprocess.run(self.build(checked), capture_output=True,
+                                  text=True, timeout=timeout)

src/apworld_tester/pipeline/generation/report.py

@@ -0,0 +1,54 @@
+"""Reading what a driver reported."""
+
+import json
+
+
+class DriverReport:
+    """The JSON one driver run wrote to stdout."""
+
+    def __init__(self, result):
+        self.result = result
+
+    @staticmethod
+    def tail(text, lines=20):
+        return "\n".join(text.splitlines()[-lines:])
+
+    @staticmethod
+    def last_json_object(text):
+        """The driver's report, however it ended up on the line.
+
+        Not "the last line starting with {": worlds write to raw stdout
+        during generation, and one that writes without a trailing
+        newline leaves its output glued to the front of the report -
+        Super Metroid's randomizer emits a bare "*" per retry, so the
+        report arrives as "*{...}" and would read as no report at all.
+        """
+        for line in reversed(text.splitlines()):
+            line = line.strip()
+            start = line.find("{")
+            while start != -1:
+                try:
+                    return json.loads(line[start:])
+                except json.JSONDecodeError:
+                    start = line.find("{", start + 1)
+        return None
+
+    @property
+    def data(self):
+        return self.last_json_object(self.result.stdout)
+
+    def parsed(self):
+        """The report, or a failure built from what it printed."""
+        report = self.data
+        if report is not None:
+            report.setdefault("stderr_tail", self.tail(self.result.stderr))
+            return report
+        return {
+            "game": None,
+            "outcome": "failed",
+            "detail": ("no JSON report on stdout "
+                       f"(exit code {self.result.returncode})"),
+            "elapsed_seconds": None,
+            "stdout_tail": self.tail(self.result.stdout),
+            "stderr_tail": self.tail(self.result.stderr),
+        }

src/apworld_tester/pipeline/generation/tester.py

@@ -0,0 +1,136 @@
+"""Testing one world by generating seeds for it."""
+
+import os
+import subprocess
+import tempfile
+
+from apworld_tester.core.model.apworld import ApworldFile
+from apworld_tester.pipeline.generation.command import ContainerCommand
+from apworld_tester.pipeline.generation.report import DriverReport
+
+
+class WorldTester:
+    """Runs every generation mode against one world."""
+
+    #: Startup slack added to the container's own ceiling, on top of
+    #: every mode's generation timeout. Not configuration: it covers
+    #: docker start and interpreter boot, which no run changes.
+    HOST_TIMEOUT_BUFFER = 30
+
+    def __init__(self, config, paths, image):
+        self.config = config
+        self.paths = paths
+        self.image = image
+        self.command = ContainerCommand(config, paths, image)
+
+    def settings(self, key, default):
+        return self.config.value("testing", key, default)
+
+    def build(self, apworld_path, output_dir, timeout, companion_range,
+              spoiler, repeats, companion_pool, game, random_repeats):
+        """The full `docker run` for one world's whole test."""
+        mount, inside = self.command.apworld_mount(apworld_path)
+        selector = ["--apworld", inside] if inside else ["--game", game]
+        return [
+            *self.command.prefix,
+            *mount,
+            *self.command.driver_mounts(output_dir),
+            self.image,
+            ContainerCommand.DRIVER_PATH,
+            *selector,
+            "--output-dir", ContainerCommand.OUTPUT_DIRECTORY,
+            "--timeout", str(timeout),
+            "--companion-min", str(companion_range[0]),
+            "--companion-max", str(companion_range[1]),
+            "--spoiler", str(spoiler),
+            "--repeats", str(repeats),
+            "--random-repeats", str(random_repeats),
+            "--companion-pool", ",".join(companion_pool or ()),
+        ]
+
+    def host_timeout(self, timeout, repeats, random_repeats):
+        """How long to let the whole container run.
+
+        Every mode runs in the one container, each bounded by its own
+        timeout, so the host allows for all of them plus startup slack.
+        A ceiling rather than an expected cost.
+        """
+        generations = 2 * repeats + 1 + 2 * random_repeats
+        return timeout * generations + self.HOST_TIMEOUT_BUFFER
+
+    def missing_file(self, path):
+        """The failure for a path that is not a file.
+
+        Docker silently creates an empty directory at a missing mount
+        source rather than failing, which turns a not-yet-written
+        apworld into a bogus directory that breaks the content hash
+        days later with a confusing "Is a directory".
+        """
+        return {
+            "game": None,
+            "outcome": "failed",
+            "detail": f"{path} does not exist (not a file)",
+            "elapsed_seconds": None,
+        }
+
+    def test(self, apworld_path=None, game=None, output_dir=None,
+             timeout=None, companion_range=(2, 5), spoiler=3, repeats=10,
+             companion_pool=(), random_repeats=3):
+        """Run every mode against one world, core or downloaded."""
+        output_dir = os.path.abspath(output_dir or self.paths.output)
+        os.makedirs(output_dir, exist_ok=True)
+        timeout = timeout or self.settings("generation_timeout", 300)
+        if apworld_path is not None:
+            apworld_path = os.path.abspath(apworld_path)
+            if not os.path.isfile(apworld_path):
+                return self.missing_file(apworld_path)
+        with tempfile.TemporaryDirectory(prefix="apworld-") as workdir:
+            tested, repaired = self.prepared(apworld_path, workdir)
+            report = self.run_once(
+                apworld_path=tested,
+                game=game,
+                output_dir=output_dir,
+                timeout=timeout,
+                companion_range=companion_range,
+                spoiler=spoiler,
+                repeats=repeats,
+                companion_pool=companion_pool,
+                random_repeats=random_repeats,
+            )
+        if repaired:
+            report["repaired"] = True
+        return report
+
+    def prepared(self, apworld_path, workdir):
+        """The path to test, and whether it had to be repaired."""
+        if not apworld_path:
+            return None, False
+        return ApworldFile(apworld_path).prepare(workdir)
+
+    def run_once(self, apworld_path, game, output_dir, timeout,
+                 companion_range, spoiler, repeats, companion_pool,
+                 random_repeats):
+        """One container run, and the report it produced."""
+        limit = self.host_timeout(timeout, repeats, random_repeats)
+        command = self.build(
+            apworld_path=apworld_path,
+            output_dir=output_dir,
+            timeout=timeout,
+            companion_range=companion_range,
+            spoiler=spoiler,
+            repeats=repeats,
+            companion_pool=companion_pool,
+            game=game,
+            random_repeats=random_repeats,
+        )
+        try:
+            result = subprocess.run(command, capture_output=True, text=True,
+                                    timeout=limit)
+        except subprocess.TimeoutExpired:
+            return {
+                "game": None,
+                "outcome": "failed",
+                "detail": f"docker run did not exit within {limit}s",
+                "elapsed_seconds": limit,
+            }
+        return DriverReport(result).parsed()

src/apworld_tester/pipeline/sheet/fetcher.py

@@ -0,0 +1,62 @@
+"""Downloading the worlds sheet."""
+
+import io
+import os
+import zipfile
+
+import requests
+
+
+class SheetFetcher:
+    """The Playable Worlds sheet, as HTML."""
+
+    def __init__(self, config, paths):
+        self.config = config
+        self.paths = paths
+
+    @property
+    def spreadsheet_id(self):
+        return self.config.value("sheet", "spreadsheet_id")
+
+    @property
+    def tab_name(self):
+        return self.config.value("sheet", "tab_name")
+
+    @property
+    def export_url(self):
+        return ("https://docs.google.com/spreadsheets/d/"
+                f"{self.spreadsheet_id}/export?format=zip")
+
+    def fetch(self):
+        """The tab's HTML, as bytes.
+
+        Google answers a private sheet with its sign-in page and status
+        200 rather than an error, so the content type is checked instead
+        of the status.
+        """
+        response = requests.get(self.export_url, timeout=120)
+        response.raise_for_status()
+        if "application/zip" not in response.headers.get("Content-Type", ""):
+            raise RuntimeError(
+                "The server did not return a zip archive. The sheet is "
+                "most likely not shared with 'anyone with the link'.")
+        with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
+            wanted = f"{self.tab_name}.html"
+            if wanted not in archive.namelist():
+                raise RuntimeError(
+                    f"The export has no tab called '{self.tab_name}'. "
+                    f"Available: {', '.join(archive.namelist())}")
+            return archive.read(wanted)
+
+    def save(self, content=None):
+        """Write the tab's HTML, returning where it landed."""
+        content = self.fetch() if content is None else content
+        with open(self.paths.sheet_html, "wb") as handle:
+            handle.write(content)
+        return self.paths.sheet_html
+
+    def ensure(self):
+        """Download the sheet only when this machine has no copy."""
+        if not os.path.exists(self.paths.sheet_html):
+            self.save()
+        return self.paths.sheet_html

src/apworld_tester/pipeline/sheet/reader.py

@@ -0,0 +1,98 @@
+"""Reading the worlds sheet into Game objects."""
+
+from bs4 import BeautifulSoup
+
+from apworld_tester.core.model.game import Game
+
+
+class SheetReader:
+    """The sheet's HTML export, as rows."""
+
+    def __init__(self, config, path):
+        self.config = config
+        self.path = path
+
+    def column(self, name):
+        return (self.config.value("sheet", "columns") or {})[name]
+
+    def rows(self):
+        """Every table row, as lists of cells.
+
+        Google repeats each row's frozen-column cells in a spacer, which
+        would shift every column index by one, so those are dropped.
+        """
+        with open(self.path, "r", encoding="utf-8") as handle:
+            soup = BeautifulSoup(handle.read(), "html.parser")
+        found = []
+        for row in soup.find_all("tr"):
+            cells = [
+                cell for cell in row.find_all("td")
+                if "freezebar-cell" not in cell.get("class", [])
+            ]
+            if cells:
+                found.append(cells)
+        return found
+
+    def header(self, rows):
+        """Where the header row is, and the headings it holds."""
+        for index, cells in enumerate(rows):
+            texts = [cell.get_text(strip=True) for cell in cells]
+            if self.column("name") in texts:
+                return index, texts
+        raise ValueError(
+            f"No '{self.column('name')}' header found in the export.")
+
+    def indexes(self, headers):
+        """Where each field sits in a row."""
+        if self.column("link") not in headers:
+            raise ValueError(f"No '{self.column('link')}' column found.")
+        optional = {}
+        for field in ("stability", "pr_status"):
+            heading = self.column(field)
+            optional[field] = (headers.index(heading)
+                               if heading in headers else None)
+        return (headers.index(self.column("name")),
+                headers.index(self.column("link")),
+                optional["stability"],
+                optional["pr_status"])
+
+    @staticmethod
+    def text_of(cells, index):
+        """One cell's text, or None when missing or empty."""
+        if index is None or index >= len(cells):
+            return None
+        return cells[index].get_text(strip=True) or None
+
+    @staticmethod
+    def links_in(cell):
+        """Every hyperlink target in one cell, in the order written."""
+        return [anchor.get("href") for anchor in cell.find_all("a")
+                if anchor.get("href")]
+
+    def build(self, cells, indexes):
+        """One row as a Game, or None when it has no name."""
+        name_index, link_index, stability_index, status_index = indexes
+        name = self.text_of(cells, name_index)
+        if name is None:
+            return None
+        game = Game(
+            name=name,
+            stability=self.text_of(cells, stability_index),
+            pr_status=self.text_of(cells, status_index),
+        )
+        if link_index < len(cells):
+            for link in self.links_in(cells[link_index]):
+                game.add_link(link)
+        return game
+
+    def games(self):
+        """Every world listed in the export."""
+        rows = self.rows()
+        header_index, headers = self.header(rows)
+        indexes = self.indexes(headers)
+        found = []
+        for cells in rows[header_index + 1:]:
+            game = self.build(cells, indexes)
+            if game is not None:
+                found.append(game)
+        return found

src/apworld_tester/pipeline/update/download_reporter.py

@@ -0,0 +1,37 @@
+"""The download stage's progress line."""
+
+import os
+
+from apworld_tester.core.display.progress import Progress
+
+
+class DownloadReporter:
+    """One line per game, saying what its apworld did.
+
+    The download run calls back serialized, so this needs no lock of its
+    own. What each game's last asset did is remembered so the line can
+    still name it once the game itself finishes.
+    """
+
+    #: What each download status means, in words.
+    STATUS_LABELS = {
+        "missing": "downloaded (new)",
+        "stale": "downloaded (updated)",
+        "verified": "already present (verified up to date)",
+        "assumed": "already present (not verified)",
+    }
+
+    def __init__(self, config, total):
+        self.bar = Progress(total, config=config,
+                            labels={"ok": "ok", "none": "none"})
+        self.last_asset = {}
+
+    def on_asset(self, game_name, path, status):
+        label = self.STATUS_LABELS.get(status, status)
+        self.last_asset[game_name] = f"{label} {os.path.basename(path)}"
+
+    def on_game(self, index, total, game_name, downloaded):
+        detail = self.last_asset.pop(game_name, "no apworld")
+        self.bar.update(
+            index, "ok" if downloaded else "none", game_name,
+            fallback_line=f"[{index}/{total}] {game_name}: {detail}")

src/apworld_tester/pipeline/update/stage_error.py

@@ -0,0 +1,15 @@
+"""A failure, named by the stage it happened in."""
+
+
+class StageError(RuntimeError):
+    """Which stage failed, so the run can say so and stop.
+
+    Raised at the stage boundary rather than inside it: a stage can fail
+    anywhere after it was opened, and an unnamed traceback makes a long
+    run's failure much harder to place.
+    """
+
+    def __init__(self, stage, original):
+        super().__init__(f"failed during '{stage}': {original}")
+        self.stage = stage
+        self.original = original

src/apworld_tester/pipeline/update/update_run.py

@@ -0,0 +1,277 @@
+"""Everything the testing half does in one pass.
+
+Update Archipelago to the newest release, refresh the worlds sheet,
+download whatever changed, and test each world by generating seeds.
+What happens to the results afterwards - promoting, publishing,
+deploying - is the other half's job and is not known about here.
+"""
+
+import contextlib
+
+import requests
+
+from apworld_tester.core.config.config import Config
+from apworld_tester.core.config.paths import Paths
+from apworld_tester.core.config.run_settings import RunSettings
+from apworld_tester.core.config.secrets import Secrets
+from apworld_tester.core.model.name import Name
+from apworld_tester.core.state.keys import StateKeys
+from apworld_tester.core.state.store import StateStore
+from apworld_tester.pipeline.batch.batch import Batch
+from apworld_tester.pipeline.batch.discontinued import Discontinued
+from apworld_tester.pipeline.batch.orphaned_records import (
+    OrphanedRecords,
+)
+from apworld_tester.pipeline.batch.download_failures import (
+    DownloadFailures,
+)
+from apworld_tester.pipeline.batch.reporter import Reporter
+from apworld_tester.pipeline.batch.results import Results
+from apworld_tester.pipeline.build.archipelago import ArchipelagoBuild
+from apworld_tester.pipeline.download.download_run import DownloadRun
+from apworld_tester.pipeline.download.update_history import (
+    UpdateHistory,
+)
+from apworld_tester.pipeline.generation.identifier import ApworldIdentifier
+from apworld_tester.pipeline.sheet.fetcher import SheetFetcher
+from apworld_tester.pipeline.sheet.reader import SheetReader
+from apworld_tester.pipeline.update.download_reporter import (
+    DownloadReporter,
+)
+from apworld_tester.pipeline.update.stage_error import StageError
+
+
+class UpdateRun:
+    """The four stages, in the order they run.
+
+    Takes no lock of its own: a caller that also publishes the results
+    holds one for the whole thing, so nothing swaps underneath a
+    generation still in progress.
+    """
+
+    def __init__(self, config=None, paths=None, stability=None, fetch=None,
+                 jobs=None, force_retest=False):
+        config = config or Config.load()
+        paths = paths or Paths(config)
+        self.config = config
+        self.paths = paths
+        self.settings = RunSettings(config)
+        self.store = StateStore(paths.state)
+        self.build = ArchipelagoBuild(config, paths)
+        self.stability = stability
+        self.fetch = fetch
+        self.jobs = jobs or self.settings.jobs
+        self.force_retest = force_retest
+        self.report = {}
+
+    @contextlib.contextmanager
+    def stage(self, name):
+        print(f"\n=== {name} ===")
+        try:
+            yield
+        except Exception as error:
+            raise StageError(name, error) from error
+
+    def snapshot_previous_version(self, previous_tag):
+        """Keep last version's results before this one overwrites them.
+
+        The only source for "did this world's outcome change at the
+        exact moment Archipelago upgraded": it is the state at the
+        instant the version changed, which no later re-test can
+        reconstruct.
+        """
+        state = self.store.load()
+        state[StateKeys.UPGRADE_SNAPSHOT] = dict(
+            state.get(StateKeys.APWORLD_TESTS, {}))
+        state[StateKeys.PREVIOUS_ARCHIPELAGO_TAG] = previous_tag
+        self.store.save(state)
+
+    def update_archipelago(self):
+        """Put this machine on the newest release and build its image."""
+        with self.stage("archipelago_build"):
+            tag, previous, changed, built = self.build.update_and_build(
+                self.store)
+            self.build.ensure_test_image(tag)
+        self.report.update({
+            "archipelago_tag": tag,
+            "archipelago_previous_tag": previous,
+            "archipelago_tag_changed": changed,
+            "archipelago_image_built": built,
+        })
+        print(f"tag {tag}" + (f" (was {previous})" if previous
+                              else " (first run)"))
+        if changed and previous:
+            self.snapshot_previous_version(previous)
+        return tag, changed
+
+    def announce_retired(self, retired):
+        gone = sorted(key for key, value in retired.items() if value)
+        back = sorted(key for key, value in retired.items() if not value)
+        if gone:
+            print(f"{len(gone)} no longer on the sheet, marked "
+                  f"discontinued: {', '.join(gone)}")
+        if back:
+            print(f"{len(back)} back on the sheet: {', '.join(back)}")
+
+    def select(self, games):
+        """The rows this pass considers.
+
+        Applied to the whole sheet, core rows included, so --fetch
+        really does bound the work: core worlds are tested like every
+        other row, and leaving them out of the limit made a "cheap,
+        bounded pass" still test all eighty of them.
+        """
+        if self.stability:
+            games = [game for game in games
+                     if (game.stability or "").lower()
+                     == self.stability.lower()]
+        return games[:self.fetch] if self.fetch is not None else games
+
+    def read_sheet(self):
+        """The sheet's rows, split into core and downloadable.
+
+        Discontinued rows are flagged straight after reading, so every
+        stage below already sees a world taken off the sheet as gone.
+        Core rows are tested by name but never downloaded: Archipelago
+        ships them.
+        """
+        with self.stage("sheet_download"):
+            path = SheetFetcher(self.config, self.paths).save()
+            games = SheetReader(self.config, path).games()
+            # Every row, before any filtering: download-time identity
+            # checks compare against the whole sheet, core rows
+            # included, and so does the discontinued flag.
+            sheet_names = [game.name for game in games]
+        self.report["games_in_sheet"] = len(games)
+        print(f"{len(games)} games in sheet")
+        retired = Discontinued(self.store).mark(sheet_names)
+        self.report["discontinued_changed"] = retired
+        self.announce_retired(retired)
+        selected = self.select(games)
+        core = [game for game in selected if game.is_core]
+        rest = [game for game in selected if not game.is_core]
+        self.report["games_core"] = len(core)
+        self.report["games_selected"] = len(rest)
+        print(f"{len(core)} core games")
+        print(f"{len(rest)} games selected")
+        return rest, core, sheet_names
+
+    def session(self):
+        """A session carrying the token, if there is one."""
+        session = requests.Session()
+        token = Secrets(self.paths).github_token
+        if token:
+            session.headers["Authorization"] = f"token {token}"
+        return session
+
+    def downloader(self, image, sheet_names):
+        return DownloadRun(
+            config=self.config,
+            paths=self.paths,
+            session=self.session(),
+            identifier=ApworldIdentifier(self.config, self.paths, image),
+            jobs=self.jobs,
+            sheet_names=sheet_names,
+            # Last run's resolved releases, so a game whose attempt
+            # failed against an unchanged release is not downloaded and
+            # re-identified all over again.
+            previous_releases=self.store.load().get(
+                StateKeys.GAME_RELEASES, {}),
+        )
+
+    def download(self, games, sheet_names, image, tag):
+        """Fetch every selected game's newest apworld."""
+        reporter = DownloadReporter(self.config, len(games))
+        run = self.downloader(image, sheet_names)
+        with self.stage("apworld_download"):
+            total, skipped, releases = run.all(
+                games, on_asset=reporter.on_asset, on_game=reporter.on_game)
+        self.report["assets_downloaded"] = total
+        self.report["games_skipped_download"] = len(skipped)
+        print(f"{total} assets downloaded, {len(skipped)} games skipped")
+        run.record_releases(releases)
+        self.record_versions()
+        DownloadFailures(self.config, self.store).record(games, skipped, tag)
+        orphaned = OrphanedRecords(self.store, self.paths.downloads).prune()
+        self.report["orphaned_records_dropped"] = len(orphaned)
+        if orphaned:
+            print(f"{len(orphaned)} result(s) dropped for apworlds no "
+                  f"longer on disk: {', '.join(sorted(orphaned))}")
+
+    def record_versions(self):
+        """One history entry per game whose apworld content changed.
+
+        Recorded here, not by whatever publishes afterwards: this is
+        the moment a new version is known, and a pass that never
+        publishes would lose it outright - the next download overwrites
+        the file and no later stage can reconstruct which version
+        existed on which date.
+        """
+        history = UpdateHistory(self.paths)
+        with self.stage("apworld_history"):
+            recorded = history.record_all(self.store.load(),
+                                          self.paths.downloads)
+        self.report["new_apworld_versions"] = len(recorded)
+        print(f"{len(recorded)} new apworld version(s) recorded")
+
+    def selected_only(self, games, core_games):
+        """This pass's rows, named the way the batch names them.
+
+        None unless something actually narrowed the sheet. The download
+        directory holds every apworld ever fetched, so an unnarrowed run
+        has to consider all of them - and Batch reads an explicit
+        selection as a reason to retest rather than serve from cache,
+        which would turn a plain pass into a full re-test of everything.
+
+        Downloadable rows are matched on their download folder, core
+        ones on the bare sheet name, because that is how each is keyed.
+        """
+        if self.fetch is None and not self.stability:
+            return None
+        return ([Name(game.name).directory for game in games]
+                + [game.name for game in core_games])
+
+    def test(self, image, tag, games, core_games, changed):
+        """Generation-test the downloaded apworlds, and the core ones.
+
+        Core worlds ship with Archipelago, so nothing was downloaded for
+        them and they are named rather than mounted, but they run the
+        same five modes and their result is measured rather than
+        assumed.
+
+        A narrowed pass tests what it narrowed to and nothing else:
+        --fetch bounded the download stage from the start, and leaving
+        the test stage unbounded meant a "cheap, bounded pass" still
+        generated against every apworld on disk.
+        """
+        only = self.selected_only(games, core_games)
+        if only is not None:
+            print(f"{len(only)} selected for testing")
+        with self.stage("generation_tests"):
+            results = Batch(
+                config=self.config,
+                paths=self.paths,
+                settings=self.settings,
+                image=image,
+                tag=tag,
+                force=changed or self.force_retest,
+                on_progress=Reporter(self.config, bar=True),
+                jobs=self.jobs,
+                core_games=[game.name for game in core_games],
+                only=only,
+                exact_only=only is not None,
+            ).run()
+        tested = Results(results, self.config)
+        self.report["test_counts"] = tested.summary
+        self.report["test_results"] = results
+        print(tested.overview)
+        return results
+
+    def run(self):
+        """Every stage, in order, returning what each one did."""
+        tag, changed = self.update_archipelago()
+        image = self.build.test_image_tag(tag)
+        games, core_games, sheet_names = self.read_sheet()
+        self.download(games, sheet_names, image, tag)
+        self.test(image, tag, games, core_games, changed)
+        return self.report

src/apworld_tester/templates/__init__.py

@@ -0,0 +1 @@
+"""The starter config shipped with the package."""

src/apworld_tester/templates/config.yaml

@@ -0,0 +1,69 @@
+# Config file
+
+general:
+  # Output directory of everything a run generates, 
+  # including the Archipelago source and downloaded APworlds
+  output_directory: /path/to/your/archipelago-world-tester-data
+  roms_directory: /path/to/your/base-roms
+  threads: 4
+
+sheet:
+  # The Community Spreadsheet
+  spreadsheet_id: 1iuzDTOAvdoNe8Ne8i461qGNucg5OuEoF-Ikqs8aUQZw
+  tab_name: Playable Worlds
+  columns:
+    name: Game
+    link: Links & Downloads
+    stability: Stability
+    pr_status: PR Status
+
+
+archipelago:
+  # Settings regarding the Archipelago repository to use for 
+  # testing worlds. 
+  version: "latest"
+  repo_url: https://github.com/ArchipelagoMW/Archipelago.git
+  repository: ArchipelagoMW/Archipelago
+
+  # The Docker image basename built from that checkout
+  image_name: archipelago
+
+# repeats is what makes a verdict a pass rate rather than a single run;
+# lowering it makes a run cheaper and its answer weaker, and the cache
+# knows the difference.
+testing:
+  generation_timeout: 300
+  companion_min: 2
+  companion_max: 5
+  repeats: 10
+  random_repeats: 3
+
+  # none: no spoiler log
+  # log: spoiler log, no playthrough
+  # playthrough: also computes the playthrough
+  # paths: also computes entrance paths
+  spoiler: playthrough
+
+
+
+# How each verdict is written in the sheet's Stability column.
+stability:
+  labels:
+    stable: Stable
+    unstable: Minor issues
+    flaky: Flaky
+    solo_only: Solo only
+    broken: Broken
+    unknown: Unknown
+
+# Tag prefixes for repositories that publish several games' releases
+# side by side, so the right release is picked per game.
+release_overrides:
+  Donkey Kong Country: dkc-
+  "Donkey Kong Country 2: Diddy's Kong Quest": dkc2-
+  Mega Man X: mmx-
+  Mega Man X2: mmx2-
+
+# Worlds not tested.
+blacklist:
+  - Ocarina of Time but it's just Master Quest Water Temple

src/archipelago_tester/__init__.py

@@ -0,0 +1,5 @@
+"""Tests Archipelago community worlds by generating seeds with them.
+
+The publishing half - archipelago-world-site - imports this package as a
+dependency. Nothing here knows that it exists.
+"""