Skilly/APWorldTester
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

initial commit

Browse Source
This commit is contained in:
Skilly
2026-09-06 15:12:38 +02:00
parent 4caf096780
commit 0d77a07291
185 changed files with 15465 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.env.example Normal file
View File

@@ -0,0 +1,5 @@
# Copy to .env and fill in. .env is gitignored; this file is not.
#
# A GitHub personal access token, needing no scopes at all - it is only
# used to raise the API rate limit from 60 requests an hour to 5000.
GITHUB_TOKEN=

190
.gitignore vendored
View File

@@ -1,176 +1,26 @@
# ---> Python
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
*.pyc
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
#uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
# Credentials. Never committed - see .env.example.
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Nothing a run generates lives here any more: config.yaml's "data_root"
# points outside the checkout (the Archipelago clone, downloads, seeds
# and state.json all live there). These remain only to catch a config
# that points data_root back inside it.
state.json
scratch_state.json
pipeline.lock
downloads/
test_output/
archipelago_src/
playable_worlds.html
failed_apworlds.txt
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
# Ruff stuff:
.ruff_cache/
# PyPI configuration file
.pypirc
# Claude file
CLAUDE.md
# Build output.
build/
dist/
*.egg-info/

29
LICENSE
View File

@@ -1,18 +1,21 @@
MIT License
Copyright (c) 2026 pascal
Copyright (c) 2026 Pascal
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
following conditions:
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial
portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

163
README.md
View File

@@ -1,3 +1,162 @@
# APWorldTester
# Archipelago World Tester
This project checks whether community Archipelago worlds can generate a seed.
It reads the community worlds spreadsheet, downloads each world's newest `.apworld` file, and runs Archipelago's generator against it in a Docker container.
Each world is tested five ways:
- by itself
- with two players of the same game
- with 2–5 of Archipelago's built-in worlds
- by itself with its options randomized
- with other worlds and its options randomized
Each test is repeated with different seeds. The results are combined into one of six stability values:
| Value | Meaning |
| --- | --- |
| Stable | Every attempt succeeded |
| Minor issues | Solo and multiworld tests worked, but a randomized test failed |
| Flaky | Some attempts succeeded and others failed |
| Solo only | Works by itself, but fails with other worlds |
| Broken | Could not generate at all |
| Unknown | No usable result, for example because the `.apworld` was missing or a ROM was required |
A successful test means the world loaded, generated a seed, and computed the spoiler log at the configured level. With the default `playthrough` level, this also checks that the seed is completable.
## Requirements
- **Python 3.10 or newer**
- **Git** — used to clone Archipelago and check out the version being tested
- **Docker** — must be usable by your user without `sudo`
- **A GitHub token** — required to access the GitHub API. It does not need any scopes
- **Base ROMs** — optional; some worlds cannot generate without them
## Installation
### System packages
On Ubuntu or Debian:
```sh
sudo apt update
sudo apt install -y git python3 python3-venv
```
Install Docker:
```sh
curl -fsSL https://get.docker.com | sudo sh
sudo usermod -aG docker "$USER"
newgrp docker
docker run --rm hello-world
```
The last command should work without `sudo`.
### Install the package
```sh
git clone <repository-url> archipelago-world-tester
cd archipelago-world-tester
```
Using `uv`:
```sh
uv sync
```
Or using `pip`:
```sh
python3 -m venv .venv
source .venv/bin/activate
pip install -e .
```
### Configuration
Write a starter config and edit the paths in it:
```bash
apworld-tester init
```
That writes `~/.config/apworld-tester/config.yaml` (or
`$XDG_CONFIG_HOME/apworld-tester/config.yaml`) and prints the path.
Set `general.output_directory` - everything a run generates lives there,
including the Archipelago checkout and the downloaded apworlds - and
`general.roms_directory` for the worlds that need a base ROM.
The config is looked for in this order, first hit wins:
1. `$APWORLD_TESTER_CONFIG`
2. `~/.config/apworld-tester/config.yaml`
3. `config.yaml` beside a source checkout
`apworld-tester config-path` prints which one is in use, or lists
every location it tried.
Create `.env` with your GitHub token:
```text
GITHUB_TOKEN=ghp_...
```
## Test worlds
Use `TestRun` to test one or more worlds:
```python
from apworld_tester import TestRun
TestRun("Baba Is You").run()
```
Multiple worlds:
```python
TestRun(["Baba Is You", "Anodyne"]).run()
```
To test all worlds that have already been downloaded:
```python
TestRun(cached=True).run()
```
The first run clones Archipelago and builds the Docker image, which can take several minutes.
## Test all worlds
`UpdateRun` updates Archipelago, refreshes the community worlds spreadsheet, downloads changed worlds, and tests everything:
```python
from apworld_tester import UpdateRun
UpdateRun().run()
```
## Test a local `.apworld`
For a world you are developing, put the `.apworld` in its own directory:
```text
~/my-apworlds/
└── My Game/
└── my_game.apworld
```
Then run:
```python
from apworld_tester import TestRun
TestRun(
"My Game",
root_directory="/home/you/my-apworlds",
).run()
```
Test Suite for Archipelago Worlds

20
container/Dockerfile.test Normal file
View File

@@ -0,0 +1,20 @@
# Unprivileged wrapper for the generation tests.
#
# Upstream's image runs as root and leaves /.local root-owned, so running
# it as the host user fails before generation starts (PermissionError on
# /.local) and running it as root leaves root-owned seeds in the mounted
# output directory. Two lines fix both, and they are the only difference
# from the base image.
#
# Built once per Archipelago tag, not per apworld test, so the cost of
# the recursive chown is paid once and amortised over a few hundred
# generations.
ARG BASE_IMAGE=archipelago:latest
FROM ${BASE_IMAGE}
ARG RUNNER_UID=1000
ARG RUNNER_GID=1000
RUN chown -R ${RUNNER_UID}:${RUNNER_GID} /app /.local 2>/dev/null || chown -R ${RUNNER_UID}:${RUNNER_GID} /app
USER ${RUNNER_UID}:${RUNNER_GID}

49
container/drivers/identify_apworld.py Normal file
View File

@@ -0,0 +1,49 @@
"""Runs inside the archipelago Docker image. Reports which game (if any)
a single mounted .apworld registers itself as under Archipelago's own
World registry - the import/registration step only, no seed generation.
Used to verify a freshly downloaded .apworld's actual content matches
the sheet row that fetched it, rather than trusting release/asset
name-matching alone (see AssetMatcher): the filename
and release title are just text an author chose, but AutoWorldRegister
is filled from the World class's own "game" attribute, which is what
Archipelago itself will treat this file as - the authoritative source.
"""
import argparse
import json
import sys
import warnings
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--apworld", required=True)
args = parser.parse_args()
warnings.simplefilter("ignore")
try:
import worlds
from worlds.AutoWorld import AutoWorldRegister
except BaseException as error:
# The apworld (or something it imports) crashed outright - that's
# itself a meaningful signal (this file is broken/wrong), not an
# infrastructure failure, so it's still reported as a clean JSON
# result rather than a non-zero exit / stack trace.
print(json.dumps({"games": [], "detail": f"{type(error).__name__}: {error}"}))
return 0
games = [
name for name, world_type in AutoWorldRegister.world_types.items()
if str(getattr(world_type, "zip_path", "") or "") == args.apworld
]
print(json.dumps({
"games": games,
"detail": None if games else f"failed_world_loads={worlds.failed_world_loads}",
}))
return 0
if __name__ == "__main__":
sys.exit(main())

827
container/drivers/run_test.py Normal file
View File

@@ -0,0 +1,827 @@
"""Runs inside the archipelago Docker image. Generates one seed for a
single installed apworld and reports the outcome as one JSON line on
stdout: passed, needs_input (missing base ROM or similar), or failed.
The logic mirrors Archipelago's own test/hosting/generate.py, but is
copied here rather than imported from it: the project's .dockerignore
excludes test/ from the image, so that module isn't present at runtime.
"""
import argparse
import io
import json
import logging
import random
import re
import shutil
import sys
import unicodedata
import time
import warnings
from multiprocessing import Manager, Process, set_start_method
from pathlib import Path
def ensure_tutorials_default():
# Monkeypatch applied AFTER whichever Archipelago version's own real
# worlds/AutoWorld.py has already loaded normally - defaults
# WebWorld.tutorials to [] if that core doesn't already declare it,
# without needing to know or duplicate anything else about that
# core's own AutoWorld.py content. Previously this was a full-file
# bind-mount replacement (container/AutoWorld.py (absent - see config.py)) built against one
# specific version's source; that broke outright when tested against
# an older Archipelago release whose real AutoWorld.py imports
# differently (e.g. no rule_builder module before it was added
# upstream) - replacing the whole file assumes it stays
# source-compatible with everything else that release's core
# expects, which isn't true across versions. This achieves the same
# fix (see the original container/AutoWorld.py (absent - see config.py) comment for why it's
# needed - WebHost.py's invalid_worlds filter vs.
# network_data_package snapshot timing) without that assumption.
from worlds.AutoWorld import WebWorld
if "tutorials" not in vars(WebWorld):
WebWorld.tutorials = []
class WebHostIncompatibleError(RuntimeError):
"""Generation succeeds, but this world would crash WebHost's own
startup - a bug class Generate.py never exercises (see
check_webhost_compatibility below)."""
def check_webhost_compatibility(game):
# Reproduces WebHost's own generate_yaml_templates() (Options.py),
# scoped to just this one world, so a bug like Option.visibility
# being set to a bare int instead of a Visibility flag - which
# crashes generate_yaml_templates() for the ENTIRE deployed site,
# not just the offending world, since it iterates every visible
# world in one loop - is caught here as a normal test failure
# instead of only in production. (A hand-run diagnostic that located
# the offending worlds used to live in container/tools/; removed.)
# which found this bug class live on the deployed instance.
from inspect import cleandoc
import yaml
from jinja2 import Template
import Options
from Utils import local_path, __version__
from worlds.AutoWorld import AutoWorldRegister
world = AutoWorldRegister.world_types[game]
if not hasattr(world.web, "tutorials"):
# WebHost.py strips any world missing this from AutoWorldRegister
# entirely at startup (its invalid_worlds set) - so it can't
# crash generate_yaml_templates (never part of that loop), but
# the world is now gone from WebHost's registry altogether. That
# breaks something worse than templates: MultiServer can't find
# the game's data (item_name_groups etc.) when starting a room,
# so *hosting* any seed generated with this world crashes with
# a KeyError - even though generation itself succeeded fine.
raise WebHostIncompatibleError(
"passes generation but has no WebWorld.tutorials - WebHost "
"strips it from its world registry entirely at startup, so "
"hosting a room for a seed generated with it crashes "
"MultiServer with a KeyError on the game name"
)
def dictify_range(option, option_val):
data = {option_val: 50}
for sub_option in ["random", "random-low", "random-high",
f"random-range-{option.range_start}-{option.range_end}"]:
if sub_option != option_val:
data[sub_option] = 0
notes = {}
for name, number in getattr(option, "special_range_names", {}).items():
notes[name] = f"equivalent to {number}"
if number in data:
data[name] = data[number]
del data[number]
elif name in data:
pass
else:
data[name] = 0
return data, notes
def yaml_dump_scalar(scalar):
return yaml.dump(scalar).replace("...\n", "").strip()
with open(local_path("data", "options.yaml")) as f:
template = Template(f.read())
try:
option_groups = Options.get_option_groups(world)
presets = world.web.options_presets.copy()
presets.update({"": {}})
for name, preset in presets.items():
template.render(
option_groups=option_groups,
__version__=__version__,
game=game,
world_version=world.world_version.as_simple_string(),
yaml_dump=yaml_dump_scalar,
dictify_range=dictify_range,
cleandoc=cleandoc,
preset_name=name,
preset=preset,
)
except Exception as error:
raise WebHostIncompatibleError(
f"passes generation but would crash WebHost's own startup "
f"(generate_yaml_templates): {type(error).__name__}: {error}"
) from error
def core_worlds(exclude):
"""Every world Archipelago ships, minus the one under test.
Core worlds are the safe companions: they are part of the build
itself, so they are always present and always match the running
version - unlike an apworld, which might be missing or broken for
reasons that have nothing to do with the world being tested.
"""
from worlds.AutoWorld import AutoWorldRegister
return sorted(
name for name, world_type in AutoWorldRegister.world_types.items()
if not str(getattr(world_type, "zip_path", "") or "") and name != exclude
)
def random_options(game):
"""One concrete random value per option of `game`.
Only the option types with a defined value space are randomized -
toggles, choices and ranges. Free text, option lists/sets and item
dicts (start inventory, plando, exclusions...) have no meaningful
"random", and forcing one would fail generation for reasons that have
nothing to do with the world being tested.
The values are RESOLVED here rather than written into the yaml as the
string "random". Both randomized modes then run the exact same option
values, so a difference between them is attributable to the extra
players and nothing else - writing "random" would have each mode roll
its own values from its own seed, and SR-passes-while-MR-fails would
say nothing about the multiworld.
Resolution goes through the option's own from_text("random"), so the
values are exactly the ones Archipelago itself would roll: weighted
ranges, named-range specials and all.
"""
from worlds.AutoWorld import AutoWorldRegister
import Options
randomizable = (Options.Toggle, Options.Choice, Options.Range)
world_type = AutoWorldRegister.world_types[game]
chosen = {}
for name, option in getattr(world_type.options_dataclass, "type_hints", {}).items():
if not issubclass(option, randomizable):
continue
try:
chosen[name] = option.from_text("random").value
except Exception:
# A world can define an option whose randomization raises;
# leaving it out means it keeps its default rather than
# failing the whole mode for one unrollable option.
continue
return chosen
def generate_one(games, dest, results, options_per_game=None, seed=None, spoiler=3):
# games is the full player list: the world under test first, then any
# companions (see RunSettings). One entry is the single-player
# test, the same game twice is the duplicate test, and the world plus
# companions is the multi-game test - all the same code path, differing
# only in who is in the seed and what options they roll.
warnings.simplefilter("ignore")
try:
from tempfile import TemporaryDirectory
import Generate
import Main
ensure_tutorials_default()
with TemporaryDirectory() as players_dir, TemporaryDirectory() as output_dir:
for number, game in enumerate(games, start=1):
player_path = Path(players_dir) / f"{number}.yaml"
# {} means every option at its default - the only point in
# option space the pipeline used to cover.
game_options = (options_per_game or {}).get(game, {})
player_path.write_text(json.dumps({
"name": f"Tester{number}",
"game": game,
game: game_options,
"description": f"ArchiUpdater compatibility test: {game}",
}), encoding="utf-8")
sys.argv = [
sys.argv[0],
"--player_files_path", players_dir,
"--outputpath", output_dir,
# Spoiler 3 computes the full playthrough, which exercises
# accessibility/reachability logic that plain generation
# never reaches - "a seed came out" becomes "the seed is
# actually completable".
"--spoiler", str(spoiler),
]
if seed is not None:
sys.argv += ["--seed", str(seed)]
Main.main(*Generate.main())
check_webhost_compatibility(games[0])
output_files = list(Path(output_dir).glob("*.zip"))
if len(output_files) != 1:
raise RuntimeError(
f"expected exactly one output file, found {len(output_files)}")
final_file = Path(dest) / output_files[0].name
# rename() can't cross the tmpdir -> bind-mounted output volume
# boundary, so move (copy + delete) instead.
shutil.move(str(output_files[0]), str(final_file))
results.append(str(final_file))
except BaseException as error:
results.append(error)
raise
def describe_load_failure(log_text, apworld_path):
"""The real reason an apworld did not load, out of the swallowed log.
worlds/__init__.py catches an import failure per world, formats the
traceback, and hands it to logging.exception - so by default it goes
nowhere and the pipeline could only report "found 0 worlds", which
says nothing an author could act on. Untitled Goose Game, for
instance, defines `option_random` on a Choice, which Archipelago
reserves; the assert fires at class-definition time during import.
Returns (summary, traceback) - summary being the exception and the
line in the world's own code that raised it.
"""
marker = "Could not load world"
TRACEBACK_HEADER = "Traceback (most recent call last):"
blocks = [block for block in log_text.split(marker) if apworld_path in block]
if not blocks:
return None, None
block = marker + blocks[-1]
lines = [line for line in block.splitlines() if line.strip()]
# The exception itself is the last line of a traceback.
exception = next((line.strip() for line in reversed(lines)
if line.strip() and not line.startswith((" ", "\t"))
and "Traceback" not in line and marker not in line), None)
# The deepest frame inside the apworld itself - core frames above it
# are just the import machinery and tell an author nothing.
where = None
for line in lines:
match = re.search(r'File "([^"]*%s[^"]*)", line (\d+)' % re.escape(apworld_path), line)
if match:
inside = match.group(1).split(apworld_path + "/", 1)[-1]
where = f"{inside}, line {match.group(2)}"
summary = exception or "import failed"
if where:
summary += f" (in {where})"
# logging.exception both formats the traceback into the message AND
# appends exc_info's own copy, so the block holds it twice - keep one.
first = block.find(TRACEBACK_HEADER)
if first != -1:
second = block.find(TRACEBACK_HEADER, first + len(TRACEBACK_HEADER))
if second != -1:
block = block[:second]
# Drop the import machinery. Every one of these frames is identical
# for every failing world and tells an author nothing; what is left is
# their own code and the line that actually raised.
# A frame is its File line plus the indented source and caret lines
# under it, so drop the whole frame - removing only the File line
# leaves orphaned source fragments behind.
noise = ("<frozen importlib", "importlib/__init__.py", "worlds/__init__.py", "_bootstrap")
kept = []
skipping = False
for line in block.splitlines():
stripped = line.lstrip()
if stripped.startswith("File "):
skipping = any(n in line for n in noise)
if skipping:
continue
elif skipping:
# Continuation of a dropped frame: still indented, and not the
# start of the next frame.
if line.startswith(" ") or not line.strip():
continue
skipping = False
kept.append(line)
return summary, "\n".join(kept)
def rooted_in(error, exception_type):
seen = set()
while error is not None and id(error) not in seen:
seen.add(id(error))
if isinstance(error, exception_type):
return True
error = error.__cause__ or error.__context__
return False
def rooted_in_named(error, class_name):
# Some apworlds define their own exception class inside their own
# zipimported package (e.g. CTJoT's InvalidYamlException) rather
# than a stable top-level module - there's no fixed path to import
# and isinstance-check it the way Fill.FillError works, so match by
# class name instead.
seen = set()
while error is not None and id(error) not in seen:
seen.add(id(error))
if type(error).__name__ == class_name:
return True
error = error.__cause__ or error.__context__
return False
def run_with_timeout(games, dest, timeout, options_per_game=None, seed=None, spoiler=3):
# fork, not spawn. Every mode runs in its own process so a generation
# can be killed on timeout and cannot leak state into the next one -
# but spawn re-imports the whole of Archipelago in each child, five
# times per apworld, which measured as the single largest cost in a
# run. Forking inherits the import the parent already did.
#
# The isolation that matters is unchanged: the child still gets its
# own copy-on-write memory, so whatever generation mutates dies with
# it and the parent's registry stays pristine for the next mode.
#
# Safe here specifically because the forking process is
# single-threaded: main() starts no threads of its own, and the
# Manager below is a separate process rather than a thread in this
# one. Forking a multi-threaded parent is the case to avoid, and this
# is not one.
try:
set_start_method("fork")
except RuntimeError:
pass
manager = Manager()
results = manager.list()
process = Process(target=generate_one,
args=(games, dest, results, options_per_game, seed, spoiler))
started = time.monotonic()
process.start()
process.join(timeout)
timed_out = process.is_alive()
if timed_out:
process.terminate()
process.join(5)
if process.is_alive():
process.kill()
process.join()
elapsed = time.monotonic() - started
return list(results), timed_out, elapsed
def classify(results, timed_out, elapsed, single_player=True, randomized=False):
if timed_out:
return "failed", f"timed out after {elapsed:.0f}s"
if not results:
return "failed", "generation process exited without a result"
result = results[0]
if isinstance(result, BaseException):
if rooted_in(result, FileNotFoundError):
return "needs_input", str(result)
if randomized and rooted_in_named(result, "OptionError"):
# The world rejected the option combination we rolled - not a
# crash and not a fill failure, but the world validating its
# own input. Archipelago rolls every option independently, so
# a world with interdependent options (Blender's min/max
# similarity percent, for example) can always be handed a
# combination it considers invalid. A player writing
# "random" for both hits exactly this, so it is worth
# reporting - but as its own thing, not as a defect.
#
# Only when the options were randomized: a world that rejects
# its OWN DEFAULTS is genuinely broken and stays "failed".
return "invalid_options", f"rejected the rolled options: {result}"
from Fill import FillError
if rooted_in(result, FillError):
# A fill failure is a failure in both modes. This used to be
# excused in single-player on the theory that some games'
# item/location balance "only works out in a real multiworld"
# - the multi-game test exists precisely to check that theory,
# and it does not hold (Dead Cells fails both ways), so the
# excuse is gone: a seed that cannot be filled is a seed that
# cannot be generated, whoever else is in it.
where = "a multi-game seed" if not single_player else "a single-player seed"
return "failed", f"fill failed in {where}: {result}"
if rooted_in_named(result, "InvalidYamlException"):
# Some worlds (e.g. CTJoT) require a yaml pre-generated by an
# external tool and reject our generic single-player yaml
# outright - not a defect, just incompatible with this
# testing methodology.
return "passed", f"requires an externally-generated yaml (not a defect): {result}"
return "failed", f"{type(result).__name__}: {result}"
return "passed", str(result)
def resolve_core_game(name, world_types):
"""Match a sheet name to a world Archipelago actually registers.
The sheet and the worlds disagree about wording often enough that an
exact comparison finds only 72 of the 81 core rows. The rest differ
by an accent ("Pokemon Emerald"), a hyphen ("Choo-Choo Charles"),
capitalisation ("EarthBound"), or a series prefix the sheet adds and
the world does not ("The Legend of Zelda: Ocarina of Time").
Tried in order: the name as written, then folded to letters and
digits, then either side of a colon folded the same way. Either side,
because the sheet puts the distinguishing part before the colon as
often as after it - "Super Mario Land 2: The Golden Coins" is
registered as "Super Mario Land 2", while "The Legend of Zelda:
Ocarina of Time" is registered as "Ocarina of Time".
"""
if name in world_types:
return name
by_slug = {}
for registered in world_types:
by_slug.setdefault(slug(registered), registered)
candidates = [name]
if ":" in name:
head, tail = name.split(":", 1)
candidates += [tail, head]
for candidate in candidates:
registered = by_slug.get(slug(candidate))
if registered:
return registered
return None
# Deliberate copy of Name.slug in core/model/name.py. This file is mounted
# alone into the container, with no access to this checkout's
# packages, so it cannot import it - but the two must stay identical
# or a world folds to one key here and a different one on the host.
def slug(text):
return re.sub(r"[^a-z0-9]", "", unicodedata.normalize("NFKD", (text or "").lower()))
def main():
parser = argparse.ArgumentParser(description=__doc__)
# Exactly one of these. --apworld is the normal case: a downloaded
# file, identified by which world claims it as its zip_path. --game
# names a world that is already part of the Archipelago build, which
# has no file to point at and so cannot be identified that way.
parser.add_argument("--apworld")
parser.add_argument("--game", help="Test a world that ships with Archipelago, by name, "
"instead of an apworld file. Mutually exclusive with --apworld.")
parser.add_argument("--output-dir", required=True)
parser.add_argument("--timeout", type=int, default=300)
parser.add_argument("--companion-pool", default="",
help="Comma-separated core games to draw companions from. Empty "
"means every core world, which risks drawing one that cannot "
"generate here (see container/verify_companions.py).")
parser.add_argument("--companion-min", type=int, default=2,
help="Fewest core games to draw as companions for a multi-game seed.")
parser.add_argument("--companion-max", type=int, default=5,
help="Most core games to draw as companions for a multi-game seed.")
parser.add_argument("--random-repeats", type=int, default=3,
help="How many times the randomized-option modes run. Each attempt "
"rolls a different option combination.")
parser.add_argument("--repeats", type=int, default=10,
help="How many times to run the single-game and multi-game tests. "
"Stability is a pass rate, not a single verdict.")
parser.add_argument("--spoiler", type=int, default=3,
help="Spoiler level passed to Generate. 3 computes the full "
"playthrough, exercising accessibility logic.")
args = parser.parse_args()
if bool(args.apworld) == bool(args.game):
parser.error("pass exactly one of --apworld or --game")
# Determine the game name from the loaded World class itself rather
# than the apworld's archipelago.json manifest: older apworlds don't
# ship a manifest at all, and the core (<0.7.0 here) tolerates that
# and loads them anyway - a manifest-only lookup would misreport a
# perfectly loadable world as failed. World.zip_path is set only for
# worlds loaded from a .apworld zip (worlds/AutoWorld.py), and equals
# the apworld's own path, so it identifies our one mounted world
# exactly regardless of manifest presence.
# worlds/__init__.py logs each failed world's traceback and carries
# on; without a handler here that detail is lost and all the pipeline
# can say is "found 0 worlds".
load_log = io.StringIO()
log_handler = logging.StreamHandler(load_log)
log_handler.setLevel(logging.ERROR)
logging.getLogger().addHandler(log_handler)
try:
import worlds
from worlds.AutoWorld import AutoWorldRegister
finally:
logging.getLogger().removeHandler(log_handler)
ensure_tutorials_default()
if args.game:
# A core world is already loaded - there is no file to identify
# it by, and none of the load-failure reporting below applies. It
# either exists in this build or it does not, and "does not" is a
# naming problem on the sheet rather than a defect in a world.
resolved = resolve_core_game(args.game, AutoWorldRegister.world_types)
if resolved:
args.game = resolved
else:
# "unknown", not "unverified". Nothing about this world is
# broken - this Archipelago build simply has no world under
# that name, which is a naming mismatch on the sheet (or a
# world that has since been removed from core), and calling
# it Broken would blame a world for the sheet's wording.
print(json.dumps({
"game": None,
"outcome": "unknown",
"detail": (f"this Archipelago build registers no world named {args.game!r}, "
f"so there was nothing to test. The sheet lists this row as Core, "
f"so either its name differs from the world's own or the world is "
f"no longer part of Archipelago."),
"elapsed_seconds": 0,
}))
return 1
custom_games = [args.game]
else:
custom_games = [
name for name, world_type in AutoWorldRegister.world_types.items()
if str(getattr(world_type, "zip_path", "") or "") == args.apworld
]
if len(custom_games) != 1:
# Its own outcome, NOT "passed". This covers several distinct
# causes (a duplicate game name against a now-built-in world, an
# import-time crash, a file that isn't a valid apworld at all,
# a genuine collision between two unrelated apworlds) - none of
# them necessarily an apworld defect, but none of them a world
# this pipeline has verified either. Calling it "passed" promoted
# files that never loaded and crash-looped real WebHost startup
# Factorio Platformer did exactly that: not a valid zip, zero
# worlds loaded, recorded as a pass, promoted, and crash-looping
# real WebHost startup - it needed a hand-written blacklist entry
# to undo. promote.py takes "passed" alone, so this verdict now
# keeps such a file out of worlds_live/ on its own.
summary, traceback_text = describe_load_failure(load_log.getvalue(), args.apworld)
if len(custom_games) > 1:
detail = (f"this file registers {len(custom_games)} worlds, not one: "
f"{', '.join(custom_games)}. Only a single-world apworld can be "
f"promoted, since the pipeline cannot tell which one this row means.")
elif summary:
detail = f"the world failed to import: {summary}"
if traceback_text:
detail += f"\n\n{traceback_text.strip()}"
else:
# No captured traceback: the file is not a loadable apworld at
# all (not a zip, wrong layout), so nothing was even attempted.
detail = (f"no world loaded from this file and no import was attempted - it is "
f"probably not a valid .apworld (failed_world_loads="
f"{worlds.failed_world_loads})")
print(json.dumps({
"game": None,
"outcome": "unverified",
"detail": detail,
"elapsed_seconds": 0,
}))
return 1
game = custom_games[0]
Path(args.output_dir).mkdir(parents=True, exist_ok=True)
def run_mode(games, options_per_game=None, seed=None, single_player=None, randomized=False):
if single_player is None:
single_player = len(games) == 1
mode_results, timed_out, elapsed = run_with_timeout(
games, args.output_dir, args.timeout,
options_per_game=options_per_game, seed=seed, spoiler=args.spoiler)
outcome, detail = classify(mode_results, timed_out, elapsed,
single_player=single_player, randomized=randomized)
return {"outcome": outcome, "detail": detail, "elapsed_seconds": round(elapsed, 1),
"timed_out": timed_out}
# A timeout anywhere abandons the rest of this apworld. One world that
# hangs would otherwise cost timeout x repeats x modes - by far the
# largest thing in a run - and a world that cannot finish one seed has
# nothing useful to say about its stability across ten.
state = {"timed_out": False}
def skipped(reason):
return {"outcome": "unknown", "detail": reason, "elapsed_seconds": None}
def run_repeated(players, repeats, single_player=None, options=None, randomized=False):
"""The same test several times, reported as a pass rate.
Stability is not something a single generation can tell you: S, D
and M each get a different Archipelago seed, so the fill differs
every time, and a world that places items successfully four times
in five is genuinely unstable. Running once records whichever
result we happened to get.
Each repeat uses an explicit seed so a failure among ten passes is
reproducible - without it the interesting case is the one that
cannot be re-examined.
"""
attempts = []
for _ in range(max(repeats, 1)):
seed = random.randint(0, 2 ** 31 - 1)
# players is a callable so the multi-game test can draw a
# different set of companions every attempt - ten runs against
# one fixed pair only ever tests that pair.
games = players() if callable(players) else players
# A callable, so the randomized modes can roll a different
# point in the option space for every attempt - repeating one
# roll would only re-measure the same combination.
per_game = options() if callable(options) else options
result = run_mode(games, options_per_game=per_game, seed=seed,
single_player=single_player, randomized=randomized)
result["seed"] = seed
if len(games) > 1:
result["companions"] = games[1:]
attempts.append(result)
if result["timed_out"]:
state["timed_out"] = True
break
passed = [a for a in attempts if a["outcome"] == "passed"]
failed = [a for a in attempts if a["outcome"] != "passed"]
if not failed:
outcome, detail = "passed", f"generated {len(passed)}/{len(attempts)} times"
elif not passed:
# Every attempt agreeing on a non-failure reason keeps that
# reason. A world that needs a base ROM reports needs_input
# once and "failed all 10 attempts" ten times over, and
# flattening those to "failed" blames the world for a file
# this pipeline does not have - which matters most for core
# worlds, where roughly seven of them need a ROM and would
# otherwise all read as Broken.
outcomes = {attempt["outcome"] for attempt in attempts}
outcome = outcomes.pop() if len(outcomes) == 1 else "failed"
detail = f"failed all {len(attempts)} attempts. {failed[0]['detail']}"
if outcome != "failed":
detail = (f"{outcome} on all {len(attempts)} attempts. "
f"{failed[0]['detail']}")
else:
# The whole point of repeating: a world that works sometimes
# is worse for a player than one that never does, because
# nobody knows to avoid it.
outcome = "flaky"
detail = (f"generated {len(passed)}/{len(attempts)} times. "
f"First failure (seed {failed[0]['seed']}): {failed[0]['detail']}")
return {
"outcome": outcome,
"detail": detail,
"elapsed_seconds": round(sum(a["elapsed_seconds"] or 0 for a in attempts), 1),
"attempts": len(attempts),
# What was asked for, as opposed to what ran - a timeout cuts
# a mode short, and the retest cache compares against this so
# such a record is not re-run forever.
"requested_attempts": max(repeats, 1),
"passed_attempts": len(passed),
"failed_seeds": [a["seed"] for a in failed],
# Which companions each failing attempt drew, so a failure
# caused by one particular combination can be reproduced.
"failed_companions": [a.get("companions") for a in failed if a.get("companions")],
}
tests = {}
# 1. the world on its own, every option at its default.
tests["single"] = run_repeated([game], args.repeats)
tests["single"]["label"] = "Single game"
# 2. the same world twice. A world that keeps state on the class
# rather than the instance works alone and corrupts itself here -
# a bug class no other mode can see.
tests["duplicate"] = (skipped("skipped - an earlier test timed out") if state["timed_out"]
else run_mode([game, game], single_player=False))
tests["duplicate"]["label"] = "Two players, same game"
# 3. the world alongside other games. Each attempt draws a fresh
# random selection of core worlds rather than reusing one fixed
# pair: ten runs against the same two companions only ever tell you
# about those two, while a different combination each time samples
# what a real multiworld looks like.
configured_pool = [name.strip() for name in args.companion_pool.split(",") if name.strip()]
if configured_pool:
# Verified by the host once per run: every one of these generated
# a seed on its own in this exact environment, so a multi-game
# failure is attributable to the world under test rather than to
# a companion that was never going to work.
pool = [name for name in configured_pool if name != game]
else:
pool = core_worlds(exclude=game)
low = max(args.companion_min, 1)
high = max(args.companion_max, low)
def draw_companions():
count = min(random.randint(low, high), len(pool))
return random.sample(pool, count)
if state["timed_out"]:
tests["multi"] = skipped("skipped - an earlier test timed out")
elif pool:
tests["multi"] = run_repeated(
lambda: [game] + draw_companions(), args.repeats, single_player=False)
tests["multi"]["companion_pool"] = len(pool)
tests["multi"]["companion_range"] = [low, high]
tests["multi"]["detail"] += f" (against {low}-{high} core games drawn from {len(pool)})"
else:
tests["multi"] = skipped("no core worlds available to pair with")
tests["multi"]["label"] = "Multi game"
# 4 and 5. The same two shapes as above - the world alone, and the
# world alongside the companions - but with its options randomized
# instead of left at their defaults. The default option set is a
# single point in a world's option space and most option-dependent
# crashes are nowhere near it.
#
# One roll each, and every roll that does not produce a seed is a
# failure, including one the world rejects with OptionError. That
# last point is the whole reason these modes exist: the yaml a
# player writes is accepted as valid - nothing in the template or
# the upload path objects to "random" on two interdependent
# options - so the conflict only surfaces when generation is
# actually attempted, possibly after everyone has submitted, and it
# is intermittent. Re-rolling until the world accepts something
# would hide exactly the failure a real player hits.
#
# Each randomized run is gated on its own default-options
# counterpart: SR only when the world generates alone, MR only when
# it generates alongside the companions. A randomized run whose
# plain equivalent already fails would fail for the same underlying
# reason and cost a generation to learn nothing.
rolls = None
if state["timed_out"]:
aborted = "skipped - an earlier test timed out"
tests["single_random"] = skipped(aborted)
tests["multi_random"] = skipped(aborted)
elif tests["single"]["outcome"] == "passed" or tests["multi"]["outcome"] == "passed":
try:
# One roll per attempt, drawn up front so SR and MR run the
# SAME combinations as each other. Pairing them is what makes
# "SR passed but MR failed" attributable to the extra players
# rather than to two different option sets.
rolls = [random_options(game) for _ in range(max(args.random_repeats, 1))]
except BaseException as error:
rolls = None
unreadable = f"could not read this world's options: {type(error).__name__}: {error}"
tests["single_random"] = skipped(unreadable)
tests["multi_random"] = skipped(unreadable)
if rolls:
def roller():
# Walks the pre-drawn rolls in order, so attempt N of SR and
# attempt N of MR use the same option values.
supply = iter(rolls)
def next_roll():
return {game: next(supply)}
return next_roll
# "flaky" counts as generating: the world produced a seed at
# least once, so a randomized run can still tell us something.
if tests["single"]["outcome"] in ("passed", "flaky"):
result = run_repeated([game], len(rolls), single_player=True,
options=roller(), randomized=True)
result["randomized_options"] = len(rolls[0])
tests["single_random"] = result
else:
tests["single_random"] = skipped(
"skipped - the single-game test did not pass")
if tests["multi"]["outcome"] in ("passed", "flaky"):
result = run_repeated(lambda: [game] + draw_companions(), len(rolls),
single_player=False, options=roller(), randomized=True)
result["randomized_options"] = len(rolls[0])
tests["multi_random"] = result
else:
tests["multi_random"] = skipped(
"skipped - the multi-game test did not pass")
elif "single_random" not in tests:
neither = "skipped - neither the single-game nor the multi-game test passed"
tests["single_random"] = skipped(neither)
tests["multi_random"] = skipped(neither)
tests["single_random"]["label"] = "Random options, alone"
tests["multi_random"]["label"] = "Random options, multi game"
# The top-level fields stay exactly what they were - the single-player
# verdict - because that is what promotion and every existing consumer
# reads. The per-mode results live alongside it.
report = {
"game": game,
"outcome": tests["single"]["outcome"],
"detail": tests["single"]["detail"],
"elapsed_seconds": tests["single"]["elapsed_seconds"],
"multi_outcome": tests["multi"]["outcome"],
"multi_detail": tests["multi"]["detail"],
"multi_elapsed_seconds": tests["multi"]["elapsed_seconds"],
"multi_companions": tests["multi"].get("companions", []),
"tests": tests,
}
print(json.dumps(report))
# Exit code still reflects the single-player result alone: it is what
# gates promotion, and the other modes are reported rather than
# blocking.
return {"passed": 0, "needs_input": 2}.get(tests["single"]["outcome"], 1)
if __name__ == "__main__":
sys.exit(main())

65
container/drivers/verify_companions.py Normal file
View File

@@ -0,0 +1,65 @@
"""Runs inside the archipelago Docker image. Reports which core worlds
can actually generate a seed in THIS environment, so the multi-game tests
only ever draw companions that work.
Core worlds are shipped with Archipelago, but that does not make them all
usable here: some need a base ROM this container has no copy of, and at
least one (Final Fantasy) needs a settings yaml generated by an external
website and refuses a generic one. Drawing such a world as a companion
fails the whole seed, and the failure gets recorded against the apworld
under test rather than the companion that caused it - Blender was marked
flaky 7/10 purely because three of its ten draws included worlds like
these.
Rather than maintain an exclusion list by hand, this generates one solo
seed per core world and keeps the ones that succeed. A world drops out
by itself when its ROM is missing, and comes back by itself once the ROM
is added.
Shares run_test.py's generation machinery rather than repeating it, so
"can generate" means exactly the same thing in both places.
"""
import argparse
import json
import sys
import warnings
sys.path.insert(0, "/app")
import run_test # noqa: E402 - mounted alongside this script
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output-dir", required=True)
parser.add_argument("--timeout", type=int, default=120,
help="Per-world timeout. Deliberately shorter than the apworld "
"timeout: a companion that is slow to generate would multiply "
"across every multi-game attempt that draws it.")
parser.add_argument("--spoiler", type=int, default=2)
args = parser.parse_args()
warnings.simplefilter("ignore")
run_test.ensure_tutorials_default()
from pathlib import Path
Path(args.output_dir).mkdir(parents=True, exist_ok=True)
verified = []
rejected = {}
for game in run_test.core_worlds(exclude=None):
results, timed_out, elapsed = run_test.run_with_timeout(
[game], args.output_dir, args.timeout, spoiler=args.spoiler)
outcome, detail = run_test.classify(results, timed_out, elapsed)
if outcome == "passed":
verified.append(game)
else:
rejected[game] = f"{outcome}: {detail}"[:300]
print(json.dumps({"verified": verified, "rejected": rejected}))
return 0
if __name__ == "__main__":
sys.exit(main())

66
notebooks/tester_playground.ipynb Normal file
View File

@@ -0,0 +1,66 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Testing worlds\n",
"\n",
"`TestSession` prints the paths it uses, downloads the apworld if it isn't\n",
"there yet, then runs all five generation modes and prints the verdicts.\n",
"\n",
"Results go to `scratch_state.json` under the data directory, not the live\n",
"`state.json` - pass `live=True` when a result is meant to count.\n",
"\n",
"Needs the package installed: `pip install -e .` from the repository root."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## One world"
]
},
{
"cell_type": "code",
"metadata": {},
"execution_count": null,
"outputs": [],
"source": [
"from archipelago_tester.pipeline.batch.session import TestSession\n",
"\n",
"TestSession(\"Baba Is You\").run()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Several worlds"
]
},
{
"cell_type": "code",
"metadata": {},
"execution_count": null,
"outputs": [],
"source": [
"TestSession([\"Baba Is You\", \"Anodyne\", \"Axiom Verge\"]).run()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python",
"pygments_lexer": "ipython3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

30
pyproject.toml Normal file
View File

@@ -0,0 +1,30 @@
[build-system]
requires = ["setuptools>=61"]
build-backend = "setuptools.build_meta"
[project]
name = "APWorldTester"
version = "0.1.0"
description = "Tests Archipelago community worlds by generating seeds with them"
readme = "README.md"
license = "MIT"
license-files = ["LICENSE"]
requires-python = ">=3.10"
dependencies = [
"requests",
"beautifulsoup4",
"pyyaml",
]
[project.scripts]
apworld-tester = "apworld_tester.cli:main"
[tool.setuptools.packages.find]
where = ["src"]
[tool.setuptools.package-data]
apworld_tester = [
"templates/*.yaml",
"container/Dockerfile.test",
"container/drivers/*.py",
]

55
src/apworld_tester/__init__.py Normal file
View File

@@ -0,0 +1,55 @@
"""Tests Archipelago community worlds by generating seeds with them.
The publishing half - archipelago-world-site - imports this package as a
dependency. Nothing here knows that it exists.
Everything downstream needs is re-exported here, so callers import from
``apworld_tester`` and never from the module layout underneath it.
"""
from apworld_tester.core.config.config import Config
from apworld_tester.core.config.paths import Paths
from apworld_tester.core.model.apworld import ApworldFile
from apworld_tester.core.model.blacklist import Blacklist
from apworld_tester.core.model.fingerprint import Fingerprint
from apworld_tester.core.model.game import Game
from apworld_tester.core.model.instant import Instant
from apworld_tester.core.model.modes import Modes
from apworld_tester.core.model.name import Name
from apworld_tester.core.model.verdict import Verdict
from apworld_tester.core.state.keys import StateKeys
from apworld_tester.core.state.lock import LockHeldError, PipelineLock
from apworld_tester.core.state.store import StateStore
from apworld_tester.pipeline.batch.run import TestRun
from apworld_tester.pipeline.build.archipelago import ArchipelagoBuild
from apworld_tester.pipeline.build.docker import Docker
from apworld_tester.pipeline.download.update_history import UpdateHistory
from apworld_tester.pipeline.generation.command import ContainerCommand
from apworld_tester.pipeline.sheet.reader import SheetReader
from apworld_tester.pipeline.update.stage_error import StageError
from apworld_tester.pipeline.update.update_run import UpdateRun
__all__ = [
"ApworldFile",
"ArchipelagoBuild",
"Blacklist",
"Config",
"ContainerCommand",
"Docker",
"Fingerprint",
"Game",
"Instant",
"LockHeldError",
"Modes",
"Name",
"Paths",
"PipelineLock",
"SheetReader",
"StageError",
"StateKeys",
"StateStore",
"TestRun",
"UpdateHistory",
"UpdateRun",
"Verdict",
]

101
src/apworld_tester/cli.py Normal file
View File

@@ -0,0 +1,101 @@
"""The command line: getting a config file into place.
The package is installed rather than cloned, so the starter config
cannot be "the example beside this script" - it is shipped as package
data and copied out on request.
"""
import argparse
import os
import shutil
import sys
from importlib import resources
from apworld_tester.core.config.config import Config, ConfigNotFound
class ConfigTemplate:
"""The starter config.yaml carried inside the package."""
PACKAGE = "apworld_tester.templates"
NAME = "config.yaml"
@classmethod
def read(cls):
return (resources.files(cls.PACKAGE) / cls.NAME).read_text(
encoding="utf-8")
class InitCommand:
"""Write a starter config where the search order will find it."""
def __init__(self, destination=None, force=False):
self.destination = destination or Config.user_path()
self.force = force
def run(self):
if os.path.exists(self.destination) and not self.force:
print(f"config already exists: {self.destination}")
print("pass --force to overwrite it")
return 1
os.makedirs(os.path.dirname(self.destination), exist_ok=True)
if os.path.exists(self.destination):
shutil.copyfile(self.destination, f"{self.destination}.bak")
print(f"kept the old one as {self.destination}.bak")
with open(self.destination, "w", encoding="utf-8") as handle:
handle.write(ConfigTemplate.read())
print(f"wrote {self.destination}")
print("Edit general.output_directory and general.roms_directory "
"before the first run.")
return 0
class PathCommand:
"""Say which config would be used, and what was tried."""
def run(self):
try:
print(Config.default_path())
return 0
except ConfigNotFound as error:
print(error, file=sys.stderr)
return 1
class Cli:
"""One entry point, so an installed copy is self-describing."""
@staticmethod
def parser():
parser = argparse.ArgumentParser(
prog="apworld-tester", description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
init = commands.add_parser(
"init", help="Write a starter config.yaml and say where.")
init.add_argument(
"--path", default=None,
help="Write here instead of the user config directory.")
init.add_argument(
"--force", action="store_true",
help="Overwrite an existing config, keeping a .bak copy.")
commands.add_parser(
"config-path",
help="Print the config that would be used, or where it "
"was looked for.")
return parser
@classmethod
def main(cls, argv=None):
args = cls.parser().parse_args(argv)
if args.command == "init":
return InitCommand(args.path, args.force).run()
return PathCommand().run()
def main(argv=None):
return Cli.main(argv)
if __name__ == "__main__":
sys.exit(main())

20
src/apworld_tester/container/Dockerfile.test Normal file
View File

@@ -0,0 +1,20 @@
# Unprivileged wrapper for the generation tests.
#
# Upstream's image runs as root and leaves /.local root-owned, so running
# it as the host user fails before generation starts (PermissionError on
# /.local) and running it as root leaves root-owned seeds in the mounted
# output directory. Two lines fix both, and they are the only difference
# from the base image.
#
# Built once per Archipelago tag, not per apworld test, so the cost of
# the recursive chown is paid once and amortised over a few hundred
# generations.
ARG BASE_IMAGE=archipelago:latest
FROM ${BASE_IMAGE}
ARG RUNNER_UID=1000
ARG RUNNER_GID=1000
RUN chown -R ${RUNNER_UID}:${RUNNER_GID} /app /.local 2>/dev/null || chown -R ${RUNNER_UID}:${RUNNER_GID} /app
USER ${RUNNER_UID}:${RUNNER_GID}

49
src/apworld_tester/container/drivers/identify_apworld.py Normal file
View File

@@ -0,0 +1,49 @@
"""Runs inside the archipelago Docker image. Reports which game (if any)
a single mounted .apworld registers itself as under Archipelago's own
World registry - the import/registration step only, no seed generation.
Used to verify a freshly downloaded .apworld's actual content matches
the sheet row that fetched it, rather than trusting release/asset
name-matching alone (see AssetMatcher): the filename
and release title are just text an author chose, but AutoWorldRegister
is filled from the World class's own "game" attribute, which is what
Archipelago itself will treat this file as - the authoritative source.
"""
import argparse
import json
import sys
import warnings
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--apworld", required=True)
args = parser.parse_args()
warnings.simplefilter("ignore")
try:
import worlds
from worlds.AutoWorld import AutoWorldRegister
except BaseException as error:
# The apworld (or something it imports) crashed outright - that's
# itself a meaningful signal (this file is broken/wrong), not an
# infrastructure failure, so it's still reported as a clean JSON
# result rather than a non-zero exit / stack trace.
print(json.dumps({"games": [], "detail": f"{type(error).__name__}: {error}"}))
return 0
games = [
name for name, world_type in AutoWorldRegister.world_types.items()
if str(getattr(world_type, "zip_path", "") or "") == args.apworld
]
print(json.dumps({
"games": games,
"detail": None if games else f"failed_world_loads={worlds.failed_world_loads}",
}))
return 0
if __name__ == "__main__":
sys.exit(main())

827
src/apworld_tester/container/drivers/run_test.py Normal file
View File

@@ -0,0 +1,827 @@
"""Runs inside the archipelago Docker image. Generates one seed for a
single installed apworld and reports the outcome as one JSON line on
stdout: passed, needs_input (missing base ROM or similar), or failed.
The logic mirrors Archipelago's own test/hosting/generate.py, but is
copied here rather than imported from it: the project's .dockerignore
excludes test/ from the image, so that module isn't present at runtime.
"""
import argparse
import io
import json
import logging
import random
import re
import shutil
import sys
import unicodedata
import time
import warnings
from multiprocessing import Manager, Process, set_start_method
from pathlib import Path
def ensure_tutorials_default():
# Monkeypatch applied AFTER whichever Archipelago version's own real
# worlds/AutoWorld.py has already loaded normally - defaults
# WebWorld.tutorials to [] if that core doesn't already declare it,
# without needing to know or duplicate anything else about that
# core's own AutoWorld.py content. Previously this was a full-file
# bind-mount replacement (container/AutoWorld.py (absent - see config.py)) built against one
# specific version's source; that broke outright when tested against
# an older Archipelago release whose real AutoWorld.py imports
# differently (e.g. no rule_builder module before it was added
# upstream) - replacing the whole file assumes it stays
# source-compatible with everything else that release's core
# expects, which isn't true across versions. This achieves the same
# fix (see the original container/AutoWorld.py (absent - see config.py) comment for why it's
# needed - WebHost.py's invalid_worlds filter vs.
# network_data_package snapshot timing) without that assumption.
from worlds.AutoWorld import WebWorld
if "tutorials" not in vars(WebWorld):
WebWorld.tutorials = []
class WebHostIncompatibleError(RuntimeError):
"""Generation succeeds, but this world would crash WebHost's own
startup - a bug class Generate.py never exercises (see
check_webhost_compatibility below)."""
def check_webhost_compatibility(game):
# Reproduces WebHost's own generate_yaml_templates() (Options.py),
# scoped to just this one world, so a bug like Option.visibility
# being set to a bare int instead of a Visibility flag - which
# crashes generate_yaml_templates() for the ENTIRE deployed site,
# not just the offending world, since it iterates every visible
# world in one loop - is caught here as a normal test failure
# instead of only in production. (A hand-run diagnostic that located
# the offending worlds used to live in container/tools/; removed.)
# which found this bug class live on the deployed instance.
from inspect import cleandoc
import yaml
from jinja2 import Template
import Options
from Utils import local_path, __version__
from worlds.AutoWorld import AutoWorldRegister
world = AutoWorldRegister.world_types[game]
if not hasattr(world.web, "tutorials"):
# WebHost.py strips any world missing this from AutoWorldRegister
# entirely at startup (its invalid_worlds set) - so it can't
# crash generate_yaml_templates (never part of that loop), but
# the world is now gone from WebHost's registry altogether. That
# breaks something worse than templates: MultiServer can't find
# the game's data (item_name_groups etc.) when starting a room,
# so *hosting* any seed generated with this world crashes with
# a KeyError - even though generation itself succeeded fine.
raise WebHostIncompatibleError(
"passes generation but has no WebWorld.tutorials - WebHost "
"strips it from its world registry entirely at startup, so "
"hosting a room for a seed generated with it crashes "
"MultiServer with a KeyError on the game name"
)
def dictify_range(option, option_val):
data = {option_val: 50}
for sub_option in ["random", "random-low", "random-high",
f"random-range-{option.range_start}-{option.range_end}"]:
if sub_option != option_val:
data[sub_option] = 0
notes = {}
for name, number in getattr(option, "special_range_names", {}).items():
notes[name] = f"equivalent to {number}"
if number in data:
data[name] = data[number]
del data[number]
elif name in data:
pass
else:
data[name] = 0
return data, notes
def yaml_dump_scalar(scalar):
return yaml.dump(scalar).replace("...\n", "").strip()
with open(local_path("data", "options.yaml")) as f:
template = Template(f.read())
try:
option_groups = Options.get_option_groups(world)
presets = world.web.options_presets.copy()
presets.update({"": {}})
for name, preset in presets.items():
template.render(
option_groups=option_groups,
__version__=__version__,
game=game,
world_version=world.world_version.as_simple_string(),
yaml_dump=yaml_dump_scalar,
dictify_range=dictify_range,
cleandoc=cleandoc,
preset_name=name,
preset=preset,
)
except Exception as error:
raise WebHostIncompatibleError(
f"passes generation but would crash WebHost's own startup "
f"(generate_yaml_templates): {type(error).__name__}: {error}"
) from error
def core_worlds(exclude):
"""Every world Archipelago ships, minus the one under test.
Core worlds are the safe companions: they are part of the build
itself, so they are always present and always match the running
version - unlike an apworld, which might be missing or broken for
reasons that have nothing to do with the world being tested.
"""
from worlds.AutoWorld import AutoWorldRegister
return sorted(
name for name, world_type in AutoWorldRegister.world_types.items()
if not str(getattr(world_type, "zip_path", "") or "") and name != exclude
)
def random_options(game):
"""One concrete random value per option of `game`.
Only the option types with a defined value space are randomized -
toggles, choices and ranges. Free text, option lists/sets and item
dicts (start inventory, plando, exclusions...) have no meaningful
"random", and forcing one would fail generation for reasons that have
nothing to do with the world being tested.
The values are RESOLVED here rather than written into the yaml as the
string "random". Both randomized modes then run the exact same option
values, so a difference between them is attributable to the extra
players and nothing else - writing "random" would have each mode roll
its own values from its own seed, and SR-passes-while-MR-fails would
say nothing about the multiworld.
Resolution goes through the option's own from_text("random"), so the
values are exactly the ones Archipelago itself would roll: weighted
ranges, named-range specials and all.
"""
from worlds.AutoWorld import AutoWorldRegister
import Options
randomizable = (Options.Toggle, Options.Choice, Options.Range)
world_type = AutoWorldRegister.world_types[game]
chosen = {}
for name, option in getattr(world_type.options_dataclass, "type_hints", {}).items():
if not issubclass(option, randomizable):
continue
try:
chosen[name] = option.from_text("random").value
except Exception:
# A world can define an option whose randomization raises;
# leaving it out means it keeps its default rather than
# failing the whole mode for one unrollable option.
continue
return chosen
def generate_one(games, dest, results, options_per_game=None, seed=None, spoiler=3):
# games is the full player list: the world under test first, then any
# companions (see RunSettings). One entry is the single-player
# test, the same game twice is the duplicate test, and the world plus
# companions is the multi-game test - all the same code path, differing
# only in who is in the seed and what options they roll.
warnings.simplefilter("ignore")
try:
from tempfile import TemporaryDirectory
import Generate
import Main
ensure_tutorials_default()
with TemporaryDirectory() as players_dir, TemporaryDirectory() as output_dir:
for number, game in enumerate(games, start=1):
player_path = Path(players_dir) / f"{number}.yaml"
# {} means every option at its default - the only point in
# option space the pipeline used to cover.
game_options = (options_per_game or {}).get(game, {})
player_path.write_text(json.dumps({
"name": f"Tester{number}",
"game": game,
game: game_options,
"description": f"ArchiUpdater compatibility test: {game}",
}), encoding="utf-8")
sys.argv = [
sys.argv[0],
"--player_files_path", players_dir,
"--outputpath", output_dir,
# Spoiler 3 computes the full playthrough, which exercises
# accessibility/reachability logic that plain generation
# never reaches - "a seed came out" becomes "the seed is
# actually completable".
"--spoiler", str(spoiler),
]
if seed is not None:
sys.argv += ["--seed", str(seed)]
Main.main(*Generate.main())
check_webhost_compatibility(games[0])
output_files = list(Path(output_dir).glob("*.zip"))
if len(output_files) != 1:
raise RuntimeError(
f"expected exactly one output file, found {len(output_files)}")
final_file = Path(dest) / output_files[0].name
# rename() can't cross the tmpdir -> bind-mounted output volume
# boundary, so move (copy + delete) instead.
shutil.move(str(output_files[0]), str(final_file))
results.append(str(final_file))
except BaseException as error:
results.append(error)
raise
def describe_load_failure(log_text, apworld_path):
"""The real reason an apworld did not load, out of the swallowed log.
worlds/__init__.py catches an import failure per world, formats the
traceback, and hands it to logging.exception - so by default it goes
nowhere and the pipeline could only report "found 0 worlds", which
says nothing an author could act on. Untitled Goose Game, for
instance, defines `option_random` on a Choice, which Archipelago
reserves; the assert fires at class-definition time during import.
Returns (summary, traceback) - summary being the exception and the
line in the world's own code that raised it.
"""
marker = "Could not load world"
TRACEBACK_HEADER = "Traceback (most recent call last):"
blocks = [block for block in log_text.split(marker) if apworld_path in block]
if not blocks:
return None, None
block = marker + blocks[-1]
lines = [line for line in block.splitlines() if line.strip()]
# The exception itself is the last line of a traceback.
exception = next((line.strip() for line in reversed(lines)
if line.strip() and not line.startswith((" ", "\t"))
and "Traceback" not in line and marker not in line), None)
# The deepest frame inside the apworld itself - core frames above it
# are just the import machinery and tell an author nothing.
where = None
for line in lines:
match = re.search(r'File "([^"]*%s[^"]*)", line (\d+)' % re.escape(apworld_path), line)
if match:
inside = match.group(1).split(apworld_path + "/", 1)[-1]
where = f"{inside}, line {match.group(2)}"
summary = exception or "import failed"
if where:
summary += f" (in {where})"
# logging.exception both formats the traceback into the message AND
# appends exc_info's own copy, so the block holds it twice - keep one.
first = block.find(TRACEBACK_HEADER)
if first != -1:
second = block.find(TRACEBACK_HEADER, first + len(TRACEBACK_HEADER))
if second != -1:
block = block[:second]
# Drop the import machinery. Every one of these frames is identical
# for every failing world and tells an author nothing; what is left is
# their own code and the line that actually raised.
# A frame is its File line plus the indented source and caret lines
# under it, so drop the whole frame - removing only the File line
# leaves orphaned source fragments behind.
noise = ("<frozen importlib", "importlib/__init__.py", "worlds/__init__.py", "_bootstrap")
kept = []
skipping = False
for line in block.splitlines():
stripped = line.lstrip()
if stripped.startswith("File "):
skipping = any(n in line for n in noise)
if skipping:
continue
elif skipping:
# Continuation of a dropped frame: still indented, and not the
# start of the next frame.
if line.startswith(" ") or not line.strip():
continue
skipping = False
kept.append(line)
return summary, "\n".join(kept)
def rooted_in(error, exception_type):
seen = set()
while error is not None and id(error) not in seen:
seen.add(id(error))
if isinstance(error, exception_type):
return True
error = error.__cause__ or error.__context__
return False
def rooted_in_named(error, class_name):
# Some apworlds define their own exception class inside their own
# zipimported package (e.g. CTJoT's InvalidYamlException) rather
# than a stable top-level module - there's no fixed path to import
# and isinstance-check it the way Fill.FillError works, so match by
# class name instead.
seen = set()
while error is not None and id(error) not in seen:
seen.add(id(error))
if type(error).__name__ == class_name:
return True
error = error.__cause__ or error.__context__
return False
def run_with_timeout(games, dest, timeout, options_per_game=None, seed=None, spoiler=3):
# fork, not spawn. Every mode runs in its own process so a generation
# can be killed on timeout and cannot leak state into the next one -
# but spawn re-imports the whole of Archipelago in each child, five
# times per apworld, which measured as the single largest cost in a
# run. Forking inherits the import the parent already did.
#
# The isolation that matters is unchanged: the child still gets its
# own copy-on-write memory, so whatever generation mutates dies with
# it and the parent's registry stays pristine for the next mode.
#
# Safe here specifically because the forking process is
# single-threaded: main() starts no threads of its own, and the
# Manager below is a separate process rather than a thread in this
# one. Forking a multi-threaded parent is the case to avoid, and this
# is not one.
try:
set_start_method("fork")
except RuntimeError:
pass
manager = Manager()
results = manager.list()
process = Process(target=generate_one,
args=(games, dest, results, options_per_game, seed, spoiler))
started = time.monotonic()
process.start()
process.join(timeout)
timed_out = process.is_alive()
if timed_out:
process.terminate()
process.join(5)
if process.is_alive():
process.kill()
process.join()
elapsed = time.monotonic() - started
return list(results), timed_out, elapsed
def classify(results, timed_out, elapsed, single_player=True, randomized=False):
if timed_out:
return "failed", f"timed out after {elapsed:.0f}s"
if not results:
return "failed", "generation process exited without a result"
result = results[0]
if isinstance(result, BaseException):
if rooted_in(result, FileNotFoundError):
return "needs_input", str(result)
if randomized and rooted_in_named(result, "OptionError"):
# The world rejected the option combination we rolled - not a
# crash and not a fill failure, but the world validating its
# own input. Archipelago rolls every option independently, so
# a world with interdependent options (Blender's min/max
# similarity percent, for example) can always be handed a
# combination it considers invalid. A player writing
# "random" for both hits exactly this, so it is worth
# reporting - but as its own thing, not as a defect.
#
# Only when the options were randomized: a world that rejects
# its OWN DEFAULTS is genuinely broken and stays "failed".
return "invalid_options", f"rejected the rolled options: {result}"
from Fill import FillError
if rooted_in(result, FillError):
# A fill failure is a failure in both modes. This used to be
# excused in single-player on the theory that some games'
# item/location balance "only works out in a real multiworld"
# - the multi-game test exists precisely to check that theory,
# and it does not hold (Dead Cells fails both ways), so the
# excuse is gone: a seed that cannot be filled is a seed that
# cannot be generated, whoever else is in it.
where = "a multi-game seed" if not single_player else "a single-player seed"
return "failed", f"fill failed in {where}: {result}"
if rooted_in_named(result, "InvalidYamlException"):
# Some worlds (e.g. CTJoT) require a yaml pre-generated by an
# external tool and reject our generic single-player yaml
# outright - not a defect, just incompatible with this
# testing methodology.
return "passed", f"requires an externally-generated yaml (not a defect): {result}"
return "failed", f"{type(result).__name__}: {result}"
return "passed", str(result)
def resolve_core_game(name, world_types):
"""Match a sheet name to a world Archipelago actually registers.
The sheet and the worlds disagree about wording often enough that an
exact comparison finds only 72 of the 81 core rows. The rest differ
by an accent ("Pokemon Emerald"), a hyphen ("Choo-Choo Charles"),
capitalisation ("EarthBound"), or a series prefix the sheet adds and
the world does not ("The Legend of Zelda: Ocarina of Time").
Tried in order: the name as written, then folded to letters and
digits, then either side of a colon folded the same way. Either side,
because the sheet puts the distinguishing part before the colon as
often as after it - "Super Mario Land 2: The Golden Coins" is
registered as "Super Mario Land 2", while "The Legend of Zelda:
Ocarina of Time" is registered as "Ocarina of Time".
"""
if name in world_types:
return name
by_slug = {}
for registered in world_types:
by_slug.setdefault(slug(registered), registered)
candidates = [name]
if ":" in name:
head, tail = name.split(":", 1)
candidates += [tail, head]
for candidate in candidates:
registered = by_slug.get(slug(candidate))
if registered:
return registered
return None
# Deliberate copy of Name.slug in core/model/name.py. This file is mounted
# alone into the container, with no access to this checkout's
# packages, so it cannot import it - but the two must stay identical
# or a world folds to one key here and a different one on the host.
def slug(text):
return re.sub(r"[^a-z0-9]", "", unicodedata.normalize("NFKD", (text or "").lower()))
def main():
parser = argparse.ArgumentParser(description=__doc__)
# Exactly one of these. --apworld is the normal case: a downloaded
# file, identified by which world claims it as its zip_path. --game
# names a world that is already part of the Archipelago build, which
# has no file to point at and so cannot be identified that way.
parser.add_argument("--apworld")
parser.add_argument("--game", help="Test a world that ships with Archipelago, by name, "
"instead of an apworld file. Mutually exclusive with --apworld.")
parser.add_argument("--output-dir", required=True)
parser.add_argument("--timeout", type=int, default=300)
parser.add_argument("--companion-pool", default="",
help="Comma-separated core games to draw companions from. Empty "
"means every core world, which risks drawing one that cannot "
"generate here (see container/verify_companions.py).")
parser.add_argument("--companion-min", type=int, default=2,
help="Fewest core games to draw as companions for a multi-game seed.")
parser.add_argument("--companion-max", type=int, default=5,
help="Most core games to draw as companions for a multi-game seed.")
parser.add_argument("--random-repeats", type=int, default=3,
help="How many times the randomized-option modes run. Each attempt "
"rolls a different option combination.")
parser.add_argument("--repeats", type=int, default=10,
help="How many times to run the single-game and multi-game tests. "
"Stability is a pass rate, not a single verdict.")
parser.add_argument("--spoiler", type=int, default=3,
help="Spoiler level passed to Generate. 3 computes the full "
"playthrough, exercising accessibility logic.")
args = parser.parse_args()
if bool(args.apworld) == bool(args.game):
parser.error("pass exactly one of --apworld or --game")
# Determine the game name from the loaded World class itself rather
# than the apworld's archipelago.json manifest: older apworlds don't
# ship a manifest at all, and the core (<0.7.0 here) tolerates that
# and loads them anyway - a manifest-only lookup would misreport a
# perfectly loadable world as failed. World.zip_path is set only for
# worlds loaded from a .apworld zip (worlds/AutoWorld.py), and equals
# the apworld's own path, so it identifies our one mounted world
# exactly regardless of manifest presence.
# worlds/__init__.py logs each failed world's traceback and carries
# on; without a handler here that detail is lost and all the pipeline
# can say is "found 0 worlds".
load_log = io.StringIO()
log_handler = logging.StreamHandler(load_log)
log_handler.setLevel(logging.ERROR)
logging.getLogger().addHandler(log_handler)
try:
import worlds
from worlds.AutoWorld import AutoWorldRegister
finally:
logging.getLogger().removeHandler(log_handler)
ensure_tutorials_default()
if args.game:
# A core world is already loaded - there is no file to identify
# it by, and none of the load-failure reporting below applies. It
# either exists in this build or it does not, and "does not" is a
# naming problem on the sheet rather than a defect in a world.
resolved = resolve_core_game(args.game, AutoWorldRegister.world_types)
if resolved:
args.game = resolved
else:
# "unknown", not "unverified". Nothing about this world is
# broken - this Archipelago build simply has no world under
# that name, which is a naming mismatch on the sheet (or a
# world that has since been removed from core), and calling
# it Broken would blame a world for the sheet's wording.
print(json.dumps({
"game": None,
"outcome": "unknown",
"detail": (f"this Archipelago build registers no world named {args.game!r}, "
f"so there was nothing to test. The sheet lists this row as Core, "
f"so either its name differs from the world's own or the world is "
f"no longer part of Archipelago."),
"elapsed_seconds": 0,
}))
return 1
custom_games = [args.game]
else:
custom_games = [
name for name, world_type in AutoWorldRegister.world_types.items()
if str(getattr(world_type, "zip_path", "") or "") == args.apworld
]
if len(custom_games) != 1:
# Its own outcome, NOT "passed". This covers several distinct
# causes (a duplicate game name against a now-built-in world, an
# import-time crash, a file that isn't a valid apworld at all,
# a genuine collision between two unrelated apworlds) - none of
# them necessarily an apworld defect, but none of them a world
# this pipeline has verified either. Calling it "passed" promoted
# files that never loaded and crash-looped real WebHost startup
# Factorio Platformer did exactly that: not a valid zip, zero
# worlds loaded, recorded as a pass, promoted, and crash-looping
# real WebHost startup - it needed a hand-written blacklist entry
# to undo. promote.py takes "passed" alone, so this verdict now
# keeps such a file out of worlds_live/ on its own.
summary, traceback_text = describe_load_failure(load_log.getvalue(), args.apworld)
if len(custom_games) > 1:
detail = (f"this file registers {len(custom_games)} worlds, not one: "
f"{', '.join(custom_games)}. Only a single-world apworld can be "
f"promoted, since the pipeline cannot tell which one this row means.")
elif summary:
detail = f"the world failed to import: {summary}"
if traceback_text:
detail += f"\n\n{traceback_text.strip()}"
else:
# No captured traceback: the file is not a loadable apworld at
# all (not a zip, wrong layout), so nothing was even attempted.
detail = (f"no world loaded from this file and no import was attempted - it is "
f"probably not a valid .apworld (failed_world_loads="
f"{worlds.failed_world_loads})")
print(json.dumps({
"game": None,
"outcome": "unverified",
"detail": detail,
"elapsed_seconds": 0,
}))
return 1
game = custom_games[0]
Path(args.output_dir).mkdir(parents=True, exist_ok=True)
def run_mode(games, options_per_game=None, seed=None, single_player=None, randomized=False):
if single_player is None:
single_player = len(games) == 1
mode_results, timed_out, elapsed = run_with_timeout(
games, args.output_dir, args.timeout,
options_per_game=options_per_game, seed=seed, spoiler=args.spoiler)
outcome, detail = classify(mode_results, timed_out, elapsed,
single_player=single_player, randomized=randomized)
return {"outcome": outcome, "detail": detail, "elapsed_seconds": round(elapsed, 1),
"timed_out": timed_out}
# A timeout anywhere abandons the rest of this apworld. One world that
# hangs would otherwise cost timeout x repeats x modes - by far the
# largest thing in a run - and a world that cannot finish one seed has
# nothing useful to say about its stability across ten.
state = {"timed_out": False}
def skipped(reason):
return {"outcome": "unknown", "detail": reason, "elapsed_seconds": None}
def run_repeated(players, repeats, single_player=None, options=None, randomized=False):
"""The same test several times, reported as a pass rate.
Stability is not something a single generation can tell you: S, D
and M each get a different Archipelago seed, so the fill differs
every time, and a world that places items successfully four times
in five is genuinely unstable. Running once records whichever
result we happened to get.
Each repeat uses an explicit seed so a failure among ten passes is
reproducible - without it the interesting case is the one that
cannot be re-examined.
"""
attempts = []
for _ in range(max(repeats, 1)):
seed = random.randint(0, 2 ** 31 - 1)
# players is a callable so the multi-game test can draw a
# different set of companions every attempt - ten runs against
# one fixed pair only ever tests that pair.
games = players() if callable(players) else players
# A callable, so the randomized modes can roll a different
# point in the option space for every attempt - repeating one
# roll would only re-measure the same combination.
per_game = options() if callable(options) else options
result = run_mode(games, options_per_game=per_game, seed=seed,
single_player=single_player, randomized=randomized)
result["seed"] = seed
if len(games) > 1:
result["companions"] = games[1:]
attempts.append(result)
if result["timed_out"]:
state["timed_out"] = True
break
passed = [a for a in attempts if a["outcome"] == "passed"]
failed = [a for a in attempts if a["outcome"] != "passed"]
if not failed:
outcome, detail = "passed", f"generated {len(passed)}/{len(attempts)} times"
elif not passed:
# Every attempt agreeing on a non-failure reason keeps that
# reason. A world that needs a base ROM reports needs_input
# once and "failed all 10 attempts" ten times over, and
# flattening those to "failed" blames the world for a file
# this pipeline does not have - which matters most for core
# worlds, where roughly seven of them need a ROM and would
# otherwise all read as Broken.
outcomes = {attempt["outcome"] for attempt in attempts}
outcome = outcomes.pop() if len(outcomes) == 1 else "failed"
detail = f"failed all {len(attempts)} attempts. {failed[0]['detail']}"
if outcome != "failed":
detail = (f"{outcome} on all {len(attempts)} attempts. "
f"{failed[0]['detail']}")
else:
# The whole point of repeating: a world that works sometimes
# is worse for a player than one that never does, because
# nobody knows to avoid it.
outcome = "flaky"
detail = (f"generated {len(passed)}/{len(attempts)} times. "
f"First failure (seed {failed[0]['seed']}): {failed[0]['detail']}")
return {
"outcome": outcome,
"detail": detail,
"elapsed_seconds": round(sum(a["elapsed_seconds"] or 0 for a in attempts), 1),
"attempts": len(attempts),
# What was asked for, as opposed to what ran - a timeout cuts
# a mode short, and the retest cache compares against this so
# such a record is not re-run forever.
"requested_attempts": max(repeats, 1),
"passed_attempts": len(passed),
"failed_seeds": [a["seed"] for a in failed],
# Which companions each failing attempt drew, so a failure
# caused by one particular combination can be reproduced.
"failed_companions": [a.get("companions") for a in failed if a.get("companions")],
}
tests = {}
# 1. the world on its own, every option at its default.
tests["single"] = run_repeated([game], args.repeats)
tests["single"]["label"] = "Single game"
# 2. the same world twice. A world that keeps state on the class
# rather than the instance works alone and corrupts itself here -
# a bug class no other mode can see.
tests["duplicate"] = (skipped("skipped - an earlier test timed out") if state["timed_out"]
else run_mode([game, game], single_player=False))
tests["duplicate"]["label"] = "Two players, same game"
# 3. the world alongside other games. Each attempt draws a fresh
# random selection of core worlds rather than reusing one fixed
# pair: ten runs against the same two companions only ever tell you
# about those two, while a different combination each time samples
# what a real multiworld looks like.
configured_pool = [name.strip() for name in args.companion_pool.split(",") if name.strip()]
if configured_pool:
# Verified by the host once per run: every one of these generated
# a seed on its own in this exact environment, so a multi-game
# failure is attributable to the world under test rather than to
# a companion that was never going to work.
pool = [name for name in configured_pool if name != game]
else:
pool = core_worlds(exclude=game)
low = max(args.companion_min, 1)
high = max(args.companion_max, low)
def draw_companions():
count = min(random.randint(low, high), len(pool))
return random.sample(pool, count)
if state["timed_out"]:
tests["multi"] = skipped("skipped - an earlier test timed out")
elif pool:
tests["multi"] = run_repeated(
lambda: [game] + draw_companions(), args.repeats, single_player=False)
tests["multi"]["companion_pool"] = len(pool)
tests["multi"]["companion_range"] = [low, high]
tests["multi"]["detail"] += f" (against {low}-{high} core games drawn from {len(pool)})"
else:
tests["multi"] = skipped("no core worlds available to pair with")
tests["multi"]["label"] = "Multi game"
# 4 and 5. The same two shapes as above - the world alone, and the
# world alongside the companions - but with its options randomized
# instead of left at their defaults. The default option set is a
# single point in a world's option space and most option-dependent
# crashes are nowhere near it.
#
# One roll each, and every roll that does not produce a seed is a
# failure, including one the world rejects with OptionError. That
# last point is the whole reason these modes exist: the yaml a
# player writes is accepted as valid - nothing in the template or
# the upload path objects to "random" on two interdependent
# options - so the conflict only surfaces when generation is
# actually attempted, possibly after everyone has submitted, and it
# is intermittent. Re-rolling until the world accepts something
# would hide exactly the failure a real player hits.
#
# Each randomized run is gated on its own default-options
# counterpart: SR only when the world generates alone, MR only when
# it generates alongside the companions. A randomized run whose
# plain equivalent already fails would fail for the same underlying
# reason and cost a generation to learn nothing.
rolls = None
if state["timed_out"]:
aborted = "skipped - an earlier test timed out"
tests["single_random"] = skipped(aborted)
tests["multi_random"] = skipped(aborted)
elif tests["single"]["outcome"] == "passed" or tests["multi"]["outcome"] == "passed":
try:
# One roll per attempt, drawn up front so SR and MR run the
# SAME combinations as each other. Pairing them is what makes
# "SR passed but MR failed" attributable to the extra players
# rather than to two different option sets.
rolls = [random_options(game) for _ in range(max(args.random_repeats, 1))]
except BaseException as error:
rolls = None
unreadable = f"could not read this world's options: {type(error).__name__}: {error}"
tests["single_random"] = skipped(unreadable)
tests["multi_random"] = skipped(unreadable)
if rolls:
def roller():
# Walks the pre-drawn rolls in order, so attempt N of SR and
# attempt N of MR use the same option values.
supply = iter(rolls)
def next_roll():
return {game: next(supply)}
return next_roll
# "flaky" counts as generating: the world produced a seed at
# least once, so a randomized run can still tell us something.
if tests["single"]["outcome"] in ("passed", "flaky"):
result = run_repeated([game], len(rolls), single_player=True,
options=roller(), randomized=True)
result["randomized_options"] = len(rolls[0])
tests["single_random"] = result
else:
tests["single_random"] = skipped(
"skipped - the single-game test did not pass")
if tests["multi"]["outcome"] in ("passed", "flaky"):
result = run_repeated(lambda: [game] + draw_companions(), len(rolls),
single_player=False, options=roller(), randomized=True)
result["randomized_options"] = len(rolls[0])
tests["multi_random"] = result
else:
tests["multi_random"] = skipped(
"skipped - the multi-game test did not pass")
elif "single_random" not in tests:
neither = "skipped - neither the single-game nor the multi-game test passed"
tests["single_random"] = skipped(neither)
tests["multi_random"] = skipped(neither)
tests["single_random"]["label"] = "Random options, alone"
tests["multi_random"]["label"] = "Random options, multi game"
# The top-level fields stay exactly what they were - the single-player
# verdict - because that is what promotion and every existing consumer
# reads. The per-mode results live alongside it.
report = {
"game": game,
"outcome": tests["single"]["outcome"],
"detail": tests["single"]["detail"],
"elapsed_seconds": tests["single"]["elapsed_seconds"],
"multi_outcome": tests["multi"]["outcome"],
"multi_detail": tests["multi"]["detail"],
"multi_elapsed_seconds": tests["multi"]["elapsed_seconds"],
"multi_companions": tests["multi"].get("companions", []),
"tests": tests,
}
print(json.dumps(report))
# Exit code still reflects the single-player result alone: it is what
# gates promotion, and the other modes are reported rather than
# blocking.
return {"passed": 0, "needs_input": 2}.get(tests["single"]["outcome"], 1)
if __name__ == "__main__":
sys.exit(main())

65
src/apworld_tester/container/drivers/verify_companions.py Normal file
View File

@@ -0,0 +1,65 @@
"""Runs inside the archipelago Docker image. Reports which core worlds
can actually generate a seed in THIS environment, so the multi-game tests
only ever draw companions that work.
Core worlds are shipped with Archipelago, but that does not make them all
usable here: some need a base ROM this container has no copy of, and at
least one (Final Fantasy) needs a settings yaml generated by an external
website and refuses a generic one. Drawing such a world as a companion
fails the whole seed, and the failure gets recorded against the apworld
under test rather than the companion that caused it - Blender was marked
flaky 7/10 purely because three of its ten draws included worlds like
these.
Rather than maintain an exclusion list by hand, this generates one solo
seed per core world and keeps the ones that succeed. A world drops out
by itself when its ROM is missing, and comes back by itself once the ROM
is added.
Shares run_test.py's generation machinery rather than repeating it, so
"can generate" means exactly the same thing in both places.
"""
import argparse
import json
import sys
import warnings
sys.path.insert(0, "/app")
import run_test # noqa: E402 - mounted alongside this script
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output-dir", required=True)
parser.add_argument("--timeout", type=int, default=120,
help="Per-world timeout. Deliberately shorter than the apworld "
"timeout: a companion that is slow to generate would multiply "
"across every multi-game attempt that draws it.")
parser.add_argument("--spoiler", type=int, default=2)
args = parser.parse_args()
warnings.simplefilter("ignore")
run_test.ensure_tutorials_default()
from pathlib import Path
Path(args.output_dir).mkdir(parents=True, exist_ok=True)
verified = []
rejected = {}
for game in run_test.core_worlds(exclude=None):
results, timed_out, elapsed = run_test.run_with_timeout(
[game], args.output_dir, args.timeout, spoiler=args.spoiler)
outcome, detail = run_test.classify(results, timed_out, elapsed)
if outcome == "passed":
verified.append(game)
else:
rejected[game] = f"{outcome}: {detail}"[:300]
print(json.dumps({"verified": verified, "rejected": rejected}))
return 0
if __name__ == "__main__":
sys.exit(main())

0
src/apworld_tester/core/__init__.py Normal file
View File

0
src/apworld_tester/core/config/__init__.py Normal file
View File

131
src/apworld_tester/core/config/config.py Normal file
View File

@@ -0,0 +1,131 @@
"""The project's configuration."""
import os
import yaml
class ConfigNotFound(FileNotFoundError):
"""No config.yaml anywhere on the search path.
Carries the places that were tried, because the fix is always to
put a file in one of them and a bare "no such file" names only the
last candidate.
"""
def __init__(self, candidates):
self.candidates = list(candidates)
listed = "\n".join(f" {path}" for path in self.candidates)
super().__init__(
"no config.yaml found. Looked in:\n"
f"{listed}\n"
"Run 'apworld-tester init' to write a starter config, "
"or pass --config / set APWORLD_TESTER_CONFIG."
)
class ConfigNotFilledIn(ValueError):
"""A config that was written by `init` and never edited.
Its own error rather than the FileNotFoundError the placeholder
path eventually causes: that one names a directory nobody chose and
reads like a bug in the tool.
"""
def __init__(self, path, setting, value):
self.path = path
self.setting = setting
super().__init__(
f"{setting} is still the example value {value!r}.\n"
f"Edit {path} and set it to a directory on this machine."
)
class Config:
"""config.yaml, read once and passed to whatever needs it."""
#: Overrides every searched location when set.
ENV_VAR = "APWORLD_TESTER_CONFIG"
#: How the shipped template writes a path the user must replace.
PLACEHOLDER = "/path/to/your"
#: Settings that must be real before anything can run.
REQUIRED_PATHS = ("output_directory",)
#: The directory name used under $XDG_CONFIG_HOME and ~/.config.
APP_DIR = "apworld-tester"
def __init__(self, values, path=None):
self.values = values
self.path = path
@classmethod
def load(cls, path=None):
"""Read config.yaml from the first place it exists."""
path = path or cls.default_path()
with open(path, "r", encoding="utf-8") as handle:
config = cls(yaml.safe_load(handle) or {}, path)
config.check_filled_in()
return config
def check_filled_in(self):
"""Refuse a config still carrying the template's paths."""
for setting in self.REQUIRED_PATHS:
value = self.value("general", setting)
if value and str(value).startswith(self.PLACEHOLDER):
raise ConfigNotFilledIn(self.path, setting, value)
@staticmethod
def user_path():
"""Where a user's own config.yaml belongs.
$XDG_CONFIG_HOME when the platform sets it, ~/.config
otherwise - the location `init` writes to and the one the
documentation names.
"""
base = os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser(
"~/.config")
return os.path.join(base, Config.APP_DIR, "config.yaml")
@staticmethod
def checkout_path():
"""config.yaml beside a source checkout, above the package.
Kept in the search order so running from a clone kept working
after installed copies moved to the user config directory. An
installed wheel resolves this to somewhere inside site-packages'
parent, which simply will not exist.
"""
here = os.path.dirname(os.path.abspath(__file__))
root = os.path.dirname(os.path.dirname(os.path.dirname(here)))
return os.path.join(os.path.dirname(root), "config.yaml")
@classmethod
def search_paths(cls):
"""Every location tried, in order."""
override = os.environ.get(cls.ENV_VAR)
paths = [override] if override else []
return paths + [cls.user_path(), cls.checkout_path()]
@classmethod
def default_path(cls):
"""The first config.yaml that exists, or raise saying where."""
candidates = cls.search_paths()
for path in candidates:
if os.path.isfile(path):
return path
raise ConfigNotFound(candidates)
def get(self, key, default=None):
"""One top-level value."""
value = self.values.get(key)
return default if value is None else value
def section(self, name):
return self.values.get(name) or {}
def value(self, section, key, default=None):
"""One value from a section."""
value = self.section(section).get(key)
return default if value is None else value

125
src/apworld_tester/core/config/paths.py Normal file
View File

@@ -0,0 +1,125 @@
"""Every path the pipeline uses."""
import os
from importlib import resources
class Paths:
"""Where everything lives, derived from one Config.
The checkout holds code and configuration; the data root holds
everything a run generates, and defaults outside the checkout
because none of that is source. Every path is absolute.
"""
def __init__(self, config):
self.config = config
@property
def project(self):
return os.path.dirname(self.config.path)
@property
def data(self):
"""Where everything a run generates lives."""
configured = self.config.value(
"general",
"output_directory",
"~/.local/share/archipelago-world-tester",
)
return os.path.abspath(os.path.expanduser(configured))
@property
def state(self):
return os.path.join(self.data, "state.json")
@property
def scratch_state(self):
return os.path.join(self.data, "scratch_state.json")
@property
def lock(self):
return os.path.join(self.data, "pipeline.lock")
@property
def downloads(self):
return os.path.join(self.data, "downloads")
@property
def output(self):
return os.path.join(self.data, "test_output")
@property
def history(self):
"""Every apworld version this pipeline has seen, per game.
Beside the downloads rather than inside them: that directory is
a cache and gets cleared to force a clean re-download, and this
is the one thing that cannot be rebuilt afterwards.
"""
return os.path.join(self.data, "history")
@property
def checkout(self):
return os.path.join(self.data, "archipelago_src")
@property
def core_worlds(self):
return os.path.join(self.checkout, "worlds")
@property
def common_client(self):
return os.path.join(self.checkout, "CommonClient.py")
@property
def failed_report(self):
return os.path.join(self.data, "failed_apworlds.txt")
@property
def sheet_html(self):
return os.path.join(self.data, "playable_worlds.html")
@property
def roms(self):
"""Base ROMs for the worlds that need one."""
configured = self.config.value("general", "roms_directory")
if not configured:
return os.path.join(self.data, "roms")
return os.path.abspath(os.path.expanduser(configured))
@property
def env(self):
return os.path.join(self.project, ".env")
@property
def container(self):
"""The Dockerfile and drivers, resolved inside the package.
Shipped as package data rather than found beside config.yaml:
an installed copy has no checkout to look next to, and the
config now lives in the user's config directory, which is not
where a Dockerfile belongs. Returned as a real filesystem path
because every use is a docker build context or a bind mount.
"""
return os.fspath(resources.files("apworld_tester") / "container")
@property
def drivers(self):
return os.path.join(self.container, "drivers")
@property
def test_dockerfile(self):
return os.path.join(self.container, "Dockerfile.test")
@property
def driver(self):
return os.path.join(self.drivers, "run_test.py")
@property
def identify_driver(self):
return os.path.join(self.drivers, "identify_apworld.py")
@property
def verify_driver(self):
return os.path.join(self.drivers, "verify_companions.py")

86
src/apworld_tester/core/config/run_settings.py Normal file
View File

@@ -0,0 +1,86 @@
"""How much testing one run asks for.
Companions are drawn at random from the worlds Archipelago ships. Core
worlds are the safe choice: part of the build itself, so always present
and always matching the running version, unlike an apworld that might
be missing or broken for reasons of its own. A fresh draw per attempt
matters - ten runs against one fixed pair only ever test that pair.
"""
class RunSettings:
"""The testing section, as the numbers a run is built from."""
#: Archipelago's spoiler levels, by what each one does. The numbers
#: are Generate.py's `--spoiler`, not a choice this project makes.
SPOILER_LEVELS = {
"none": 0,
"log": 1,
"playthrough": 2,
"paths": 3,
}
def __init__(self, config):
self.config = config
def number(self, key, default):
return int(self.config.value("testing", key, default))
@property
def companion_range(self):
"""How many core worlds join the world under test.
Redrawn every attempt, so ten repeats sample ten combinations.
Clamped so a mistyped config cannot ask for an empty or
backwards range.
"""
low = max(self.number("companion_min", 2), 1)
return low, max(self.number("companion_max", 5), low)
@property
def repeats(self):
"""How often the single and multi tests are repeated.
Stability is a pass rate across runs, not one verdict: each
repeat gets a different seed, so the fill differs every time.
"""
return self.number("repeats", 10)
@property
def random_repeats(self):
"""How often the two randomized modes are repeated.
Lower on purpose: single and multi re-roll only the seed, so
ten runs measure one thing precisely, while each randomized run
rolls a different point in the option space and three samples
say much more than one did.
"""
return self.number("random_repeats", 3)
@property
def spoiler(self):
"""How much of the spoiler log each generation computes.
"playthrough" is the level worth paying for: it exercises the
accessibility and reachability logic that plain generation
skips, so a world whose logic is wrong fails here rather than
in a player's seed. "paths" adds entrance paths on top and
"log" writes the file without checking anything.
"""
name = str(self.config.value("testing", "spoiler", "playthrough"))
try:
return self.SPOILER_LEVELS[name.strip().casefold()]
except KeyError:
known = ", ".join(self.SPOILER_LEVELS)
raise ValueError(
f"testing.spoiler is {name!r}; expected one of {known}"
) from None
@property
def jobs(self):
"""How many apworlds are tested at once.
Each test is one isolated `docker run` and subprocess.run
releases the GIL while it waits, so threads are all this needs.
"""
return int(self.config.value("general", "threads", 4))

41
src/apworld_tester/core/config/secrets.py Normal file
View File

@@ -0,0 +1,41 @@
"""Credentials, kept out of config.yaml."""
import os
class Secrets:
"""The .env file at the checkout's root.
Kept out of config.yaml: that file gets opened, edited and shared,
and a token in it leaks the first time it is.
"""
#: The environment variable a GitHub token is read from.
GITHUB_TOKEN = "GITHUB_TOKEN"
def __init__(self, paths):
self.paths = paths
def values(self):
"""NAME -> value for every line in the file."""
if not os.path.exists(self.paths.env):
return {}
values = {}
with open(self.paths.env, "r", encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
name, value = line.split("=", 1)
values[name.strip()] = value.strip().strip("\"'")
return values
def get(self, name, default=None):
"""One secret, or `default` when it is unset or empty."""
value = self.values().get(name)
return default if value in (None, "") else value
@property
def github_token(self):
"""The GitHub token, or None."""
return self.get(self.GITHUB_TOKEN)

0
src/apworld_tester/core/display/__init__.py Normal file
View File

108
src/apworld_tester/core/display/progress.py Normal file
View File

@@ -0,0 +1,108 @@
"""Progress reporting for the long stages."""
import shutil
import sys
import time
class Progress:
"""One stage's progress, tallied by outcome as it goes.
Both the download stage and the generation tests walk ~650 games, so
per-line output scrolls past uselessly when a run is watched. Under
cron the opposite is true - stdout is a log file, where a bar
redrawn with carriage returns is noise - so each caller supplies the
line it would have printed, and that is what a non-terminal gets.
"""
def __init__(self, total, config=None, labels=None, stream=None):
self.total = total
self.config = config
self.labels = dict(labels or self.configured_labels())
self.stream = stream or sys.stdout
self.counts = {}
self.started = None
def configured_labels(self):
"""Short labels for each outcome."""
if self.config is None:
return {}
return self.config.value("display", "outcome_labels", {})
@property
def width(self):
"""How many characters wide the bar is drawn."""
if self.config is None:
return 28
return int(self.config.value("display", "progress_bar_width", 28))
@staticmethod
def duration(seconds):
"""Seconds as m:ss, or h:mm:ss past an hour."""
seconds = int(seconds)
hours, rest = divmod(seconds, 3600)
minutes, seconds = divmod(rest, 60)
if hours:
return f"{hours}:{minutes:02d}:{seconds:02d}"
return f"{minutes:02d}:{seconds:02d}"
@property
def tally(self):
"""The running counts, labelled keys first."""
ordered = [key for key in self.labels if self.counts.get(key)]
ordered += [key for key in self.counts if key not in self.labels]
return " ".join(
f"{self.labels.get(key, key)} {self.counts[key]}"
for key in ordered
)
def bar(self, fraction):
"""The bar itself, filled to `fraction`."""
filled = int(self.width * fraction)
return "#" * filled + "-" * (self.width - filled)
def eta(self, index, elapsed):
"""Time remaining at the current rate, blank once finished."""
if not index or index >= self.total:
return ""
return f" eta {self.duration(elapsed / index * (self.total - index))}"
def line(self, index, text):
"""The whole status line for this update.
Position and time only. The running tally and the item's name
belong to the summary printed when the stage ends - on the bar
they change every redraw and are unreadable while doing it.
"""
fraction = index / self.total if self.total else 1
elapsed = time.monotonic() - self.started
return (f"[{self.bar(fraction)}] {index}/{self.total} "
f"{fraction * 100:3.0f}% "
f"{self.duration(elapsed)}{self.eta(index, elapsed)}")
def draw(self, index, text):
"""Redraw the bar in place, ending the line when finished.
Padded to overwrite a previously longer line, then trimmed to
the terminal: a wrapped line cannot be overwritten by a carriage
return, and the bar starts spilling down the screen.
"""
columns = shutil.get_terminal_size((100, 24)).columns
padded = self.line(index, text)[:columns - 1].ljust(columns - 1)
self.stream.write("\r" + padded)
self.stream.flush()
if index >= self.total:
self.stream.write("\n")
self.stream.flush()
def update(self, index, key=None, text="", fallback_line=None):
"""Record one finished item and show where the stage is."""
if self.started is None:
self.started = time.monotonic()
if key is not None:
self.counts[key] = self.counts.get(key, 0) + 1
if not self.stream.isatty():
if fallback_line is not None:
print(fallback_line, file=self.stream)
return
self.draw(index, text)

0
src/apworld_tester/core/model/__init__.py Normal file
View File

120
src/apworld_tester/core/model/apworld.py Normal file
View File

@@ -0,0 +1,120 @@
"""The filename an apworld needs before Archipelago will import it."""
import json
import os
import re
import zipfile
class ApworldFile:
"""One downloaded apworld, and what its name has to be.
Archipelago derives the module name from the FILENAME, so a version
in the name reads as package separators and a backslash is just a
character. The right name is the zip's own package folder.
"""
#: The extension an apworld is recognised by.
SUFFIX = ".apworld"
def __init__(self, path):
self.path = path
@property
def entries(self):
"""Every entry name in the zip, empty if it will not open."""
try:
with zipfile.ZipFile(self.path) as archive:
return archive.namelist()
except (OSError, zipfile.BadZipFile):
return []
@property
def uses_backslash_separators(self):
"""Whether the zip stores its paths with backslashes.
Invisible on Windows and fatal on Linux: zipimport swaps "/" for
the platform separator when reading, so on Windows both forms
match and on Linux the backslash entry never matches the
"pkg/__init__.py" it looks for. The author sees a working
apworld and every Linux user sees ModuleNotFoundError.
"""
names = self.entries
if not names:
return False
return (not any("/" in name for name in names)
and any("\\" in name for name in names))
@property
def manifest(self):
"""The apworld's own archipelago.json, or None.
Its self-declared "game" and "world_version", sitting in the
file's content regardless of which release was resolved for it -
or whether one ever was. Older apworlds predate the convention
and ship none, so this is best-effort display information rather
than something worth failing over.
"""
try:
with zipfile.ZipFile(self.path) as archive:
name = next((entry for entry in archive.namelist()
if entry.endswith("/archipelago.json")), None)
if name is None:
return None
with archive.open(name) as handle:
return json.load(handle)
except (OSError, zipfile.BadZipFile, json.JSONDecodeError, KeyError):
return None
@property
def package_folder(self):
"""The single top-level package inside the zip, or None."""
names = [name.replace("\\", "/") for name in self.entries]
packages = {
name.split("/", 1)[0]
for name in names
if name.count("/") == 1 and name.endswith("/__init__.py")
}
return packages.pop() if len(packages) == 1 else None
@property
def importable_name(self):
"""What this file must be called for Archipelago to import it."""
base = os.path.basename(self.path.replace("\\", "/"))
if base.lower().endswith(ApworldFile.SUFFIX):
stem = base[:-len(ApworldFile.SUFFIX)]
else:
stem = base
stem = self.package_folder or self.sanitize(stem)
return stem + ApworldFile.SUFFIX
def sanitize(self, stem):
"""A filename stem reduced to something importable."""
base = os.path.basename(stem.replace("\\", "/"))
cleaned = re.sub(r"[^0-9A-Za-z_]", "_", base).strip("_")
if cleaned and cleaned[0].isdigit():
cleaned = "_" + cleaned
return cleaned or "apworld"
def prepare(self, working_directory):
"""The path to hand Archipelago, and whether it was repaired."""
if not self.uses_backslash_separators:
return self.path, False
os.makedirs(working_directory, exist_ok=True)
destination = os.path.join(working_directory, self.importable_name)
return self.write_normalized(destination), True
def write_normalized(self, destination):
"""Copy the apworld with its separators repaired."""
with zipfile.ZipFile(self.path) as source:
with zipfile.ZipFile(destination, "w",
zipfile.ZIP_DEFLATED) as target:
for item in source.infolist():
repaired = zipfile.ZipInfo(
filename=item.filename.replace("\\", "/"),
date_time=item.date_time,
)
repaired.compress_type = item.compress_type
repaired.external_attr = item.external_attr
target.writestr(repaired, source.read(item.filename))
return destination

30
src/apworld_tester/core/model/blacklist.py Normal file
View File

@@ -0,0 +1,30 @@
"""Worlds this pipeline declines to test."""
from apworld_tester.core.model.name import Name
class Blacklist:
"""Worlds not tested.
A world that times out is the most expensive thing in a run and the
least informative. Entries are permanent until removed by hand -
"do not retry" is the whole reason one is listed.
"""
#: Recorded verdict for a world that was never tested.
OUTCOME = "unknown"
#: What the status page shows in place of a test result.
DETAIL = "not tested: blacklisted"
def __init__(self, config):
self.config = config
@property
def entries(self):
"""Every listed world, by folded name."""
listed = self.config.get("blacklist", []) or []
return {Name(name).slug for name in listed if Name(name).slug}
def listed(self, game_name):
return Name(game_name).slug in self.entries

30
src/apworld_tester/core/model/fingerprint.py Normal file
View File

@@ -0,0 +1,30 @@
"""Content hashes, used to decide what has changed."""
import hashlib
import os
class Fingerprint:
"""The content of a file or a directory, as one hash."""
@staticmethod
def of_file(path):
"""One file's sha256, read in chunks."""
digest = hashlib.sha256()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(65536), b""):
digest.update(chunk)
return digest.hexdigest()
@classmethod
def of_directory(cls, directory):
"""One hash covering every file directly inside a directory."""
digest = hashlib.sha256()
if not os.path.isdir(directory):
return digest.hexdigest()
for name in sorted(os.listdir(directory)):
path = os.path.join(directory, name)
if os.path.isfile(path):
digest.update(name.encode("utf-8"))
digest.update(cls.of_file(path).encode("utf-8"))
return digest.hexdigest()

47
src/apworld_tester/core/model/game.py Normal file
View File

@@ -0,0 +1,47 @@
"""One game from the worlds sheet."""
import re
class Game:
"""A sheet row: a name, its links, and what the sheet says about it."""
def __init__(self, name, links=None, release=None, stability=None,
pr_status=None):
self.name = name
self.links = list(links or [])
self.release = release
self.stability = stability
self.pr_status = pr_status
for link in self.links:
self.remember_release(link)
@staticmethod
def is_release_link(link):
"""Whether a link points at a GitHub releases page."""
return bool(re.match(
r"^https?://(?:www\.)?github\.com/[^/]+/[^/]+/releases"
r"(?:[/?#]|$)",
link or "",
re.IGNORECASE,
))
@property
def is_core(self):
return (self.pr_status or "").strip().lower() == "core"
def remember_release(self, link):
"""Keep this link as the release URL if none was found yet."""
if self.release is None and self.is_release_link(link):
self.release = link
return self.release
def add_link(self, link):
"""Add a link, ignoring one already recorded."""
if link not in self.links:
self.links.append(link)
self.remember_release(link)
return self.links
def __str__(self):
return f"{self.name} ({len(self.links)} links)"

35
src/apworld_tester/core/model/instant.py Normal file
View File

@@ -0,0 +1,35 @@
"""One timestamp, as the moment it names."""
import datetime
class Instant:
"""An ISO 8601 timestamp parsed into a comparable moment.
Hosts do not agree on how to write the offset: GitHub sends a
trailing "Z", this pipeline's own clock writes "+00:00", and GitLab
and Gitea send a local one ("...19:55:36-04:00" is really 23:55Z).
Compared as text those sort by the numbers on the clock face rather
than by when they happened, so anything comparing two timestamps
from different sources has to parse them first.
"""
@staticmethod
def of(text):
"""One timestamp, or None when it is not a usable one.
A naive timestamp is read as UTC: everything written here is,
and the alternative is a comparison that raises rather than one
that is merely approximate. "Z" is spelled out because
fromisoformat only accepts it from 3.11.
"""
if not text:
return None
try:
parsed = datetime.datetime.fromisoformat(
str(text).replace("Z", "+00:00"))
except (TypeError, ValueError):
return None
if parsed.tzinfo is None:
return parsed.replace(tzinfo=datetime.timezone.utc)
return parsed

22
src/apworld_tester/core/model/modes.py Normal file
View File

@@ -0,0 +1,22 @@
"""The generation modes a complete result carries."""
class Modes:
"""What the driver runs against every world.
Not configuration: each name is a branch in the container driver
and in Verdict, so a mode cannot be added or removed by editing a
list. Changing this would only desynchronise the cache check and
the results table from what actually ran.
"""
SINGLE = "single"
DUPLICATE = "duplicate"
MULTI = "multi"
SINGLE_RANDOM = "single_random"
MULTI_RANDOM = "multi_random"
ALL = (SINGLE, DUPLICATE, MULTI, SINGLE_RANDOM, MULTI_RANDOM)
#: The modes that cannot demote a world on their own.
SECONDARY = (DUPLICATE, SINGLE_RANDOM, MULTI_RANDOM)

42
src/apworld_tester/core/model/name.py Normal file
View File

@@ -0,0 +1,42 @@
"""Folding a game name down to a comparison key."""
import re
import unicodedata
class Name:
"""One game name, and the two ways it gets folded.
One implementation, because five copies once disagreed and a world
silently lost its blacklist entry.
"""
def __init__(self, text):
self.text = text or ""
@property
def slug(self):
"""The key for matching this name against another."""
folded = unicodedata.normalize("NFKD", self.text.lower())
return re.compile(r"[^a-z0-9]").sub("", folded)
@property
def report_slug(self):
return re.compile(r"[^a-z0-9]+").sub("-", self.text.lower()).strip("-")
@property
def directory(self):
return re.sub(r'[<>:"/\\|?*]', "_", self.text).strip().rstrip(".")
def matches(self, other):
"""Whether an internally registered name means this game.
One-directional: the internal name is usually the shorter,
canonical one and the sheet's is often longer, so checking
containment the other way would let a short internal name match
an unrelated row that happens to start the same way.
"""
if not self.text or not other:
return False
internal = Name(other).slug
return internal == self.slug or internal in self.slug

71
src/apworld_tester/core/model/verdict.py Normal file
View File

@@ -0,0 +1,71 @@
"""Where a world's test record lands on the stability ladder."""
from apworld_tester.core.model.modes import Modes
class Verdict:
"""One world's tested stability.
The ladder: broken, unknown, flaky, solo_only, unstable, stable.
Single and multi decide the first four, because those are what a
player does; the other modes can only stop a world being stable.
"""
#: Outcomes that count as a failure in a secondary mode.
FAILURES = ("failed", "invalid_options", "flaky")
def __init__(self, record, config):
self.record = record or {}
self.config = config
@property
def modes(self):
return self.record.get("tests") or {}
@property
def never_imported(self):
return self.record.get("outcome") == "unverified"
def outcome_of(self, mode):
return (self.modes.get(mode) or {}).get("outcome")
@property
def primary(self):
"""What the single and multi runs decide, or None if they pass."""
single = self.outcome_of(Modes.SINGLE)
multi = self.outcome_of(Modes.MULTI)
if single == "needs_input" or multi == "needs_input":
return "unknown"
if single == "failed":
return "broken"
if single == "flaky" or multi == "flaky":
return "flaky"
if multi == "failed":
return "solo_only"
if single != "passed":
return "unknown"
return None
@property
def secondary_failed(self):
"""Whether a mode that cannot demote alone still failed."""
return any(self.outcome_of(mode) in self.FAILURES
for mode in Modes.SECONDARY)
@property
def name(self):
"""Where this record lands on the ladder."""
if self.never_imported:
return "broken"
if not self.modes or not self.outcome_of(Modes.SINGLE):
return "unknown"
decided = self.primary
if decided is not None:
return decided
return "unstable" if self.secondary_failed else "stable"
@property
def label(self):
labels = self.config.value("stability", "labels", {})
return labels.get(self.name, "Unknown")

0
src/apworld_tester/core/state/__init__.py Normal file
View File

23
src/apworld_tester/core/state/keys.py Normal file
View File

@@ -0,0 +1,23 @@
"""The keys this pipeline writes into state.json."""
class StateKeys:
"""Names of the entries in the state file.
Not configuration: state.json is written and read only by this
pipeline, so renaming a key here would not migrate the file, it
would orphan every record already in it.
"""
APWORLD_TESTS = "apworld_tests"
UPGRADE_SNAPSHOT = "apworld_tests_at_last_upgrade"
ARCHIPELAGO_TAG = "archipelago_tag"
ARCHIPELAGO_TAG_PUBLISHED_AT = "archipelago_tag_published_at"
PREVIOUS_ARCHIPELAGO_TAG = "previous_archipelago_tag"
COMPANION_POOL = "verified_companions"
GAME_RELEASES = "game_releases"
DISCONTINUED = "discontinued"
LAST_CHECKED_AT = "last_checked_at"
FAILED_RELEASE = "failed_release"
FAILED_REASON = "failed_reason"
STABILITY_HISTORY = "stability_history"

37
src/apworld_tester/core/state/lock.py Normal file
View File

@@ -0,0 +1,37 @@
"""The exclusive lock a whole run holds."""
import fcntl
class LockHeldError(RuntimeError):
"""Another run already holds the lock."""
class PipelineLock:
"""Stops two runs, or a run and a promotion, from overlapping.
Non-blocking: a second run fails fast rather than queuing. Queuing
would be worse - a full pass takes hours, so a queued one would
start against an Archipelago version that had moved on.
"""
def __init__(self, path):
self.path = path
self.handle = None
def __enter__(self):
self.handle = open(self.path, "a+")
try:
fcntl.flock(self.handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError as error:
self.handle.close()
raise LockHeldError(
f"Another run already holds {self.path} - refusing to "
"start."
) from error
return self
def __exit__(self, *unused):
fcntl.flock(self.handle, fcntl.LOCK_UN)
self.handle.close()
return False

32
src/apworld_tester/core/state/scratch.py Normal file
View File

@@ -0,0 +1,32 @@
"""Choosing between the live record and a scratch copy of it."""
import os
import shutil
class ScratchState:
"""Where a run's results go, live or not.
A run is an experiment unless it says otherwise: promotion and the
status page read the live file, so a hand-run test that wrote there
would put a world nobody can download into the deployable record.
Seeding the scratch copy from the live one is what makes that cheap
rather than punishing - without it a scratch run has no recorded
Archipelago version to test against and no verified companion pool,
so it would stop for the first and re-check every core world for
the second.
"""
def __init__(self, paths, live=False):
self.paths = paths
self.live = live
@property
def path(self):
if self.live:
return self.paths.state
scratch = self.paths.scratch_state
if not os.path.exists(scratch) and os.path.exists(self.paths.state):
shutil.copy(self.paths.state, scratch)
return scratch

55
src/apworld_tester/core/state/store.py Normal file
View File

@@ -0,0 +1,55 @@
"""Reading and writing the pipeline's JSON state."""
import json
import os
import tempfile
class StateStore:
"""What every world last did, and the companion pool behind it."""
def __init__(self, path):
self.path = path
def load(self):
return self.read(self.path, {})
def save(self, data):
self.write(self.path, data)
@staticmethod
def read(path, default=None):
"""A JSON file's contents, or `default` when it is absent."""
if not os.path.exists(path):
return {} if default is None else default
with open(path, "r", encoding="utf-8") as handle:
return json.load(handle)
@staticmethod
def write(path, data):
"""Write JSON so a reader never sees a partial file.
A kill mid-write used to leave state.json truncated mid-object -
valid up to the cut point and invalid after it - silently
costing every recorded result on the next load. The batch saves
after every single test, so that window was hit often.
os.replace is atomic: the file is always either the old complete
version or the new one.
"""
directory = os.path.dirname(os.path.abspath(path)) or "."
handle_id, temporary = tempfile.mkstemp(
dir=directory, prefix=".state_", suffix=".tmp")
try:
with os.fdopen(handle_id, "w", encoding="utf-8") as handle:
json.dump(data, handle, indent=2, sort_keys=True)
handle.write("\n")
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
except BaseException:
try:
os.remove(temporary)
except OSError:
pass
raise

0
src/apworld_tester/pipeline/__init__.py Normal file
View File

0
src/apworld_tester/pipeline/batch/__init__.py Normal file
View File

67
src/apworld_tester/pipeline/batch/apworld_finder.py Normal file
View File

@@ -0,0 +1,67 @@
"""Finding the apworlds already downloaded."""
import os
class ApworldFinder:
"""The download directory, as the worlds a run can test."""
def __init__(self, root):
self.root = root
@staticmethod
def matches(game_name, only, exact):
if not only:
return True
if exact:
return any(t.casefold() == game_name.casefold() for t in only)
return any(t.lower() in game_name.lower() for t in only)
def in_folder(self, game_dir):
"""Every apworld directly inside one game's folder.
A directory where a file should be is skipped rather than
crashing the batch: that is almost always docker auto-creating
an empty directory at a bind-mount source that did not exist
yet, and a fresh download replaces it next time.
"""
found = []
for entry in sorted(os.listdir(game_dir)):
if not entry.lower().endswith(".apworld"):
continue
path = os.path.join(game_dir, entry)
if not os.path.isfile(path):
print(f"Skipping {path} - not a regular file (directory?)")
continue
found.append(path)
return found
def find(self, only=None, exact=False):
"""Every downloaded apworld, optionally narrowed to `only`.
`only` matches on substring, which is what --only wants. That
is ambiguous when one game's name contains another's - "Hollow
Knight" also selects "Hollow Knight_ Silksong" - so a caller
that already resolved a term to a real folder passes exact.
"""
found = []
if not os.path.isdir(self.root):
return found
for game_name in sorted(os.listdir(self.root)):
if not self.matches(game_name, only, exact):
continue
game_dir = os.path.join(self.root, game_name)
if not os.path.isdir(game_dir):
continue
found += [(game_name, path) for path in self.in_folder(game_dir)]
return found
@property
def available(self):
return sorted({name for name, _ in self.find()})
def resolve(self, term):
"""The folders a name selects, an exact match winning."""
matches = sorted({name for name, _ in self.find(only=[term])})
exact = [n for n in matches if n.casefold() == term.casefold()]
return exact or matches

119
src/apworld_tester/pipeline/batch/batch.py Normal file
View File

@@ -0,0 +1,119 @@
"""Testing every downloaded apworld and recording what each did."""
from apworld_tester.core.model.fingerprint import Fingerprint
from apworld_tester.core.state.keys import StateKeys
from apworld_tester.core.state.store import StateStore
from apworld_tester.pipeline.batch.companion_pool import CompanionPool
from apworld_tester.pipeline.batch.failed_report import FailedReport
from apworld_tester.pipeline.batch.options import Options
from apworld_tester.pipeline.batch.planner import Planner
from apworld_tester.pipeline.batch.runner import Runner
from apworld_tester.pipeline.batch.work_list import WorkList
class Batch:
"""One run: plan it, serve what is cached, test the rest.
An apworld whose file content and Archipelago version both match
its last recorded test is served from cache. The caller forces a
full re-test when the version changes, because last week's passes
no longer apply to anything.
"""
def __init__(self, config, paths, settings, image, tag, force=False,
on_progress=None, **asked):
self.config = config
self.paths = paths
self.settings = settings
self.image = image
self.tag = tag
self.force = force or bool(asked.get("only"))
self.on_progress = on_progress
self.options = Options(paths, settings, **asked).resolve()
self.store = StateStore(self.options["state_path"])
@property
def writes_live_report(self):
"""Whether this run owns the "what is broken" report.
The report is about the deployable, current-version state: a
previous-version comparison writing here would conflate "not
promotable now" with "does not work on a release nobody runs",
and a run pointed at a scratch state file is not the live
record either.
"""
return (self.options["state_key"]
== StateKeys.APWORLD_TESTS
and self.options["state_path"] == self.paths.state)
def pool(self):
return CompanionPool(
config=self.config,
paths=self.paths,
store=self.store,
image=self.image,
tag=self.tag,
roms_directory=self.options["roms_directory"],
).resolve()
def plan(self, tests, fingerprint, state):
"""Everything to consider, what is known, what to run.
Blacklisted rows and the confirmation date on a cached one are
the entries written outside the runner, which is what normally
persists state - without the save here they would exist only
in memory, and a run that tested nothing would write nothing.
"""
work = WorkList(
root=self.options["root_directory"],
only=self.options["only"],
exact=self.options["exact_only"],
core_games=self.options["core_games"],
).items()
planner = Planner(
config=self.config,
settings=self.settings,
tests=tests,
tag=self.tag,
roms_fingerprint=fingerprint,
force=self.force,
repeats=self.options["repeats"],
random_repeats=self.options["random_repeats"],
companion_range=self.options["companion_range"],
# A release published after a world's last result
# invalidates it, whatever the file on disk hashes to.
releases=state.get(StateKeys.GAME_RELEASES, {}),
)
results, to_test = planner.plan(work)
if planner.wrote_state:
self.store.save(state)
return work, results, to_test
def report_cached(self, results, total):
"""Free to produce, so reported without waiting on the pool."""
for index, report in enumerate(results, start=1):
if self.on_progress:
self.on_progress(index, total, report)
def run(self):
self.options["pool"] = self.pool()
state = self.store.load()
tests = state.setdefault(self.options["state_key"], {})
fingerprint = Fingerprint.of_directory(self.options["roms_directory"])
work, results, to_test = self.plan(tests, fingerprint, state)
self.report_cached(results, len(work))
if to_test:
results += Runner(
config=self.config,
paths=self.paths,
store=self.store,
options=self.options,
state=state,
tests=tests,
tag=self.tag,
image=self.image,
roms_fingerprint=fingerprint,
).run(to_test, len(work), len(results), self.on_progress)
if self.writes_live_report:
FailedReport(state, self.paths.failed_report).write()
return results

120
src/apworld_tester/pipeline/batch/cache_check.py Normal file
View File

@@ -0,0 +1,120 @@
"""Whether a previous result still answers this run's question."""
import re
from apworld_tester.core.model.instant import Instant
from apworld_tester.core.model.modes import Modes
class CacheCheck:
"""One world's recorded result, against what is being asked now."""
def __init__(self, settings, previous):
self.settings = settings
self.previous = previous
#: Failure text pointing at a base ROM, not the world's logic. A
#: wrong or missing ROM surfaces under many exception types, so the
#: type cannot recognise one and the message is what is left.
#: Word-bounded so "from" does not read as "rom".
ROM_FAILURE = re.compile(
r"\b(rom|md5|sha1|checksum)\b"
r"|\.(sfc|smc|z64|n64|nes|gba|gbc|gb|gen|md|iso|bin)\b",
re.IGNORECASE,
)
def mentions_rom(self, detail):
return bool(detail and self.ROM_FAILURE.search(str(detail)))
def needs_rom_retry(self, roms_fingerprint):
"""Whether a changed ROM folder invalidates this result.
A missing ROM raises FileNotFoundError and lands in
"needs_input", but a WRONG one raises whatever the world throws
after reading it and lands in "failed" - so retrying only
needs_input meant replacing a bad dump could never clear the
result. A "failed" result additionally has to look like a ROM
problem: most failures carrying a stale fingerprint were fill
failures and deprecated APIs that no ROM will fix.
"""
if self.previous is None:
return False
outcome = self.previous.get("outcome")
rom_related = (outcome == "needs_input"
or (outcome == "failed"
and self.mentions_rom(self.previous.get("detail"))))
return (rom_related and self.previous.get("roms_fingerprint")
!= roms_fingerprint)
def released_since(self, released_at):
"""Whether the row's apworld was published after it was tested.
The hash catches a file whose content changed, but not a host
that offered a new one and had it turn out identical - and
"identical" is only ever this pipeline's word for it, resting
on the copy it happened to hold. A publication later than the
answer means the answer was reached before whatever is being
offered now, and testing again is the only thing that settles
it. It settles it for good, too: the new result is dated after
the release, so this stops firing until the release moves
again.
"""
if self.previous is None:
return False
released = Instant.of(released_at)
tested = Instant.of(self.previous.get("tested_at"))
if released is None or tested is None:
return False
return tested < released
@staticmethod
def attempts(recorded_modes):
"""How many attempts a cached record was reached with.
The randomized modes used to run exactly once and carried no
count at all, so a missing value there means "one".
"""
if not recorded_modes:
return None, None
return (
(recorded_modes.get("single") or {}).get("requested_attempts"),
(recorded_modes.get("single_random") or {})
.get("requested_attempts"),
)
def answers(self, repeats, random_repeats, companion_range):
"""Whether the record answers what this run is asking.
It does not when the record predates per-mode results, when the
set of modes has changed, when it was reached with fewer
repeats - "it generated once" is not "it generated ten times
out of ten" - or when the companion range differs. A world that
never imported is exempt: "unverified" is a complete answer
about this exact file, and the file has to change for the
answer to.
"""
if self.previous is None:
return True
if self.previous.get("outcome") == "unverified":
return True
recorded = self.previous.get("tests")
if recorded is None or set(recorded) != set(Modes.ALL):
return False
single, randomized = self.attempts(recorded)
if single != repeats:
return False
if randomized is not None and randomized != random_repeats:
return False
recorded_range = (recorded.get("multi") or {}).get(
"companion_range") or []
return list(recorded_range) == list(companion_range)
def serves(self, current_hash, tag, roms_fingerprint, repeats,
random_repeats, companion_range, released_at=None):
"""Whether this world can be served from its previous result."""
return (self.previous is not None
and self.previous.get("hash") == current_hash
and self.previous.get("tag") == tag
and not self.needs_rom_retry(roms_fingerprint)
and not self.released_since(released_at)
and self.answers(repeats, random_repeats, companion_range))

65
src/apworld_tester/pipeline/batch/companion_pool.py Normal file
View File

@@ -0,0 +1,65 @@
"""The core worlds a run may use as companions."""
from datetime import datetime, timezone
from apworld_tester.core.model.fingerprint import Fingerprint
from apworld_tester.core.state.keys import StateKeys
from apworld_tester.pipeline.generation.companion_check import (
CompanionCheck,
)
class CompanionPool:
"""Verified once per run and cached against version and ROMs.
The answer only changes when Archipelago or the ROMs folder does,
and what it rejects would otherwise fail seeds belonging to the
world under test.
"""
def __init__(self, config, paths, store, image, tag, roms_directory):
self.config = config
self.paths = paths
self.store = store
self.image = image
self.tag = tag
self.fingerprint = Fingerprint.of_directory(roms_directory)
@property
def key(self):
return StateKeys.COMPANION_POOL
def cached(self, state):
cached = state.get(self.key) or {}
if (cached.get("tag") == self.tag
and cached.get("roms_fingerprint") == self.fingerprint):
return cached.get("games") or []
return None
def store_result(self, state, result):
state[self.key] = {
"tag": self.tag,
"roms_fingerprint": self.fingerprint,
"games": result["verified"],
"rejected": result["rejected"],
"checked_at": datetime.now(timezone.utc).isoformat(),
}
self.store.save(state)
print(f"{len(result['verified'])} core worlds usable as companions, "
f"{len(result['rejected'])} rejected")
for game, reason in sorted(result["rejected"].items()):
print(f" {game}: {reason[:90]}")
return result["verified"]
def resolve(self):
state = self.store.load()
cached = self.cached(state)
if cached is not None:
return cached
result = CompanionCheck(self.config, self.paths, self.image).run(
self.paths.output)
if not result.get("ran"):
print(f"companion verification failed ({result.get('detail')}) - "
"multi-game tests will draw from every core world")
return []
return self.store_result(state, result)

69
src/apworld_tester/pipeline/batch/discontinued.py Normal file
View File

@@ -0,0 +1,69 @@
"""Flagging test records whose game left the sheet."""
from apworld_tester.core.model.name import Name
from apworld_tester.core.state.keys import StateKeys
class Discontinued:
"""A world taken off the sheet stops being one this pipeline
tracks: it should leave the status page and stop being deployed.
The record is kept and labelled rather than deleted, so its history
survives and a row that comes back - or that vanished because of a
bad sheet parse - is one flag away from being live again.
"""
def __init__(self, store):
self.store = store
@staticmethod
def sheet_name(key, record):
"""The sheet row a record's key came from.
A download- or core-stage record is keyed on the bare sheet
name, since nothing was ever fetched for it. Everything else is
keyed "<folder>/<file>.apworld". The record's own "game" is
deliberately not used: that is the name Archipelago registered
the world under, which can differ from the sheet's wording.
"""
if (record or {}).get("stage") in ("download", "core"):
return key
return key.split("/", 1)[0]
def apply(self, tests, live):
"""Set or clear the flag, reporting only what moved.
Comparison is on the folded directory name of both sides, which
is what turned the sheet name into the download folder in the
first place, so punctuation and case cannot make a still-listed
game look discontinued. An unchanged run leaves the state file
byte-identical.
"""
flag = StateKeys.DISCONTINUED
changed = {}
for key, record in tests.items():
name = self.sheet_name(key, record)
gone = Name(name).directory not in live
if gone != bool(record.get(flag)):
changed[key] = gone
if gone:
record[flag] = True
else:
record.pop(flag, None)
return changed
def mark(self, sheet_names):
"""Flag every record whose game is no longer listed.
Does nothing when handed no sheet names: an empty list means
the sheet was not read, not that every world on earth was
retired, and acting on it would empty the page.
"""
if not sheet_names:
return {}
state = self.store.load()
tests = state.get(StateKeys.APWORLD_TESTS, {})
changed = self.apply(
tests, {Name(name).directory for name in sheet_names})
if changed:
self.store.save(state)
return changed

88
src/apworld_tester/pipeline/batch/download_failures.py Normal file
View File

@@ -0,0 +1,88 @@
"""Recording games whose apworld could never be fetched."""
from datetime import datetime, timezone
from apworld_tester.core.model.name import Name
from apworld_tester.core.state.keys import StateKeys
from apworld_tester.pipeline.batch.record import TestRecord
class DownloadFailures:
"""A game with no file has nothing to key a result on the way a
tested world does, so it is keyed on its own sheet name - and still
shows up as an unknown row rather than vanishing between the sheet
and the results. Real result keys always contain "/", so a bare
name cannot collide with one.
"""
def __init__(self, config, store):
self.config = config
self.store = store
@staticmethod
def tested_already(tests, name):
"""Whether a real result already answers for this game.
A download row exists so a game with no file still shows up
rather than vanishing between the sheet and the results. When
an apworld is on disk and was tested, nothing is vanishing -
the link broke, which the download report says, while the file
that world was last tested from is still there and still the
answer. Writing a second row then reports one game twice, and
the two rows collide at promotion because they claim the same
game name.
"""
folded = Name(name).directory
return any(
"/" in key and Name(key.split("/", 1)[0]).directory == folded
for key in tests
)
@staticmethod
def detail_for(reason):
return ("apworld could not be downloaded, so it was never tested: "
f"{reason}")
def clear_stale(self, tests, games, skipped):
"""Drop rows for games that downloaded this time.
Only ever touches entries this class wrote, never a real
generation-test result.
"""
failed = {name for name, _ in skipped}
for game in games:
previous = tests.get(game.name)
if (game.name not in failed and previous is not None
and previous.get("stage") == "download"):
del tests[game.name]
def note(self, tests, name, reason, tag, now):
"""Record one failure, keeping an unchanged row's date.
A row failing the same way as last time keeps the date of the
attempt that reached that answer and only has its last-checked
stamp moved: a test date that advanced every run would claim a
world was looked at when nothing about it was.
"""
detail = self.detail_for(reason)
previous = tests.get(name)
if (previous is not None and previous.get("stage") == "download"
and previous.get("detail") == detail):
previous["tag"] = tag
previous[StateKeys.LAST_CHECKED_AT] = now
return
tests[name] = TestRecord(name).download_failure(
detail, tag)
def record(self, games, skipped, tag):
state = self.store.load()
tests = state.setdefault(StateKeys.APWORLD_TESTS, {})
self.clear_stale(tests, games, skipped)
now = datetime.now(timezone.utc).isoformat()
for name, reason in skipped:
if self.tested_already(tests, name):
tests.pop(name, None)
continue
self.note(tests, name, reason, tag, now)
self.store.save(state)
return state

36
src/apworld_tester/pipeline/batch/failed_report.py Normal file
View File

@@ -0,0 +1,36 @@
"""The list of worlds that were tested and did not pass."""
from apworld_tester.core.state.keys import StateKeys
class FailedReport:
"""Written after every run, from the whole accumulated state.
Not just this run's results: a targeted run touches a couple of
entries and the report must still reflect every other known
failure. Kept out of state.json, which is bookkeeping rather than a
report, so answering the question does not mean writing a query.
"""
def __init__(self, state, path):
self.state = state
self.path = path
def failed(self):
tests = self.state.get(StateKeys.APWORLD_TESTS, {})
return sorted(
((key, record) for key, record in tests.items()
if record.get("outcome") == "failed"),
key=lambda item: item[1].get("game") or item[0],
)
def write(self):
with open(self.path, "w", encoding="utf-8") as handle:
failed = self.failed()
if not failed:
handle.write("No failed apworlds.\n")
return
for key, record in failed:
detail = (record.get("detail") or "").splitlines()[0]
handle.write(
f"{record.get('game') or key} ({key}): {detail}\n")

92
src/apworld_tester/pipeline/batch/fetcher.py Normal file
View File

@@ -0,0 +1,92 @@
"""Downloading the worlds a run was asked for but does not have."""
import os
import requests
from apworld_tester.core.config.secrets import Secrets
from apworld_tester.pipeline.batch.apworld_finder import ApworldFinder
from apworld_tester.pipeline.download.download_run import DownloadRun
from apworld_tester.pipeline.sheet.fetcher import SheetFetcher
from apworld_tester.pipeline.sheet.reader import SheetReader
class Fetcher:
"""Fills the download directory from the sheet, on demand.
That directory is a working directory this pipeline fills itself,
from the same sheet a full pass reads - not somewhere a user
points at their own copy. A world never fetched is work not done
yet rather than an error.
"""
def __init__(self, config, paths, root=None):
self.config = config
self.paths = paths
self.root = root or paths.downloads
self.finder = ApworldFinder(self.root)
def session(self):
session = requests.Session()
token = Secrets(self.paths).get("GITHUB_TOKEN")
if token:
session.headers["Authorization"] = f"token {token}"
return session
def listed(self):
"""The sheet's rows, fetching the sheet if this box has none."""
fetcher = SheetFetcher(self.config, self.paths)
if not os.path.exists(self.paths.sheet_html):
print(f"fetching the {fetcher.tab_name} sheet "
f"-> {self.paths.sheet_html}")
fetcher.ensure()
return SheetReader(self.config, self.paths.sheet_html).games()
@staticmethod
def row_for(term, listed):
"""The one sheet row a name selects.
An exact name wins over a substring, and a term still matching
several rows is refused rather than guessed at - downloading
and testing two worlds because a name was ambiguous is worse
than saying so.
"""
rows = [g for g in listed if term.lower() in g.name.lower()]
exact = [g for g in rows if g.name.casefold() == term.casefold()]
rows = exact or rows
if not rows:
raise LookupError(
f"{term!r} is not on the worlds sheet - nothing to "
"download and nothing to test")
if len(rows) > 1:
names = ", ".join(g.name for g in rows)
raise LookupError(f"{term!r} matches {len(rows)} sheet rows: "
f"{names}. Name one of them exactly.")
return rows[0]
def report(self, terms):
"""Say where each requested world landed, or raise."""
for term in terms:
found = self.finder.find(only=[term])
if not found:
raise LookupError(
f"no apworld could be obtained for {term!r}")
for name, path in found:
print(f" {name}: {path}")
def ensure(self, terms):
"""Download whichever of `terms` is not already here."""
missing = [t for t in terms if not self.finder.find(only=[t])]
if not missing:
return []
listed = self.listed()
wanted = [self.row_for(term, listed) for term in missing]
print(f"downloading {', '.join(g.name for g in wanted)} "
f"-> {self.root}")
run = DownloadRun(self.config, self.paths, self.session(),
root_directory=self.root)
_, skipped, _ = run.all(wanted)
for name, reason in skipped:
print(f" {name} could not be downloaded: {reason}")
self.report(missing)
return [g.name for g in wanted]

48
src/apworld_tester/pipeline/batch/options.py Normal file
View File

@@ -0,0 +1,48 @@
"""What one batch run was asked for, with the gaps filled in."""
from apworld_tester.core.state.keys import StateKeys
class Options:
"""The caller's choices, over config.yaml's defaults.
`repeats` of None means "whatever config.yaml says". An explicit
count trades confidence for time, and the cache check compares
against it, so a cheap run cannot masquerade as a full one.
"""
def __init__(self, paths, settings, **asked):
self.paths = paths
self.settings = settings
self.asked = asked
def chosen(self, key, default):
value = self.asked.get(key)
return default if value is None else value
def counted(self, key, default):
value = self.asked.get(key)
return default if value is None else int(value)
def resolve(self):
return {
"root_directory": self.chosen("root_directory",
self.paths.downloads),
"output_directory": self.chosen("output_directory",
self.paths.output),
"state_path": self.chosen("state_path", self.paths.state),
"roms_directory": self.chosen("roms_directory", self.paths.roms),
"jobs": self.chosen("jobs", self.settings.jobs),
"state_key": self.chosen(
"state_key", StateKeys.APWORLD_TESTS),
"companion_range": self.chosen("companion_range",
self.settings.companion_range),
"repeats": self.counted("repeats", self.settings.repeats),
"random_repeats": self.counted("random_repeats",
self.settings.random_repeats),
"spoiler": self.settings.spoiler,
"timeout": self.asked.get("timeout"),
"only": self.asked.get("only"),
"exact_only": bool(self.asked.get("exact_only")),
"core_games": self.asked.get("core_games") or (),
}

57
src/apworld_tester/pipeline/batch/orphaned_records.py Normal file
View File

@@ -0,0 +1,57 @@
"""Dropping test records whose apworld is no longer on disk."""
import os
from apworld_tester.core.state.keys import StateKeys
class OrphanedRecords:
"""Results for files that have since gone.
A record is keyed on the path its apworld had under the downloads
root, so a file that is removed - a release that renamed its asset,
a folder cleared by hand - leaves its result behind. Nothing reads
that record as stale: it still names a tag and a verdict, so the
status page goes on showing the world, and where the replacement
file sits beside it the game is reported twice.
Only records with a file behind them are considered. A download- or
core-stage record is keyed on the bare sheet name and never had one,
and Discontinued is what retires those.
"""
def __init__(self, store, downloads):
self.store = store
self.downloads = downloads
def orphaned(self, tests):
"""Every keyed-on-a-path record whose file is missing."""
return [key for key, record in tests.items()
if "/" in key
and (record or {}).get("stage") not in ("download", "core")
and not os.path.isfile(os.path.join(self.downloads, key))]
def safe(self):
"""Whether the downloads folder looks like itself.
An unreadable or empty root would make every record an orphan
and empty the page, which is the one outcome worth refusing:
a genuinely empty downloads directory has nothing to prune for
anyway, so declining costs nothing and guessing costs the lot.
"""
if not os.path.isdir(self.downloads):
return False
return any(os.scandir(self.downloads))
def prune(self):
"""Remove them, returning the keys that went."""
if not self.safe():
return []
state = self.store.load()
tests = state.get(StateKeys.APWORLD_TESTS, {})
gone = self.orphaned(tests)
for key in gone:
del tests[key]
if gone:
self.store.save(state)
return gone

104
src/apworld_tester/pipeline/batch/planner.py Normal file
View File

@@ -0,0 +1,104 @@
"""Splitting the work into known results and tests still to run."""
from apworld_tester.core.model.blacklist import Blacklist
from apworld_tester.core.model.name import Name
from apworld_tester.core.state.keys import StateKeys
from apworld_tester.pipeline.batch.cache_check import CacheCheck
from apworld_tester.pipeline.batch.record import TestRecord
class Planner:
"""What is already answered, and what has to be generated.
The blacklist is checked before the cache and is unaffected by a
force flag: "do not retry" is the entire reason a world is listed,
so a flag that retried it would defeat the point.
"""
def __init__(self, config, settings, tests, tag, roms_fingerprint,
force, repeats, random_repeats, companion_range,
releases=None):
self.config = config
self.settings = settings
self.tests = tests
self.tag = tag
self.roms_fingerprint = roms_fingerprint
self.force = force
self.repeats = repeats
self.random_repeats = random_repeats
self.companion_range = companion_range
self.blacklist = Blacklist(config)
# Keyed on the folded name: the work list names a row by
# its download folder, while game_releases keys on the
# sheet's own wording, and the two differ wherever the
# sheet uses a colon or its own capitalisation.
self.releases = {
Name(name).directory: (release or {}).get("published_at")
for name, release in (releases or {}).items()
}
self.wrote_state = False
def record_for(self, item):
game_name, apworld_path, key, _ = item
return TestRecord(game_name, key,
is_core=apworld_path is None)
def decline(self, item):
"""Write and return the result for a declined world."""
_, _, key, current_hash = item
record = self.record_for(item)
self.tests[key] = record.blacklisted(
blacklist=self.blacklist,
current_hash=current_hash,
tag=self.tag,
roms_fingerprint=self.roms_fingerprint,
)
self.wrote_state = True
return record.as_result(self.tests[key])
def confirm(self, previous):
"""Note that this answer was looked at and still holds.
A cached result is not re-run, so its tested_at stays at the
run that reached it - which on a page refreshed today reads as
a world nobody has looked at in weeks rather than as a result
nothing has been able to change. This is the date that says
which of the two it is, and only a download failure was
recording it, so it was missing from every row that passed.
"""
previous[StateKeys.LAST_CHECKED_AT] = TestRecord.now()
self.wrote_state = True
def known(self, item):
"""This world's answer already, or None if it must be tested."""
game_name, _, key, current_hash = item
if self.blacklist.listed(game_name):
return self.decline(item)
previous = self.tests.get(key)
cache = CacheCheck(self.settings, previous)
served = not self.force and cache.serves(
current_hash=current_hash,
tag=self.tag,
roms_fingerprint=self.roms_fingerprint,
repeats=self.repeats,
random_repeats=self.random_repeats,
companion_range=self.companion_range,
# A core world has no release of its own here, so this
# is None for one and the rule cannot fire: what dates
# it is the Archipelago build, already compared above.
released_at=self.releases.get(game_name),
)
if not served:
return None
self.confirm(previous)
return self.record_for(item).as_result(previous)
def plan(self, work):
results, to_test = [], []
for item in work:
answer = self.known(item)
if answer is None:
to_test.append(item)
else:
results.append(answer)
return results, to_test

111
src/apworld_tester/pipeline/batch/record.py Normal file
View File

@@ -0,0 +1,111 @@
"""The state entry written for one world."""
from datetime import datetime, timezone
from apworld_tester.core.model.blacklist import Blacklist
from apworld_tester.core.state.keys import StateKeys
class TestRecord:
"""What a finished, declined or undownloadable world records.
A core world has no file behind it, so it is marked as such and
carries no apworld path.
"""
def __init__(self, game_name, key=None, is_core=False):
self.game_name = game_name
self.key = key
self.is_core = is_core
@staticmethod
def now():
return datetime.now(timezone.utc).isoformat()
def cored(self, record):
if self.is_core:
record["stage"] = "core"
record["apworld"] = None
return record
def from_report(self, report, current_hash, tag, internal_game,
roms_fingerprint):
"""One finished test.
Single and multi verdicts stay separate: different questions.
Every mode's own verdict is kept too, so adding one needs no
change here. "repaired" is true when the file had to be fixed
before it would load - a real result, but for a corrected copy
rather than the file the author published.
"""
return self.cored({
"hash": current_hash,
"tag": tag,
"game": internal_game,
"outcome": report.get("outcome"),
"detail": report.get("detail"),
"elapsed_seconds": report.get("elapsed_seconds"),
"multi_outcome": report.get("multi_outcome"),
"multi_detail": report.get("multi_detail"),
"multi_elapsed_seconds": report.get("multi_elapsed_seconds"),
"multi_companions": report.get("multi_companions"),
"tests": report.get("tests"),
"roms_fingerprint": roms_fingerprint,
"tested_at": self.now(),
"repaired": bool(report.get("repaired")),
})
def blacklisted(self, blacklist, current_hash, tag,
roms_fingerprint):
"""A world this pipeline declines to test."""
return self.cored({
"hash": current_hash,
"tag": tag,
"game": self.game_name,
"outcome": Blacklist.OUTCOME,
"detail": blacklist.DETAIL,
"elapsed_seconds": None,
"multi_outcome": None,
"multi_detail": None,
"multi_elapsed_seconds": None,
"multi_companions": None,
"tests": None,
"roms_fingerprint": roms_fingerprint,
"tested_at": self.now(),
"blacklisted": True,
})
def download_failure(self, detail, tag):
"""A game whose apworld could not be fetched.
"unknown", never "failed": nothing was tested, so nothing is
known to be broken. Whether the link was a Discord message, an
unsupported host, or a release with no apworld in it, the
conclusion is the same - it could not be checked.
"""
return {
"game": self.game_name,
"apworld": None,
"outcome": "unknown",
"detail": detail,
"elapsed_seconds": None,
"stage": "download",
"tag": tag,
"hash": None,
"tested_at": self.now(),
StateKeys.LAST_CHECKED_AT: self.now(),
}
def as_result(self, record):
"""One entry of the caller-facing result list."""
return {
"game": self.game_name,
"apworld": None if self.is_core else self.key,
"outcome": record.get("outcome"),
"detail": record.get("detail"),
"elapsed_seconds": record.get("elapsed_seconds"),
"multi_outcome": record.get("multi_outcome"),
"multi_detail": record.get("multi_detail"),
"tests": record.get("tests"),
"skipped": True,
}

49
src/apworld_tester/pipeline/batch/reporter.py Normal file
View File

@@ -0,0 +1,49 @@
"""Saying what each finished world did."""
from apworld_tester.core.display.progress import Progress
from apworld_tester.core.model.verdict import Verdict
class Reporter:
"""One run's progress callback.
Holds its own bar rather than a module variable: two runs in one
process would otherwise share a bar and a tally.
"""
def __init__(self, config, bar=False):
self.config = config
self.wants_bar = bar
self.bar = None
@staticmethod
def marker(report):
return ("cached" if report.get("skipped")
else f"{report.get('elapsed_seconds')}s")
def verdicts(self, report):
text = f"{report['outcome']}"
multi = report.get("multi_outcome")
return f"{text} / multi:{multi}" if multi else text
def drawn(self, index, total, report):
if self.bar is None or self.bar.total != total:
self.bar = Progress(total=total, config=self.config)
self.bar.update(
index=index,
key=report.get("outcome") or "?",
text=report.get("game") or "",
fallback_line=(f"[{index}/{total}] {report['game']}: "
f"{self.verdicts(report)} ({self.marker(report)})"),
)
def printed(self, index, total, report):
print(f"[{index}/{total}] {report.get('game')}: "
f"{Verdict(report, self.config).label} ({self.marker(report)})",
flush=True)
def __call__(self, index, total, report):
if self.wants_bar:
self.drawn(index, total, report)
else:
self.printed(index, total, report)

95
src/apworld_tester/pipeline/batch/results.py Normal file
View File

@@ -0,0 +1,95 @@
"""The reports one run produced."""
from apworld_tester.core.model.modes import Modes
from apworld_tester.core.model.verdict import Verdict
class Results(list):
"""A run's reports, printable as a table.
A plain list of the same dicts the batch returns, so anything that
worked on those still works. The repr is the addition, because the
first thing anyone does with these is look at them.
"""
def __init__(self, reports, config):
super().__init__(reports)
self.config = config
def verdict(self, report):
return Verdict(report, self.config).name
@property
def counts(self):
tally = {}
for report in self:
name = self.verdict(report)
tally[name] = tally.get(name, 0) + 1
return dict(sorted(tally.items(), key=lambda item: -item[1]))
@property
def summary(self):
"""Outcome counts, multi verdicts under their own prefix.
They are a separate question, so they are not merged into one
total.
"""
counts = {}
for report in self:
outcome = report.get("outcome")
counts[outcome] = counts.get(outcome, 0) + 1
multi = report.get("multi_outcome")
if multi:
counts[f"multi:{multi}"] = counts.get(f"multi:{multi}", 0) + 1
return counts
#: Best to worst, so the shape of a run reads top to bottom.
LADDER = ("stable", "unstable", "flaky", "solo_only", "broken",
"unknown")
@property
def overview(self):
"""The run's verdicts, in the words the sheet uses.
The counts alone answer "what happened", but only in this
pipeline's vocabulary; a reader comparing against the sheet
needs the labels, and a reader watching a run needs them in a
fixed order rather than sorted by however this run landed.
"""
labels = self.config.value("stability", "labels", {})
counts = self.counts
names = [name for name in self.LADDER if name in counts]
names += [name for name in counts if name not in self.LADDER]
width = max((len(labels.get(name, name)) for name in names),
default=0)
lines = [f"{len(self)} worlds tested", ""]
lines += [
f" {labels.get(name, name):{width}s} {counts[name]:>4d}"
for name in names
]
return "\n".join(lines)
def row(self, report, width):
"""One world's line: name, verdict, every mode's outcome."""
mode_results = report.get("tests") or {}
per_mode = " ".join(
f"{mode}={(mode_results.get(mode) or {}).get('outcome') or '-'}"
for mode in Modes.ALL
)
marker = " (cached)" if report.get("skipped") else ""
return (f"{report.get('game') or '?':{width}s} "
f"{Verdict(report, self.config).label:12s} "
f"{per_mode}{marker}")
def __repr__(self):
if not self:
return "no worlds tested"
width = max(len(report.get("game") or "") for report in self)
lines = [
self.row(report, width)
for report in sorted(self, key=lambda r: r.get("game") or "")
]
if len(self) > 1:
lines += ["", " ".join(f"{name}: {count}"
for name, count in self.counts.items())]
return "\n".join(lines)

92
src/apworld_tester/pipeline/batch/run.py Normal file
View File

@@ -0,0 +1,92 @@
"""Testing worlds by hand, from the notebook or a shell."""
from apworld_tester.core.config.config import Config
from apworld_tester.core.config.paths import Paths
from apworld_tester.core.config.run_settings import RunSettings
from apworld_tester.core.state.lock import PipelineLock
from apworld_tester.core.state.scratch import ScratchState
from apworld_tester.core.state.store import StateStore
from apworld_tester.pipeline.batch.apworld_finder import ApworldFinder
from apworld_tester.pipeline.batch.batch import Batch
from apworld_tester.pipeline.batch.fetcher import Fetcher
from apworld_tester.pipeline.batch.reporter import Reporter
from apworld_tester.pipeline.batch.results import Results
from apworld_tester.pipeline.batch.version import Version
from apworld_tester.pipeline.build.archipelago import ArchipelagoBuild
class TestRun:
"""Test one world, several, or every downloaded one.
`live` records into the real state file - off by default, because a
hand-run test is an experiment. `cached` only affects a whole-set
run; a named world is always retested.
"""
def __init__(self, game=None, repeats=None, random_repeats=None, jobs=2,
live=False, cached=False, root_directory=None, quiet=False,
update=False):
self.config = Config.load()
self.paths = Paths(self.config)
self.settings = RunSettings(self.config)
self.game = game
self.repeats = repeats
self.random_repeats = random_repeats
self.jobs = jobs
self.live = live
self.cached = cached
self.root = root_directory or self.paths.downloads
self.quiet = quiet
self.update = update
@property
def state_path(self):
return ScratchState(self.paths, live=self.live).path
def worlds(self):
"""The folders to test, downloading any that are missing.
Resolved to real folder names and matched exactly from here on,
so the set tested is the set that was just named.
"""
if self.game is None:
return None
terms = [self.game] if isinstance(self.game, str) else list(self.game)
Fetcher(self.config, self.paths, self.root).ensure(terms)
finder = ApworldFinder(self.root)
return [name for term in terms for name in finder.resolve(term)]
def announce(self, tag, only):
"""What is about to run, and nothing else."""
named = ", ".join(only) if only else "every downloaded apworld"
print(f"archipelago {tag}")
print(f"testing {named}")
def run(self):
"""The reports, printable as a table."""
store = StateStore(self.paths.state)
build = ArchipelagoBuild(self.config, self.paths)
tag, _ = Version(self.config, self.paths, store).prepare(
update=self.update)
only = self.worlds()
build.ensure_test_image(tag)
state_path = self.state_path
if not self.quiet:
self.announce(tag, only)
with PipelineLock(self.paths.lock):
return Results(Batch(
config=self.config,
paths=self.paths,
settings=self.settings,
image=build.test_image_tag(tag),
tag=tag,
force=not self.cached,
on_progress=None if self.quiet else Reporter(self.config),
root_directory=self.root,
state_path=state_path,
only=only,
exact_only=only is not None,
jobs=self.jobs,
repeats=self.repeats,
random_repeats=self.random_repeats,
).run(), self.config)

86
src/apworld_tester/pipeline/batch/runner.py Normal file
View File

@@ -0,0 +1,86 @@
"""Running the tests the cache could not answer."""
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from apworld_tester.pipeline.batch.record import TestRecord
from apworld_tester.pipeline.generation.tester import WorldTester
class Runner:
"""Tests each outstanding world and persists what it did."""
def __init__(self, config, paths, store, options, state, tests, tag,
image, roms_fingerprint):
self.config = config
self.paths = paths
self.store = store
self.options = options
self.state = state
self.tests = tests
self.tag = tag
self.tester = WorldTester(config, paths, image)
self.roms_fingerprint = roms_fingerprint
self.lock = threading.Lock()
@staticmethod
def promotion_name(report, game_name):
"""The name Archipelago itself registered this world under.
Recorded for promotion's collision check, while the report's
own "game" becomes the sheet folder name for display. A world
that never loaded has no real name, so it falls back to a
marked sheet name - otherwise every such entry would collide
under one None key and none would promote.
"""
return report.get("game") or f"unverified:{game_name}"
def store_record(self, key, record):
"""Persist after every test, not just at the end, so a killed
run does not lose the tests it already completed.
"""
with self.lock:
self.tests[key] = record
self.store.save(self.state)
def test_one(self, game_name, apworld_path, key, current_hash):
options = self.options
report = self.tester.test(
apworld_path=apworld_path,
game=None if apworld_path else game_name,
output_dir=options["output_directory"],
timeout=options["timeout"],
companion_range=options["companion_range"],
spoiler=options["spoiler"],
repeats=options["repeats"],
companion_pool=options["pool"],
random_repeats=options["random_repeats"],
)
record = TestRecord(game_name, key,
is_core=apworld_path is None)
self.store_record(key, record.from_report(
report=report,
current_hash=current_hash,
tag=self.tag,
internal_game=self.promotion_name(report, game_name),
roms_fingerprint=self.roms_fingerprint,
))
report["game"] = game_name
report["apworld"] = key
report["skipped"] = False
return report
def run(self, to_test, total, completed, on_progress):
"""Run the outstanding tests, reporting as they land."""
results = []
reporting = threading.Lock()
with ThreadPoolExecutor(max_workers=self.options["jobs"]) as pool:
futures = [pool.submit(self.test_one, *args) for args in to_test]
for future in as_completed(futures):
report = future.result()
results.append(report)
with reporting:
completed += 1
if on_progress:
on_progress(completed, total, report)
return results

92
src/apworld_tester/pipeline/batch/session.py Normal file
View File

@@ -0,0 +1,92 @@
"""Testing worlds by hand, from the notebook or a shell."""
from apworld_tester.core.config.config import Config
from apworld_tester.core.config.paths import Paths
from apworld_tester.core.config.run_settings import RunSettings
from apworld_tester.core.state.lock import PipelineLock
from apworld_tester.core.state.scratch import ScratchState
from apworld_tester.core.state.store import StateStore
from apworld_tester.pipeline.batch.apworld_finder import ApworldFinder
from apworld_tester.pipeline.batch.batch import Batch
from apworld_tester.pipeline.batch.fetcher import Fetcher
from apworld_tester.pipeline.batch.reporter import Reporter
from apworld_tester.pipeline.batch.results import Results
from apworld_tester.pipeline.batch.version import Version
from apworld_tester.pipeline.build.archipelago import ArchipelagoBuild
class TestSession:
"""Test one world, several, or every downloaded one.
`live` records into the real state file - off by default, because a
hand-run test is an experiment. `cached` only affects a whole-set
run; a named world is always retested.
"""
def __init__(self, game=None, repeats=None, random_repeats=None, jobs=2,
live=False, cached=False, root_directory=None, quiet=False,
update=False):
self.config = Config.load()
self.paths = Paths(self.config)
self.settings = RunSettings(self.config)
self.game = game
self.repeats = repeats
self.random_repeats = random_repeats
self.jobs = jobs
self.live = live
self.cached = cached
self.root = root_directory or self.paths.downloads
self.quiet = quiet
self.update = update
@property
def state_path(self):
return ScratchState(self.paths, live=self.live).path
def worlds(self):
"""The folders to test, downloading any that are missing.
Resolved to real folder names and matched exactly from here on,
so the set tested is the set that was just named.
"""
if self.game is None:
return None
terms = [self.game] if isinstance(self.game, str) else list(self.game)
Fetcher(self.config, self.paths, self.root).ensure(terms)
finder = ApworldFinder(self.root)
return [name for term in terms for name in finder.resolve(term)]
def announce(self, tag, only):
"""What is about to run, and nothing else."""
named = ", ".join(only) if only else "every downloaded apworld"
print(f"archipelago {tag}")
print(f"testing {named}")
def run(self):
"""The reports, printable as a table."""
store = StateStore(self.paths.state)
build = ArchipelagoBuild(self.config, self.paths)
tag, _ = Version(self.config, self.paths, store).prepare(
update=self.update)
only = self.worlds()
build.ensure_test_image(tag)
state_path = self.state_path
if not self.quiet:
self.announce(tag, only)
with PipelineLock(self.paths.lock):
return Results(Batch(
config=self.config,
paths=self.paths,
settings=self.settings,
image=build.test_image_tag(tag),
tag=tag,
force=not self.cached,
on_progress=None if self.quiet else Reporter(self.config),
root_directory=self.root,
state_path=state_path,
only=only,
exact_only=only is not None,
jobs=self.jobs,
repeats=self.repeats,
random_repeats=self.random_repeats,
).run(), self.config)

87
src/apworld_tester/pipeline/batch/version.py Normal file
View File

@@ -0,0 +1,87 @@
"""Which Archipelago version a run tests against."""
from apworld_tester.core.state.keys import StateKeys
from apworld_tester.pipeline.build.archipelago import ArchipelagoBuild
from apworld_tester.pipeline.build.docker import Docker
class Version:
"""Resolve the version, and build it if this machine lacks it."""
def __init__(self, config, paths, store):
self.paths = paths
self.store = store
self.build = ArchipelagoBuild(config, paths)
def resolve(self):
"""The version to test, and where that choice came from.
A pin is a decision already made, so nothing else revisits it.
Failing that the recorded version wins, then whichever image is
on this machine - a test runs entirely inside the container, so
an image already here is everything it needs.
"""
pinned = self.build.pinned_version
if pinned:
return pinned, "pinned in config.yaml"
recorded = self.store.load().get(StateKeys.ARCHIPELAGO_TAG)
if recorded:
return recorded, "from state.json"
local = next(iter(self.build.local_tags()), None)
return (local, "newest image on this machine") if local else (None,
None)
def ensure_built(self, tag):
"""Build what this version needs, if anything is missing.
Building needs the Archipelago source, which is the one job the
checkout has - so this is also the only path that clones.
"""
if tag is None:
print("no Archipelago image on this machine - cloning the "
"newest release to build one (several minutes) ...")
built, _, _, _ = self.build.update_and_build(self.store)
return built, "just built"
if not Docker.image_exists(self.build.image_tag(tag)):
print(f"Archipelago {tag} is not built on this machine - "
"cloning and building it (several minutes) ...")
self.build.build_version(tag)
return tag, None
def note_others(self, tag, source):
"""Say so when an unpinned run is not using the newest image.
An unpinned tag can come from an image that happens to be lying
around, so a run can quietly answer for a version that is not
the current one. A pinned tag was chosen deliberately.
"""
local = self.build.local_tags()
if self.build.pinned_version or not local or tag == local[0]:
return source
return f"{source}, but {local[0]} is also built here"
def update(self):
"""Resolve the newest release, build it, and record it.
Never automatic. A new version invalidates every cached result,
so the full re-test it forces is the caller's decision, not a
side effect of running. A pinned version stays pinned: the
resolution behind this honours the pin, so an update then only
ensures that version is built.
"""
tag, previous, changed, _ = self.build.update_and_build(self.store)
if self.build.pinned_version:
return tag, "pinned in config.yaml"
if not changed:
return tag, "already the newest release"
if previous is None:
return tag, "newest release, recorded for the first time"
return tag, f"updated from {previous}"
def prepare(self, update=False):
"""The version to test against, built if it is missing."""
if update:
return self.update()
tag, source = self.resolve()
tag, built = self.ensure_built(tag)
return tag, built or self.note_others(tag, source)

41
src/apworld_tester/pipeline/batch/work_list.py Normal file
View File

@@ -0,0 +1,41 @@
"""Everything one run will consider testing."""
import os
from apworld_tester.core.model.fingerprint import Fingerprint
from apworld_tester.pipeline.batch.apworld_finder import ApworldFinder
class WorkList:
"""Each world to consider, with its cache key and hash.
A core world is tested by name with no file behind it, so it is
keyed on the bare sheet name - the same "no slash means no apworld"
convention download failures follow - and gets a stand-in hash
whose real content is the Archipelago build beside it.
"""
#: Stands in for a core world's file hash. Its real content is
#: the Archipelago build, so the version compared beside it in the
#: cache check is what decides staleness.
CORE_HASH = "core"
def __init__(self, root, only=None, exact=False, core_games=()):
self.root = root
self.only = only
self.exact = exact
self.core_games = core_games
def items(self):
found = ApworldFinder(self.root).find(only=self.only,
exact=self.exact)
work = [
(name, path, os.path.relpath(path, self.root),
Fingerprint.of_file(path))
for name, path in found
]
return work + [
(name, None, name, WorkList.CORE_HASH)
for name in self.core_games
if self.only is None or name in self.only
]

0
src/apworld_tester/pipeline/download/__init__.py Normal file
View File

82
src/apworld_tester/pipeline/download/asset_matcher.py Normal file
View File

@@ -0,0 +1,82 @@
"""Choosing which asset in a release belongs to this game."""
import re
from apworld_tester.core.model.name import Name
class AssetMatcher:
"""Matches release assets and release titles to one sheet row."""
#: Title slugs that name no game at all, so they are never read as
#: evidence that a release belongs to a different sheet row. Not
#: configuration: they are the words every release title uses about
#: itself, and a release titled "Apworld Release" claims nobody.
UNINFORMATIVE = frozenset({
"archipelago",
"apworld",
"release",
"latest",
"update",
})
def __init__(self, config, game_name):
self.config = config
self.game = Name(game_name)
def named(self, assets, exact):
"""The assets whose filename names this game."""
matched = []
for asset in assets:
name = Name(asset["name"].rsplit(".", 1)[0]).slug
if (name == self.game.slug) if exact else (name in self.game.slug):
matched.append(asset)
return matched
def select(self, assets):
"""The assets to download for this game."""
if len(assets) <= 1:
return assets
for matches in (self.named(assets, True), self.named(assets, False)):
if len(matches) == 1:
return matches
return assets
def title_slug(self, release):
"""A release's own title, reduced to the game it is for."""
title = release.get("name") or release.get("tag_name") or ""
title = re.sub(r"\bv?\d+(?:\.\d+)*\b", " ", title)
title = re.sub(r"\bapworld\b", " ", title, flags=re.IGNORECASE)
return Name(title).slug
def claimed_by(self, title, sheet_names):
"""The other sheet row this title names, or None.
Exact equality only. This is not "the title looks a bit off" -
it is positive evidence that a DIFFERENT row is named, which is
the only thing strong enough to skip a release the link points
at. "Mega Man X2" is exactly a row and is skipped when resolving
"Mega Man X"; "Mega Man X1" is nobody's row and is accepted.
"""
if not sheet_names or not self.game.text:
return None
if not title or title in self.UNINFORMATIVE:
return None
if title == self.game.slug:
return None
for other in sheet_names:
slug = Name(other).slug
if slug != self.game.slug and slug == title:
return other
return None
def release_claimed(self, release, sheet_names):
return self.claimed_by(self.title_slug(release), sheet_names)
def identity_claimed(self, found_games, sheet_names):
"""Whether a downloaded file identifies as another row's game."""
for found in found_games or []:
other = self.claimed_by(Name(found).slug, sheet_names)
if other:
return other
return None

23
src/apworld_tester/pipeline/download/auth_error.py Normal file
View File

@@ -0,0 +1,23 @@
"""A token failure, which is the whole run's problem."""
import requests
class GitHubAuthError(RuntimeError):
"""A bad or expired token affects every remaining request."""
@classmethod
def matching(cls, game, error):
"""This error as an auth failure, or None if it is not one.
Raised loudly so the whole pool stops, rather than quietly
failing game after game against a token that will not work.
"""
if not isinstance(error, requests.HTTPError):
return None
response = error.response
status = response.status_code if response is not None else None
if status not in (401, 403):
return None
return cls(f"GitHub rejected the request for {game.name} with "
f"{status} - check .env / token expiry.")

58
src/apworld_tester/pipeline/download/context.py Normal file
View File

@@ -0,0 +1,58 @@
"""What every download shares."""
import os
from apworld_tester.core.model.name import Name
from apworld_tester.pipeline.download.asset_matcher import AssetMatcher
class DownloadContext:
"""Who is downloading what, and where it goes.
One object rather than the same six arguments threaded through
every handler and helper.
"""
def __init__(self, config, paths, session, game, identifier=None,
sheet_names=None, root_directory=None):
self.config = config
self.paths = paths
self.session = session
self.game = game
self.identifier = identifier
self.sheet_names = sheet_names
self.root = root_directory or paths.downloads
self.matcher = AssetMatcher(config, game.name)
@property
def target_directory(self):
return os.path.join(self.root, Name(self.game.name).directory)
def target_path(self, filename):
return os.path.join(self.target_directory, filename)
@property
def plain_headers(self):
"""Headers for a host that is not GitHub.
The session carries a GitHub token, which must never be sent to
an unrelated site.
"""
return {"Authorization": None, "Accept": "*/*"}
@property
def json_headers(self):
"""Headers for another forge's JSON API, without the token."""
return {"Authorization": None, "Accept": "application/json"}
def wrong_game(self, path):
"""Why this file is another game's, or None to keep it."""
if self.identifier is None:
return None
identity = self.identifier.identify(path)
if not identity.get("ran"):
return None
found = identity.get("games") or []
if any(Name(self.game.name).matches(name) for name in found):
return None
return found or identity.get("detail")

156
src/apworld_tester/pipeline/download/download_run.py Normal file
View File

@@ -0,0 +1,156 @@
"""Fetching every game's apworld, concurrently."""
import os
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests
from apworld_tester.core.state.keys import StateKeys
from apworld_tester.core.state.store import StateStore
from apworld_tester.pipeline.download.auth_error import GitHubAuthError
from apworld_tester.pipeline.download.context import DownloadContext
from apworld_tester.pipeline.download.downloader import Downloader
from apworld_tester.pipeline.download.superseded_assets import (
SupersededAssets,
)
class DownloadRun:
"""One pass over the sheet's rows.
`sheet_names` should be the FULL sheet: a file can identify as a
core row, which never reaches here.
"""
def __init__(self, config, paths, session, root_directory=None,
identifier=None, jobs=None, sheet_names=None,
previous_releases=None):
self.config = config
self.paths = paths
self.session = session
self.root = root_directory or paths.downloads
self.identifier = identifier
self.jobs = jobs or int(config.value("general", "threads", 4))
self.sheet_names = sheet_names
self.previous_releases = previous_releases or {}
@staticmethod
def split(games):
"""The games worth a network call, and those answered now.
A row with no link has nothing to look up, so it never takes a
slot in the thread pool.
"""
downloadable, skipped = [], []
for game in games:
if not game.release and not game.links:
skipped.append((game.name, "no release link in the sheet"))
else:
downloadable.append(game)
return downloadable, skipped
def context_for(self, game):
return DownloadContext(
config=self.config,
paths=self.paths,
session=self.session,
game=game,
identifier=self.identifier,
sheet_names=self.sheet_names,
root_directory=self.root,
)
def one(self, game):
"""Download one game, which never raises for that game alone.
A network failure is that game's own result, not the run's -
except an auth failure, which is everyone's.
"""
try:
downloaded, reason, info = Downloader(
context=self.context_for(game),
previous=self.previous_releases.get(game.name),
).run()
except requests.RequestException as error:
auth = GitHubAuthError.matching(game, error)
if auth is not None:
raise auth from error
return game, None, str(error), None
return game, downloaded, reason, info
@staticmethod
def source_url(game):
"""Where this apworld came from, for the status page.
game.release is only ever a real releases link, so for anything
resolved through the plain links the first one stands in.
"""
return game.release or (game.links[0] if game.links else None)
def record(self, finished, releases, skipped, completed, total,
on_asset, on_game):
"""Record one finished game and tell the caller about it.
The release is recorded even on failure - it was still found,
so its version is worth showing. on_game is called for failures
too, unlike on_asset: a progress bar has to count every game
that finished, not only the ones that produced a file.
"""
game, downloaded, reason, info = finished
releases[game.name] = {"url": self.source_url(game), **(info or {})}
if downloaded is None:
skipped.append((game.name, reason))
if on_game:
on_game(completed, total, game.name, False)
return 0
paths = [path for path, _ in downloaded]
for path in SupersededAssets(os.path.dirname(paths[0]),
paths).prune():
print(f"{game.name}: removed superseded {os.path.basename(path)}"
f" - the release no longer offers it")
for path, status in downloaded:
if on_asset:
on_asset(game.name, path, status)
if on_game:
on_game(completed, total, game.name, True)
return len(downloaded)
@staticmethod
def result_of(future, executor):
"""One finished download, stopping the pool on auth failure."""
try:
return future.result()
except GitHubAuthError:
executor.shutdown(cancel_futures=True)
raise
def all(self, games, on_asset=None, on_game=None):
"""Every game's apworld: how many files, what was skipped, and
which release each row resolved to.
Recording holds a lock, so on_asset always runs serialized and
a caller's own progress counter needs none of its own.
"""
self.sheet_names = self.sheet_names or [g.name for g in games]
downloadable, skipped = self.split(games)
releases, lock = {}, threading.Lock()
total = completed = 0
with ThreadPoolExecutor(max_workers=self.jobs) as executor:
futures = [executor.submit(self.one, g) for g in downloadable]
for future in as_completed(futures):
finished = self.result_of(future, executor)
with lock:
completed += 1
total += self.record(
finished, releases, skipped, completed,
len(downloadable), on_asset, on_game)
return total, skipped, releases
def record_releases(self, releases, state_path=None):
"""Write each game's resolved release into the state file."""
store = StateStore(state_path or self.paths.state)
state = store.load()
state.setdefault(StateKeys.GAME_RELEASES, {}).update(releases)
store.save(state)
return state

75
src/apworld_tester/pipeline/download/downloader.py Normal file
View File

@@ -0,0 +1,75 @@
"""Fetching one sheet row's apworld."""
from apworld_tester.pipeline.download.handlers.gitea_release import (
GiteaRelease,
)
from apworld_tester.pipeline.download.handlers.github_file import (
GitHubFile,
)
from apworld_tester.pipeline.download.handlers.github_release import (
GitHubRelease,
)
from apworld_tester.pipeline.download.handlers.github_repo import (
GitHubRepo,
)
from apworld_tester.pipeline.download.handlers.gitlab_file import (
GitLabFile,
)
from apworld_tester.pipeline.download.handlers.gitlab_release import (
GitLabRelease,
)
from apworld_tester.pipeline.download.handlers.plain_file import PlainFile
from apworld_tester.pipeline.download.unusable_links import UnusableLinks
from apworld_tester.pipeline.download.url_shapes import UrlShapes
class Downloader:
"""Whatever shape this game's link is, get the apworld.
A releases link is resolved directly; anything else is tried
against every handler in turn, most specific first, so an exact
file beats browsing a whole repo and the plain-URL handler can
never shadow a forge-specific one.
"""
def __init__(self, context, previous=None):
self.context = context
self.shapes = UrlShapes(context.config)
self.previous = previous
def handlers(self):
return (
GitHubFile(self.shapes),
GitHubRepo(self.shapes),
GitLabFile(self.shapes),
GitLabRelease(self.shapes),
GiteaRelease(self.shapes, previous=self.previous),
PlainFile(self.shapes),
)
def from_release_link(self):
handler = GitHubRelease(self.shapes, previous=self.previous)
parsed = handler.parse(self.context.game.release)
if parsed is None:
return None, "not a recognized releases URL", None
return handler.download(parsed, self.context)
def from_link(self, link):
"""This link's result, or None if no handler recognised it."""
for handler in self.handlers():
parsed = handler.parse(link)
if parsed is not None:
return handler.download(parsed, self.context)
return None
def run(self):
"""The files downloaded, why none were, and the release info."""
game = self.context.game
if game.release:
return self.from_release_link()
for link in game.links:
result = self.from_link(link)
if result is not None:
return result
return None, UnusableLinks(self.context.config, game.links).detail(), \
None

0
src/apworld_tester/pipeline/download/handlers/__init__.py Normal file
View File

45
src/apworld_tester/pipeline/download/handlers/base.py Normal file
View File

@@ -0,0 +1,45 @@
"""What every link handler shares."""
import os
from apworld_tester.pipeline.download.local_file import LocalFile
class LinkHandler:
"""One recognised shape of sheet link, and how to fetch from it.
parse() returning None means "not my shape", which is different from
"mine, and it failed": the caller only moves on to the next link in
the first case.
"""
def __init__(self, shapes):
self.shapes = shapes
def parse(self, link):
raise NotImplementedError
def download(self, parsed, context):
raise NotImplementedError
def file(self, context, filename):
return LocalFile(context.session, context.target_path(filename))
def collect(self, candidates, context, fetch, verify):
"""Download each candidate, dropping any that is another game."""
downloaded, kept, rejected = [], [], []
for candidate in candidates:
path, status = fetch(candidate)
if status in LocalFile.NEEDS_DOWNLOAD and verify:
mismatch = context.wrong_game(path)
if mismatch is not None:
os.remove(path)
rejected.append((candidate["name"], mismatch))
continue
downloaded.append((path, status))
kept.append(candidate)
return downloaded, kept, rejected
def rejected_detail(self, rejected, subject):
details = "; ".join(f"{name} -> {found}" for name, found in rejected)
return f"downloaded {subject} didn't match this game: {details}"

44
src/apworld_tester/pipeline/download/handlers/gitea_release.py Normal file
View File

@@ -0,0 +1,44 @@
"""A releases page on a self-hosted Gitea or Forgejo instance."""
from apworld_tester.pipeline.download.handlers.base import LinkHandler
from apworld_tester.pipeline.download.release_downloader import (
ReleaseDownloader,
)
class GiteaRelease(LinkHandler):
"""Codeberg and friends.
The release JSON is close enough to GitHub's that ReleaseDownloader
handles it unchanged, given headers that omit the GitHub token.
"""
def __init__(self, shapes, previous=None):
super().__init__(shapes)
self.previous = previous
def parse(self, link):
return self.shapes.gitea_releases(link)
def latest(self, context, host, owner, repository):
response = context.session.get(
f"https://{host}/api/v1/repos/{owner}/{repository}"
"/releases/latest",
headers=context.json_headers,
)
if response.status_code == 404:
return None
response.raise_for_status()
return response.json()
def download(self, parsed, context):
host, owner, repository = parsed
release = self.latest(context, host, owner, repository)
if release is None:
return None, "no release found on this host", None
return ReleaseDownloader(
context=context,
release=release,
headers=context.plain_headers,
previous_published_at=(self.previous or {}).get("published_at"),
).run()

58
src/apworld_tester/pipeline/download/handlers/github_file.py Normal file
View File

@@ -0,0 +1,58 @@
"""A link straight at an apworld committed in a GitHub repo."""
import os
import urllib.parse
import requests
from apworld_tester.pipeline.build.archipelago import (
ArchipelagoBuild,
)
from apworld_tester.pipeline.download.local_file import LocalFile
from apworld_tester.pipeline.download.handlers.base import LinkHandler
class GitHubFile(LinkHandler):
"""A committed file, named exactly by the sheet.
No identity check: the link names one file, so there is no second
candidate a name comparison could choose between.
"""
def parse(self, link):
return self.shapes.github_file(link)
def download(self, parsed, context):
"""Fetch the file as it currently stands on that branch."""
owner, repository, branch, path = parsed
filename = os.path.basename(urllib.parse.unquote(path))
local = self.file(context, filename)
url = ("https://raw.githubusercontent.com/"
f"{owner}/{repository}/{branch}/{path}")
status = local.status(local.remote_size(url))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url)
published = self.published_at(context, owner, repository, path, branch)
return [(local.path, status)], None, {"published_at": published}
def published_at(self, context, owner, repository, path, branch):
"""When this file was last committed, or None.
Best effort: a blob link has no release to read a date from, and
a download should not fail over display information.
"""
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
"/commits",
params={"path": path, "sha": branch, "per_page": 1},
)
response.raise_for_status()
commits = response.json()
except requests.RequestException:
return None
if not commits:
return None
commit = commits[0].get("commit", {})
return ((commit.get("committer") or {}).get("date")
or (commit.get("author") or {}).get("date"))

81
src/apworld_tester/pipeline/download/handlers/github_release.py Normal file
View File

@@ -0,0 +1,81 @@
"""A GitHub releases page, the sheet's usual link."""
from apworld_tester.pipeline.download.handlers.base import LinkHandler
from apworld_tester.pipeline.download.release_downloader import (
ReleaseDownloader,
)
from apworld_tester.pipeline.download.release_identity import (
ReleaseIdentity,
)
from apworld_tester.pipeline.download.release_picker import ReleasePicker
from apworld_tester.pipeline.download.walk_back import WalkBack
class GitHubRelease(LinkHandler):
"""Resolve which release the link means, then fetch its apworld."""
def __init__(self, shapes, previous=None):
super().__init__(shapes)
self.previous = previous
def parse(self, link):
return self.shapes.releases(link)
def resolve(self, parsed, context):
"""The release this link resolves to, or None.
A hand-pinned tag prefix wins, then a link naming one exact
tag, then the search term. Runs every pass: it is the one call
that can report a new release, which is the only thing able to
change a previously failed answer.
"""
owner, repository, term, tag = parsed
picker = ReleasePicker(context, owner, repository, term)
overrides = context.config.get("release_overrides") or {}
prefix = overrides.get(context.game.name)
if prefix:
return picker.by_tag_prefix(prefix)
if tag:
return picker.by_tag(tag)
return picker.pick()
def fetch(self, parsed, context, release):
"""This release's apworld, walking back if it ships none.
Only that one cause is walked back from. "Ships another game's
apworld" must not be, since an older release of a multi-world
repo is precisely the wrong answer there.
"""
owner, repository = parsed[0], parsed[1]
downloader = ReleaseDownloader(
context=context,
release=release,
# What this row's date already says, so a re-upload of
# unchanged bytes keeps it instead of moving it forward.
previous_published_at=(self.previous or {}).get("published_at"),
)
downloaded, reason, info = downloader.run()
if (downloaded is not None or context.identifier is None
or downloader.cause != ReleaseDownloader.NO_APWORLD):
return downloaded, reason, info
fallback, reason = WalkBack(
context=context,
owner=owner,
repository=repository,
release=release,
).run(reason)
return fallback if fallback is not None else (None, reason, info)
def download(self, parsed, context):
release = self.resolve(parsed, context)
if release is None:
return None, "no release matched the search term in the link", None
identity = ReleaseIdentity(release)
repeat = identity.repeated(self.previous)
if repeat is not None:
return None, repeat, identity.failure(
{"published_at": None}, repeat)
downloaded, reason, info = self.fetch(parsed, context, release)
if downloaded is None:
return None, reason, identity.failure(info, reason)
return downloaded, reason, info

115
src/apworld_tester/pipeline/download/handlers/github_repo.py Normal file
View File

@@ -0,0 +1,115 @@
"""A bare repository link, for a world never cut into a release."""
import os
import requests
from apworld_tester.pipeline.build.archipelago import (
ArchipelagoBuild,
)
from apworld_tester.pipeline.download.local_file import LocalFile
from apworld_tester.core.model.apworld import ApworldFile
from apworld_tester.pipeline.download.handlers.base import LinkHandler
class GitHubRepo(LinkHandler):
"""Browses the repo's own file tree for an apworld."""
def parse(self, link):
return self.shapes.github_repo(link)
def resolve_branch(self, context, owner, repository, branch):
"""The real default branch, so a version reads as a name.
"HEAD" works as a ref, but resolving it means the status page
shows a branch rather than the literal string.
"""
if branch != "HEAD":
return branch
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}")
response.raise_for_status()
return response.json().get("default_branch") or "HEAD"
except requests.RequestException:
return "HEAD"
def tree_entries(self, context, owner, repository, branch):
"""Every apworld in the tree, with its size.
One call: the trees API takes recursive=1 itself, and returns
each blob's size for free.
"""
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
f"/git/trees/{branch}",
params={"recursive": "1"},
)
response.raise_for_status()
except requests.RequestException:
return []
return [
(entry["path"], entry.get("size"))
for entry in response.json().get("tree", [])
if entry.get("type") == "blob"
and entry["path"].lower().endswith(ApworldFile.SUFFIX)
]
def candidates(self, entries):
return [
{"name": os.path.basename(path), "path": path, "size": size}
for path, size in entries
]
def download(self, parsed, context):
owner, repository, branch = parsed
branch = self.resolve_branch(context, owner, repository, branch)
entries = self.tree_entries(context, owner, repository, branch)
if not entries:
return None, ("repo link has no release and no .apworld file "
"found in it"), None
chosen = context.matcher.select(self.candidates(entries))
downloaded, kept, rejected = self.collect(
candidates=chosen,
context=context,
fetch=lambda candidate: self.fetch(
candidate, context, owner, repository, branch),
verify=len(chosen) > 1,
)
if not downloaded:
detail = self.rejected_detail(rejected, "file(s) from repo")
return None, detail, None
published = self.published_at(
context, owner, repository, kept[0]["path"], branch)
return downloaded, None, {"published_at": published}
def fetch(self, candidate, context, owner, repository, branch):
local = self.file(context, candidate["name"])
status = local.status(candidate.get("size"))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(
f"https://raw.githubusercontent.com/{owner}/{repository}"
f"/{branch}/{candidate['path']}")
return local.path, status
def published_at(self, context, owner, repository, path, branch):
"""When the first kept file was last committed.
Only the first: one API call per candidate otherwise.
"""
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
"/commits",
params={"path": path, "sha": branch, "per_page": 1},
)
response.raise_for_status()
commits = response.json()
except requests.RequestException:
return None
if not commits:
return None
commit = commits[0].get("commit", {})
return ((commit.get("committer") or {}).get("date")
or (commit.get("author") or {}).get("date"))

44
src/apworld_tester/pipeline/download/handlers/gitlab_file.py Normal file
View File

@@ -0,0 +1,44 @@
"""A link straight at an apworld committed in a GitLab repo."""
import os
import urllib.parse
import requests
from apworld_tester.pipeline.download.local_file import LocalFile
from apworld_tester.pipeline.download.handlers.base import LinkHandler
class GitLabFile(LinkHandler):
"""A committed file on GitLab, named exactly by the sheet."""
def parse(self, link):
return self.shapes.gitlab_file(link)
def download(self, parsed, context):
project_path, branch, path = parsed
filename = os.path.basename(urllib.parse.unquote(path))
local = self.file(context, filename)
url = f"https://gitlab.com/{project_path}/-/raw/{branch}/{path}"
headers = context.json_headers
status = local.status(local.remote_size(url, headers=headers))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url, headers=headers)
published = self.published_at(context, project_path, path, branch)
return [(local.path, status)], None, {"published_at": published}
def published_at(self, context, project_path, path, branch):
"""When this file was last committed, or None."""
encoded = urllib.parse.quote(project_path, safe="")
try:
response = context.session.get(
f"https://gitlab.com/api/v4/projects/{encoded}"
"/repository/commits",
params={"path": path, "ref_name": branch, "per_page": 1},
headers=context.json_headers,
)
response.raise_for_status()
commits = response.json()
except requests.RequestException:
return None
return commits[0].get("committed_date") if commits else None

74
src/apworld_tester/pipeline/download/handlers/gitlab_release.py Normal file
View File

@@ -0,0 +1,74 @@
"""A GitLab releases page."""
import urllib.parse
from apworld_tester.core.model.apworld import ApworldFile
from apworld_tester.pipeline.download.handlers.base import LinkHandler
from apworld_tester.pipeline.download.local_file import LocalFile
class GitLabRelease(LinkHandler):
"""The newest release of a GitLab project.
GitLab's assets are shaped nothing like GitHub's - a release-level
list of {name, url, direct_asset_url} with no per-asset timestamp -
so they are reshaped here rather than reusing the release path.
"""
def parse(self, link):
return self.shapes.gitlab_releases(link)
def latest(self, context, project_path):
"""GitLab's own newest-release shortcut, or None."""
encoded = urllib.parse.quote(project_path, safe="")
response = context.session.get(
f"https://gitlab.com/api/v4/projects/{encoded}"
"/releases/permalink/latest",
headers=context.json_headers,
)
if response.status_code == 404:
return None
response.raise_for_status()
return response.json()
def assets(self, release, context):
"""The release's apworld links, in GitHub's asset shape."""
links = (release.get("assets") or {}).get("links", [])
return [
{
"name": link["name"],
"browser_download_url": (link.get("direct_asset_url")
or link["url"]),
}
for link in links
if (link.get("name") or "").lower().endswith(ApworldFile.SUFFIX)
]
def fetch(self, asset, context):
local = self.file(context, asset["name"])
url = asset["browser_download_url"]
headers = context.plain_headers
status = local.status(local.remote_size(url, headers=headers))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url, headers=headers)
return local.path, status
def download(self, parsed, context):
release = self.latest(context, parsed)
if release is None:
return None, "no GitLab release found for this project", None
info = {"published_at": (release.get("released_at")
or release.get("created_at"))}
assets = self.assets(release, context)
if not assets:
return None, "latest GitLab release has no .apworld asset", info
chosen = context.matcher.select(assets)
downloaded, _, rejected = self.collect(
candidates=chosen,
context=context,
fetch=lambda asset: self.fetch(asset, context),
verify=len(chosen) > 1,
)
if not downloaded:
return None, self.rejected_detail(rejected, "asset(s)"), info
return downloaded, None, info

48
src/apworld_tester/pipeline/download/handlers/plain_file.py Normal file
View File

@@ -0,0 +1,48 @@
"""A plain URL ending in .apworld, on any host."""
import email.utils
import requests
from apworld_tester.pipeline.download.local_file import LocalFile
from apworld_tester.pipeline.download.handlers.base import LinkHandler
class PlainFile(LinkHandler):
"""Any other host. Tried last, so a forge handler always wins."""
def parse(self, link):
return self.shapes.plain_file(link)
def download(self, parsed, context):
url, filename = parsed
local = self.file(context, filename)
size, published = self.remote_facts(context, url)
status = local.status(size)
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url, headers=context.plain_headers)
return [(local.path, status)], None, {"published_at": published}
def remote_facts(self, context, url):
"""The size and date one HEAD request can offer.
A plain URL has no version to compare, so size is the whole
staleness check. Neither is required.
"""
try:
response = context.session.head(
url, headers=context.plain_headers, allow_redirects=True)
response.raise_for_status()
except requests.RequestException:
return None, None
length = response.headers.get("Content-Length")
size = int(length) if length and length.isdigit() else None
return size, self.modified_at(response.headers.get("Last-Modified"))
def modified_at(self, modified):
if not modified:
return None
try:
return email.utils.parsedate_to_datetime(modified).isoformat()
except (TypeError, ValueError):
return None

83
src/apworld_tester/pipeline/download/local_file.py Normal file
View File

@@ -0,0 +1,83 @@
"""Deciding whether a local copy is still the current one."""
import datetime
import os
import requests
from apworld_tester.core.model.instant import Instant
class LocalFile:
"""One file on disk, against what the remote currently offers.
"It exists" is not "it is current": an author can replace an
asset's content in place, same filename, same version. Two signals
catch that - the asset's own timestamp, and its size - and both are
needed. A replacement that happens to be the same length is
invisible to size alone, and a remote that reports no length at all
makes size no signal whatsoever.
"""
#: The statuses that mean the file has to be fetched. The other
#: two - "verified" and "assumed" - mean the copy on disk stands.
NEEDS_DOWNLOAD = ("missing", "stale")
def __init__(self, session, path):
self.session = session
self.path = path
def outdated(self, remote_updated_at):
"""Whether the remote was touched after this copy arrived.
mtime is when download() wrote this file, so a remote timestamp
later than it means the copy on disk predates what is being
offered now. Without this, a release that replaced its asset
with one of the same length was never fetched, and the recorded
test result went on describing content that no longer exists -
while the page showed the new release's date beside it.
"""
stamp = Instant.of(remote_updated_at)
if stamp is None:
return False
arrived = datetime.datetime.fromtimestamp(
os.path.getmtime(self.path), datetime.timezone.utc)
return arrived < stamp
def status(self, remote_size, remote_updated_at=None):
"""Whether this file is missing, stale, verified or assumed.
The timestamp is checked before the size, and before a missing
size falls through to "assumed": it is the stronger signal, and
the two cases it catches are exactly the ones size cannot.
"""
if not os.path.exists(self.path):
return "missing"
if self.outdated(remote_updated_at):
return "stale"
if remote_size is None:
return "assumed"
if os.path.getsize(self.path) == remote_size:
return "verified"
return "stale"
def remote_size(self, url, headers=None):
"""The size a HEAD request reports, or None."""
try:
response = self.session.head(url, headers=headers,
allow_redirects=True)
response.raise_for_status()
except requests.RequestException:
return None
length = response.headers.get("Content-Length")
return int(length) if length and length.isdigit() else None
def download(self, url, headers=None):
"""Stream a URL into this path, creating its directory."""
response = self.session.get(url, stream=True, headers=headers)
response.raise_for_status()
os.makedirs(os.path.dirname(self.path), exist_ok=True)
with open(self.path, "wb") as handle:
for chunk in response.iter_content(chunk_size=65536):
handle.write(chunk)
return self.path

179
src/apworld_tester/pipeline/download/release_downloader.py Normal file
View File

@@ -0,0 +1,179 @@
"""Fetching the apworld a release carries."""
import os
from apworld_tester.core.model.apworld import ApworldFile
from apworld_tester.core.model.fingerprint import Fingerprint
from apworld_tester.pipeline.download.local_file import LocalFile
from apworld_tester.pipeline.download.zip_search import ZipSearch
class ReleaseDownloader:
"""A GitHub-shaped release, and the apworld it ships.
Shared with Gitea and Forgejo, whose release JSON is close enough
that the same logic applies unchanged.
"""
#: Why run() came back empty. The caller decides what to do about
#: it from this, never from the message beside it: a release that
#: ships no apworld can be walked back from, and one that ships
#: another game's must not be.
NO_APWORLD = "no_apworld"
WRONG_GAME = "wrong_game"
NO_APWORLD_DETAIL = "latest release has no .apworld asset"
def __init__(self, context, release, headers=None,
require_identity=False, previous_published_at=None):
self.context = context
self.release = release
self.headers = headers
self.require_identity = require_identity
self.previous_published_at = previous_published_at
self.cause = None
def named_assets(self):
return [
asset for asset in self.release.get("assets", [])
if asset["name"].lower().endswith(ApworldFile.SUFFIX)
]
def missing_detail(self):
detail = self.NO_APWORLD_DETAIL
if any(asset["name"].lower().endswith(ZipSearch.SUFFIX)
for asset in self.release.get("assets", [])):
detail += " (checked inside its .zip asset(s) too, none found)"
return detail
def assets(self):
"""Every apworld this release offers, zips included."""
found = self.named_assets()
if found:
return found
return ZipSearch(self.context, self.release, self.headers).assets()
def verifies(self, assets):
"""Whether a file has to prove which game it is.
With one candidate there is nothing to choose between, and the
sheet's link points at this release - so that file is the row's
apworld whatever its World class calls itself. A walk-back
release overrides that, since the link no longer points at it.
"""
return len(assets) > 1 or self.require_identity
@staticmethod
def touched_at(asset):
"""When this asset was last written, as the remote reports it.
The same value published_at() reports for the row, read here so
the two cannot disagree: a file whose recorded release date is
newer than the copy on disk is a file that was never fetched.
"""
return asset.get("updated_at") or asset.get("created_at")
def fetch(self, asset):
"""The asset's path, how it got there, and whether it changed.
An asset whose timestamp moved is fetched again - the copy on
disk can no longer be trusted to be current - and only then can
the two be compared. Identical bytes mean the author re-uploaded
the same file, so nothing about this row actually changed. A
file lifted out of a zip is taken as changed: ZipSearch does its
own fetching, and there is no earlier copy here to compare to.
"""
if "extracted_path" in asset:
return asset["extracted_path"], asset["status"], True
local = LocalFile(self.context.session,
self.context.target_path(asset["name"]))
url = asset["browser_download_url"]
status = local.status(
local.remote_size(url, headers=self.headers),
remote_updated_at=self.touched_at(asset),
)
if status not in LocalFile.NEEDS_DOWNLOAD:
return local.path, status, False
before = (Fingerprint.of_file(local.path)
if os.path.exists(local.path) else None)
local.download(url, headers=self.headers)
return local.path, status, Fingerprint.of_file(local.path) != before
def rejection(self, path, assets):
"""Why this file is not ours, or None to keep it.
A walk-back file that registered no world at all is kept:
nothing was proven either way, and generation testing reports
the import failure far more usefully than a silent drop.
"""
identity = self.context.identifier.identify(path)
if not identity.get("ran"):
return None
found = identity.get("games") or []
if self.require_identity and not found and len(assets) == 1:
return None
other = self.context.matcher.identity_claimed(
found, self.context.sheet_names)
matches = bool(found) and self.context.matcher.game.matches(
found[0])
if other:
return f'{found} - belongs to the "{other}" row'
if self.verifies(assets) and not matches:
return found or identity.get("detail")
return None
def checks(self, status, assets):
return (status in LocalFile.NEEDS_DOWNLOAD
and self.context.identifier is not None
and (self.verifies(assets) or self.context.sheet_names))
def published_at(self, kept):
"""When the content this row now holds was published.
Each asset carries its own timestamp; a release's own stays
fixed at creation even when its files are replaced later, so
the assets' is the one to read. But that timestamp moves for a
re-upload of identical bytes too, and then it dates the upload
rather than the world: the page would show a release newer than
the test beside it while the tested file never changed, and the
Compatibility column would call a world current on the strength
of an upload that added nothing. So only an asset whose content
actually changed sets this date, and otherwise the one already
recorded stands. With nothing recorded there is nothing to
hold, and the asset's own timestamp is the best available.
"""
stamps = [(self.touched_at(asset), changed)
for asset, changed in kept]
stamps = [pair for pair in stamps if pair[0]]
replaced = [stamp for stamp, changed in stamps if changed]
if replaced:
return max(replaced)
if self.previous_published_at:
return self.previous_published_at
return max((stamp for stamp, _ in stamps), default=None)
def run(self):
"""The files downloaded, why none were, and the release info."""
assets = self.assets()
if not assets:
self.cause = self.NO_APWORLD
return None, self.missing_detail(), {"published_at": None}
assets = self.context.matcher.select(assets)
downloaded, kept, rejected = [], [], []
for asset in assets:
path, status, changed = self.fetch(asset)
reason = (self.rejection(path, assets)
if self.checks(status, assets) else None)
if reason is not None:
os.remove(path)
rejected.append((asset["name"], reason))
continue
downloaded.append((path, status))
kept.append((asset, changed))
if not downloaded:
self.cause = self.WRONG_GAME
details = "; ".join(f"{name} -> {found}"
for name, found in rejected)
return None, (f"downloaded asset(s) didn't match this game: "
f"{details}"), {"published_at": None}
return downloaded, None, {"published_at": self.published_at(kept)}

55
src/apworld_tester/pipeline/download/release_identity.py Normal file
View File

@@ -0,0 +1,55 @@
"""Remembering which release a failure was decided on."""
from apworld_tester.core.state.keys import StateKeys
class ReleaseIdentity:
"""What makes a resolved release the same one as last run.
Its tag and publication date, plus every asset's name and size. The
sizes matter because an author can replace an asset in place
without cutting a new tag.
"""
def __init__(self, release):
self.release = release
@property
def value(self):
assets = sorted(
(asset.get("name"), asset.get("size"))
for asset in self.release.get("assets") or []
)
return {
"tag": self.release.get("tag_name"),
"published_at": self.release.get("published_at"),
"assets": [{"name": name, "size": size} for name, size in assets],
}
def repeated(self, previous):
"""The recorded reason, if this exact release failed before.
A release that ships no apworld, or only another row's, fails
the same way every run - and finding that out costs a download
of every zip asset plus a container run to identify what came
back. None of that can reach a different answer while the
release is unchanged. Keyed on the release rather than the
link, because the link is usually /releases/latest and a newly
published release is exactly what would fix a world in this
state. Only ever holds a verdict about content: a network error
escapes as an exception and a container that could not start
keeps the file, so neither is cached here.
"""
if not previous:
return None
if previous.get(StateKeys.FAILED_RELEASE) != self.value:
return None
return previous.get(StateKeys.FAILED_REASON) or None
def failure(self, info, reason):
"""A failure recorded against the release it was decided on."""
return {
**(info or {}),
StateKeys.FAILED_RELEASE: self.value,
StateKeys.FAILED_REASON: reason,
}

151
src/apworld_tester/pipeline/download/release_picker.py Normal file
View File

@@ -0,0 +1,151 @@
"""Which release a sheet link means."""
import re
import urllib.parse
from apworld_tester.pipeline.build.archipelago import (
ArchipelagoBuild,
)
from apworld_tester.core.model.apworld import ApworldFile
from apworld_tester.core.model.name import Name
class ReleasePicker:
"""One repository's releases, narrowed to this game's.
A repo dedicated to one world can answer with its latest release. A
repo hosting several has to be matched on the link's search term and
the release's own title, and must never answer with a release that
another sheet row already claims.
"""
def __init__(self, context, owner, repository, term=None):
self.context = context
self.owner = owner
self.repository = repository
self.term = term
self.matcher = context.matcher
@property
def repo_url(self):
return (f"{ArchipelagoBuild.API_ROOT}/repos/"
f"{self.owner}/{self.repository}")
def all(self, max_pages=10):
"""Every release, newest first, across every page.
GitHub pages at 30 by default and several source repos hold
more than that, which would silently hide real releases from
every lookup below.
"""
releases = []
for page in range(1, max_pages + 1):
response = self.context.session.get(
f"{self.repo_url}/releases",
params={"per_page": 100, "page": page},
)
response.raise_for_status()
batch = response.json()
releases.extend(batch)
if len(batch) < 100:
break
return releases
def by_tag(self, tag):
"""The release for exactly this tag, or None."""
encoded = urllib.parse.quote(tag, safe="")
response = self.context.session.get(
f"{self.repo_url}/releases/tags/{encoded}")
if response.status_code == 404:
return None
response.raise_for_status()
return response.json()
def by_tag_prefix(self, prefix):
"""The newest release whose tag starts with this prefix."""
for release in self.all():
if (release.get("tag_name") or "").startswith(prefix):
return release
return None
def claimed(self, release):
return self.matcher.release_claimed(release, self.context.sheet_names)
def newest(self):
"""GitHub's own newest full release.
Right whenever it resolves, since it skips in-progress
prereleases. The list's first entry is the fallback for a repo
where every release is flagged prerelease, which /latest
excludes rather than relaxing.
"""
response = self.context.session.get(f"{self.repo_url}/releases/latest")
if response.status_code != 404:
response.raise_for_status()
return response.json()
releases = self.all()
return releases[0] if releases else None
def titled_for_this_row(self):
for release in self.all():
if self.matcher.title_slug(release) == self.matcher.game.slug:
return release
return None
def without_a_term(self):
"""The release to use when the link named none.
"Newest in the repo" is a fine answer for a repo holding one
world and a coin flip for one holding twelve, so a newest that
is titled for somebody else's row gives way to the one titled
for this row.
"""
latest = self.newest()
if latest is None or not self.claimed(latest):
return latest
return self.titled_for_this_row() or latest
def by_title(self, releases):
"""The first release whose title holds every word of the term.
The sheet's term is built for GitHub's own search box, which
matches per word: requiring the whole phrase rejects real terms
like "Jurassic Park Randomizer (SNES) Jurassic Park", where the
link generator repeats the game name. That looseness is what
lets a term land on a sibling world, so a release titled for
another row is skipped and the search continues.
"""
words = re.findall(r"[a-z0-9]+", self.term.lower())
if not words:
return None
for release in releases:
title = (f"{release.get('name') or ''} "
f"{release.get('tag_name') or ''}").lower()
if all(word in title for word in words) \
and not self.claimed(release):
return release
return None
def by_asset_name(self, releases):
"""The release carrying an asset named for this game.
A fork mirroring many worlds often tags every release by build
date alone, so the game's name appears only in which apworlds
the release bundles.
"""
term = Name(self.term).slug
for release in releases:
for asset in release.get("assets", []):
name = asset["name"]
if not name.lower().endswith(ApworldFile.SUFFIX):
continue
if Name(name.rsplit(".", 1)[0]).slug == term:
return release
return None
def pick(self):
"""The release this link means, or None."""
if self.term is None:
return self.without_a_term()
releases = self.all()
return self.by_title(releases) or self.by_asset_name(releases)

58
src/apworld_tester/pipeline/download/superseded_assets.py Normal file
View File

@@ -0,0 +1,58 @@
"""Clearing apworlds a game's release no longer offers."""
import os
class SupersededAssets:
"""The files left behind when an author renames their asset.
A download writes each asset under its own filename, so a release
that renamed one leaves the previous name sitting beside it. Both
are then real files in the game's folder, and everything downstream
keys on the path rather than the game - two work items, two test
records, two rows on the status page, and a promotion that refuses
both because they claim one game name between them.
Only ever run for a game whose download succeeded: with nothing
fetched there is no current set to compare against, and pruning on
a failed network call would delete the copy that still works.
"""
#: What this class considers. A game's folder holds nothing else,
#: but naming it here keeps a stray file from being deleted on the
#: strength of living in the wrong directory.
SUFFIX = ".apworld"
def __init__(self, directory, kept):
self.directory = directory
self.kept = {os.path.abspath(path) for path in kept}
def stale(self):
"""Every apworld in the folder that this download did not write."""
if not self.kept or not os.path.isdir(self.directory):
return []
found = []
for entry in sorted(os.listdir(self.directory)):
if not entry.lower().endswith(self.SUFFIX):
continue
path = os.path.join(self.directory, entry)
if os.path.isfile(path) and os.path.abspath(path) not in self.kept:
found.append(path)
return found
def prune(self):
"""Delete them, returning what went.
A file that cannot be removed is not worth failing a download
over - it means the next run sees a duplicate, which is what
was already happening.
"""
removed = []
for path in self.stale():
try:
os.remove(path)
except OSError as error:
print(f"could not remove superseded {path}: {error}")
continue
removed.append(path)
return removed

56
src/apworld_tester/pipeline/download/unusable_links.py Normal file
View File

@@ -0,0 +1,56 @@
"""Why none of a game's links could be downloaded from."""
import urllib.parse
class UnusableLinks:
"""Categorise links no handler recognised.
Reaching here means every recognised shape was tried against every
link, so the status page can say something concrete instead of a
generic "failed".
"""
#: Hosts that can never serve a downloadable file, whatever handler
#: is written for them: a Discord link is a channel or message, not
#: an asset. Not configuration - no setting makes Discord fetchable.
UNREACHABLE = {
"discord.com": "Discord link only (channel/message link, not a "
"downloadable file) - can't be fetched automatically",
"discordapp.com": "Discord link only (channel/message link, not a "
"downloadable file) - can't be fetched "
"automatically",
}
def __init__(self, config, links):
self.config = config
self.links = links
def domains(self):
"""Each link's host, or a reason it has none."""
found = []
for link in self.links:
parsed = urllib.parse.urlparse(link)
if parsed.scheme not in ("http", "https"):
protocol = parsed.scheme or "(none)"
return None, f"unsupported link protocol {protocol!r}: {link}"
if not parsed.netloc:
return None, f"not a valid URL: {link}"
netloc = parsed.netloc.lower()
found.append(netloc[4:] if netloc.startswith("www.") else netloc)
return found, None
def detail(self):
domains, problem = self.domains()
if problem is not None:
return problem
for domain in domains:
if domain in self.UNREACHABLE:
return self.UNREACHABLE[domain]
if "github.com" in domains:
return ("GitHub link found, but not a recognized "
"releases/repo/file URL")
if "gitlab.com" in domains:
return "GitLab link found, but not a recognized releases/blob URL"
hosts = ", ".join(sorted(set(domains)))
return f"no supported download source (custom host: {hosts})"

168
src/apworld_tester/pipeline/download/update_history.py Normal file
View File

@@ -0,0 +1,168 @@
"""Every distinct apworld file this pipeline has seen, per game.
One file per game, written only when that game gains content it has not
seen before - so a run touches the handful of games that actually
changed rather than rewriting one shared blob. Deliberately NOT under
downloads/: that directory is a cache and gets cleared to force a clean
re-download, which is harmless for everything else in it. History is the
one thing that cannot be rebuilt afterwards - which version existed on
which date has no other source once it is gone.
"""
import os
from datetime import datetime, timezone
from apworld_tester.core.model.apworld import ApworldFile
from apworld_tester.core.model.fingerprint import Fingerprint
from apworld_tester.core.model.name import Name
from apworld_tester.core.state.keys import StateKeys
from apworld_tester.core.state.store import StateStore
class UpdateHistory:
"""What each game's apworld was, and when.
Identity is the file's sha256, so a re-download of unchanged content
adds nothing and recording is safe to repeat within a run. The
game's sheet name is stored inside the file as well as encoded in
its name, because the filename is a folded directory name and a
sheet rename would otherwise leave a file nothing could identify.
"""
def __init__(self, paths, directory=None):
self.directory = directory or paths.history
def path_for(self, game_name):
folded = Name(game_name).directory
return os.path.join(self.directory, f"{folded}.json")
def load(self, game_name):
return StateStore.read(
self.path_for(game_name),
{"game": game_name, "watching_since": None,
"baseline_at": None, "files": []})
def save(self, game_name, history):
os.makedirs(self.directory, exist_ok=True)
StateStore(self.path_for(game_name)).save(history)
#: Everything recorded before this field existed was gathered from
#: whatever the hosts happened to say, at whatever moment the file
#: was first downloaded, and none of it measures how often a world
#: changes under observation. The first pass to touch a game draws
#: a line: that instant is the game's one starting point, and only
#: changes seen after it count. See UpdateCadence, which reads it.
@staticmethod
def watching_since(history, stamp):
"""When this pipeline began watching this game.
The earliest content it has already recorded, so a file written
before this field existed still gets an honest answer rather
than a window that starts the day the field was added. Only a
game with no history at all starts watching now.
"""
seen = [item.get("first_seen_at") for item in history["files"]
if item.get("first_seen_at")]
return min(seen) if seen else stamp
def entry_for(self, apworld_path, sha256, released_at, source_url, now):
"""One file's record.
`released_at` is the upload date the download stage saw;
first_seen_at is when this pipeline observed it, which is the
only honest timestamp for a file whose host reports no date.
"""
manifest = ApworldFile(apworld_path).manifest or {}
return {
"version": manifest.get("world_version"),
"released_at": released_at,
"first_seen_at": (now or datetime.now(timezone.utc)).isoformat(),
"sha256": sha256,
"filename": os.path.basename(apworld_path),
"source_url": source_url,
}
def record_file(self, game_name, apworld_path, sha256, released_at,
source_url=None, now=None):
"""Add this file to the game's history if its content is new.
A game whose content has not changed still records that it was
looked at. How often a world updates is only meaningful against
how long it has been watched, and without that a game seen once
and a game seen all year are indistinguishable.
"""
history = self.load(game_name)
now = now or datetime.now(timezone.utc)
opened = not history.get("watching_since")
if opened:
history["watching_since"] = self.watching_since(
history, now.isoformat())
if not history.get("baseline_at"):
opened = True
history["baseline_at"] = now.isoformat()
if any(item.get("sha256") == sha256 for item in history["files"]):
if opened:
self.save(game_name, history)
return None
entry = self.entry_for(apworld_path, sha256, released_at,
source_url, now)
history["game"] = game_name
history["files"].append(entry)
# Oldest first. released_at is missing often enough that it
# cannot be the only key - first_seen_at always exists and
# breaks those ties in the order the pipeline saw them.
history["files"].sort(
key=lambda item: (item.get("released_at") or "",
item.get("first_seen_at") or ""))
self.save(game_name, history)
return entry
def files_in(self, game_directory):
return [
name for name in sorted(os.listdir(game_directory))
if name.lower().endswith(ApworldFile.SUFFIX)
and os.path.isfile(os.path.join(game_directory, name))
]
def record_all(self, state, root_directory, now=None):
"""Record every apworld on disk that is not yet in history.
Hashes the files directly rather than reading apworld_tests:
that dict only gains a hash once a world has been TESTED, so a
file downloaded this run would not appear until the next one -
and this runs right after downloading, which is the point at
which a new version is known.
"""
releases = state.get(StateKeys.GAME_RELEASES) or {}
# game_releases is keyed by the sheet name while the download
# folder is that name folded - map one to the other rather than
# assuming they match as written.
by_folder = {Name(name).directory: info
for name, info in releases.items()}
added = []
if not os.path.isdir(root_directory):
return added
for folder in sorted(os.listdir(root_directory)):
game_directory = os.path.join(root_directory, folder)
if not os.path.isdir(game_directory):
continue
added += self.record_folder(
folder, game_directory, by_folder.get(folder) or {}, now)
return added
def record_folder(self, folder, game_directory, release, now):
"""Every new apworld in one game's download folder."""
added = []
for filename in self.files_in(game_directory):
path = os.path.join(game_directory, filename)
entry = self.record_file(
game_name=folder,
apworld_path=path,
sha256=Fingerprint.of_file(path),
released_at=release.get("published_at"),
source_url=release.get("url"),
now=now,
)
if entry:
added.append((folder, entry))
return added

98
src/apworld_tester/pipeline/download/url_shapes.py Normal file
View File

@@ -0,0 +1,98 @@
"""Recognising the shapes a sheet link can take."""
import os
import re
import urllib.parse
from apworld_tester.core.model.apworld import ApworldFile
class UrlShapes:
"""The link shapes this pipeline knows how to fetch from."""
def __init__(self, config):
self.config = config
def releases(self, url):
"""Owner, repo, search term and tag from a releases link.
A link naming one exact tag is read as such: without it, a repo
hosting several worlds would fall back to "latest", which drifts
to whichever world was tagged most recently.
"""
match = re.match(
r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+)/releases",
url or "", re.IGNORECASE)
if match is None:
return None
parsed = urllib.parse.urlparse(url)
terms = urllib.parse.parse_qs(parsed.query).get("q", [])
tag_match = re.match(
r"^/[^/]+/[^/]+/releases/tag/([^/?#]+)", parsed.path,
re.IGNORECASE)
return (
match.group(1),
match.group(2),
terms[0].strip('"') if terms else None,
urllib.parse.unquote(tag_match.group(1)) if tag_match else None,
)
def github_file(self, url):
"""Owner, repo, branch and path from a committed-file link."""
match = re.match(
r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+)"
r"/(?:blob|raw)/([^/]+)/(.+\.apworld)$",
url or "", re.IGNORECASE)
return match.groups() if match else None
def github_repo(self, url):
"""Owner, repo and branch from a bare repository link.
Deliberately strict, so it does not swallow issues or wiki
links: those should keep falling through to the generic "not a
releases URL" message rather than triggering a tree search that
could never succeed.
"""
match = re.match(
r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+?)"
r"(?:/tree/([^/?#]+))?/?$",
url or "", re.IGNORECASE)
if match is None:
return None
owner, repository, branch = match.groups()
return owner, repository, branch or "HEAD"
def gitlab_file(self, url):
"""Project path, branch and file path from a GitLab blob link."""
match = re.match(
r"^https?://(?:www\.)?gitlab\.com/([^/]+(?:/[^/]+)*?)"
r"/-/blob/([^/]+)/(.+\.apworld)$",
url or "", re.IGNORECASE)
return match.groups() if match else None
def gitlab_releases(self, url):
"""The project path from a GitLab releases link."""
match = re.match(
r"^https?://(?:www\.)?gitlab\.com/([^/]+(?:/[^/]+)*?)"
r"/-/releases(?:[/?#]|$)",
url or "", re.IGNORECASE)
return match.group(1) if match else None
def gitea_releases(self, url):
"""Host, owner and repo from a Gitea-style releases link."""
match = re.match(
r"^https?://(?!(?:www\.)?(?:github|gitlab)\.com)"
r"([^/]+)/([^/]+)/([^/]+)/releases(?:[/?#]|$)",
url or "", re.IGNORECASE)
return match.groups() if match else None
def plain_file(self, url):
"""A URL ending in .apworld on any host, and its filename."""
parsed = urllib.parse.urlparse(url or "")
if parsed.scheme not in ("http", "https") or not parsed.netloc:
return None
path = urllib.parse.unquote(parsed.path)
if not path.lower().endswith(ApworldFile.SUFFIX):
return None
filename = os.path.basename(path)
return (url, filename) if filename != ApworldFile.SUFFIX else None

77
src/apworld_tester/pipeline/download/walk_back.py Normal file
View File

@@ -0,0 +1,77 @@
"""Falling back to an older release that still ships an apworld."""
import requests
from apworld_tester.core.model.apworld import ApworldFile
from apworld_tester.pipeline.download.release_downloader import (
ReleaseDownloader,
)
from apworld_tester.pipeline.download.release_picker import ReleasePicker
class WalkBack:
"""One narrow rescue: the newest release forgot the apworld.
An author cuts a release and re-attaches only a mod zip, leaving
the world perfectly downloadable one release back. Every other
failure is left alone on purpose - in particular "did not match
this game", where an older release holding a different game's
apworld is precisely the wrong answer.
"""
def __init__(self, context, owner, repository, release):
self.context = context
self.owner = owner
self.repository = repository
self.release = release
def candidate(self, release):
"""Whether this is an older release that ships an apworld.
Matches on asset filename alone: this runs over a repo's whole
history, and opening every zip in it would cost far more than
the problem is worth.
"""
cutoff = self.release.get("published_at")
published_at = release.get("published_at")
return bool(
not release.get("draft")
and release.get("tag_name") != self.release.get("tag_name")
and published_at
and (not cutoff or published_at < cutoff)
and any(asset["name"].lower().endswith(ApworldFile.SUFFIX)
for asset in release.get("assets", []))
)
def older(self):
"""The newest earlier release with an apworld, by date.
Ordering comes from published_at rather than the order GitHub
returns, which follows the underlying tag and is not the same
thing.
"""
picker = ReleasePicker(self.context, self.owner, self.repository)
try:
releases = picker.all()
except requests.RequestException:
return None
candidates = [r for r in releases if self.candidate(r)]
if not candidates:
return None
return max(candidates, key=lambda release: release["published_at"])
def run(self, reason):
"""The fallback's result, or None and a combined reason."""
older = self.older()
if older is None:
return None, reason
fallback = ReleaseDownloader(
context=self.context,
release=older,
require_identity=True,
).run()
if fallback[0] is not None:
fallback[2]["fallback_release_tag"] = older.get("tag_name")
return fallback, None
return None, (f"{reason}; older release {older.get('tag_name')!r} "
f"was tried too: {fallback[1]}")

96
src/apworld_tester/pipeline/download/zip_search.py Normal file
View File

@@ -0,0 +1,96 @@
"""Looking for an apworld inside a release's zip assets."""
import os
import tempfile
import zipfile
import requests
from apworld_tester.core.model.apworld import ApworldFile
from apworld_tester.pipeline.download.local_file import LocalFile
class ZipSearch:
"""For a release that publishes a zip instead of an apworld.
Some repos attach a source snapshot with the built apworld nested
inside. Every zip is re-downloaded to scratch space each time, since
the member's own filename - what a normal download caches on - is
not known until the zip is opened.
"""
#: The extension a release's archive asset is recognised by.
SUFFIX = ".zip"
def __init__(self, context, release, headers=None):
self.context = context
self.release = release
self.headers = headers
def zip_assets(self):
return [
asset for asset in self.release.get("assets", [])
if asset["name"].lower().endswith(ZipSearch.SUFFIX)
]
def members(self, zip_path):
"""Every apworld inside the zip, with its size."""
try:
with zipfile.ZipFile(zip_path) as archive:
return [
(info.filename, info.file_size)
for info in archive.infolist()
if not info.filename.endswith("/")
and info.filename.lower().endswith(ApworldFile.SUFFIX)
]
except zipfile.BadZipFile:
return []
def extract(self, zip_path, member):
"""Unpack one member into the game's folder."""
target = self.context.target_path(os.path.basename(member))
os.makedirs(os.path.dirname(target), exist_ok=True)
with zipfile.ZipFile(zip_path) as archive:
with archive.open(member) as source:
with open(target, "wb") as handle:
handle.write(source.read())
return target
def unpack(self, zip_path, members, zip_asset):
"""The members as assets, already on disk."""
results = []
for member, size in members:
target = self.context.target_path(os.path.basename(member))
status = LocalFile(self.context.session, target).status(size)
if status in LocalFile.NEEDS_DOWNLOAD:
self.extract(zip_path, member)
results.append({
"name": os.path.basename(member),
"updated_at": zip_asset.get("updated_at"),
"created_at": zip_asset.get("created_at"),
"extracted_path": target,
"status": status,
})
return results
def search_one(self, zip_asset):
with tempfile.TemporaryDirectory() as scratch:
zip_path = os.path.join(scratch, zip_asset["name"])
local = LocalFile(self.context.session, zip_path)
try:
local.download(zip_asset["browser_download_url"],
headers=self.headers)
except requests.RequestException:
return []
members = self.members(zip_path)
if not members:
return []
return self.unpack(zip_path, members, zip_asset)
def assets(self):
"""The first zip that holds an apworld wins."""
for zip_asset in self.zip_assets():
found = self.search_one(zip_asset)
if found:
return found
return []

0
src/apworld_tester/pipeline/generation/__init__.py Normal file
View File

99
src/apworld_tester/pipeline/generation/command.py Normal file
View File

@@ -0,0 +1,99 @@
"""Building the docker invocations the drivers run in."""
import os
from apworld_tester.core.model.apworld import ApworldFile
class ContainerCommand:
"""The `docker run` for one driver, and the mounts it needs.
The paths below are where this pipeline mounts things inside its
own container. Both ends of that contract live in this repository -
these mount arguments and the drivers that receive them - so they
are part of the code rather than something to configure.
"""
#: Where each driver script is mounted.
DRIVER_PATH = "/app/run_test.py"
IDENTIFY_DRIVER_PATH = "/app/identify_apworld.py"
VERIFY_DRIVER_PATH = "/app/verify_companions.py"
#: Where the apworld under test, the seeds and the ROMs go.
APWORLD_DIRECTORY = "/app/custom_worlds"
OUTPUT_DIRECTORY = "/data/output"
DATA_DIRECTORY = "/app/data"
#: A writable cache directory. The image runs as uid 1000 but
#: leaves HOME as "/", which is root-owned, so anything resolving
#: "~/.cache" fails with PermissionError before generation starts.
CACHE_ENVIRONMENT = {"HOME": "/tmp", "XDG_CACHE_HOME": "/tmp/.cache"}
def __init__(self, config, paths, image):
self.config = config
self.paths = paths
self.image = image
@property
def prefix(self):
"""The part of every invocation that never varies."""
return ["docker", "run", "--rm", "--network", "none",
"--entrypoint", "python", *self.cache_environment]
@property
def cache_environment(self):
args = []
for name, value in self.CACHE_ENVIRONMENT.items():
args += ["-e", f"{name}={value}"]
return args
@property
def common_client_mount(self):
"""CommonClient.py, mounted back in from the checkout.
The image excludes every client file, correctly - but a few
apworlds import their own during generation and fail with "No
module named 'CommonClient'" purely because it is absent.
Absent source means no mount: it exists only after a clone.
"""
if not os.path.isfile(self.paths.common_client):
return []
return ["-v", f"{self.paths.common_client}:/app/CommonClient.py:ro"]
@property
def rom_mounts(self):
"""Every base ROM, mounted individually.
Per file rather than mounting the directory over /app/data,
which would shadow the image's own shipped data. Worlds declare
rom_file inconsistently - some embed "data/", some do not - so
each is mounted at both locations.
"""
directory = self.paths.roms
if not os.path.isdir(directory):
return []
data_in = self.DATA_DIRECTORY
args = []
for name in sorted(os.listdir(directory)):
path = os.path.join(directory, name)
if os.path.isfile(path):
args += ["-v", f"{path}:{data_in}/{name}:ro"]
args += ["-v", f"{path}:/app/{name}:ro"]
return args
def apworld_mount(self, apworld_path):
"""The mount and container path for the world under test."""
if not apworld_path:
return [], None
name = ApworldFile(apworld_path).importable_name
inside = f"{self.APWORLD_DIRECTORY}/{name}"
return ["-v", f"{apworld_path}:{inside}:ro"], inside
def driver_mounts(self, output_dir):
"""The driver, the output directory, the ROMs and the client."""
return [
"-v", f"{self.paths.driver}:{self.DRIVER_PATH}:ro",
"-v", f"{output_dir}:{self.OUTPUT_DIRECTORY}",
*self.rom_mounts,
*self.common_client_mount,
]

83
src/apworld_tester/pipeline/generation/companion_check.py Normal file
View File

@@ -0,0 +1,83 @@
"""Which core worlds can actually generate in this container."""
import os
import subprocess
from apworld_tester.core.config.run_settings import RunSettings
from apworld_tester.pipeline.generation.command import ContainerCommand
from apworld_tester.pipeline.generation.report import DriverReport
class CompanionCheck:
"""Tries every core world once, so the rest can trust the pool.
Several core worlds need a base ROM this container has no copy of,
and drawing one fails the seed - with the failure landing on the
apworld under test rather than on the companion that caused it.
"""
#: The whole verification's ceiling, and each world's share of it.
#: Not configuration: the per-world limit is deliberately shorter
#: than a generation timeout, because a companion slow to generate
#: would multiply that cost across every seed that draws it.
TIMEOUT = 1800
PER_WORLD_TIMEOUT = 120
def __init__(self, config, paths, image):
self.config = config
self.paths = paths
self.image = image
self.command = ContainerCommand(config, paths, image)
def build(self, output_dir, per_world_timeout, spoiler):
verify_in = ContainerCommand.VERIFY_DRIVER_PATH
return [
*self.command.prefix,
"-v", f"{self.paths.verify_driver}:{verify_in}:ro",
"-v", (f"{self.paths.driver}:"
f"{ContainerCommand.DRIVER_PATH}:ro"),
"-v", (f"{output_dir}:"
f"{ContainerCommand.OUTPUT_DIRECTORY}"),
*self.command.rom_mounts,
*self.command.common_client_mount,
self.image,
verify_in,
"--output-dir", ContainerCommand.OUTPUT_DIRECTORY,
"--timeout", str(per_world_timeout),
"--spoiler", str(spoiler),
]
@staticmethod
def not_verified(detail):
return {"ran": False, "verified": [], "rejected": {},
"detail": detail}
def parse(self, result):
data = DriverReport(result).data
if data is not None:
data.setdefault("ran", True)
return data
return self.not_verified(
f"no JSON output (exit {result.returncode}): "
f"{DriverReport.tail(result.stderr)}")
def run(self, output_dir, timeout=None, per_world_timeout=None,
spoiler=None):
"""The pool of usable companions, or why there is none."""
timeout = timeout or self.TIMEOUT
per_world_timeout = per_world_timeout or self.PER_WORLD_TIMEOUT
if spoiler is None:
spoiler = RunSettings(self.config).spoiler
output_dir = os.path.abspath(output_dir)
os.makedirs(output_dir, exist_ok=True)
try:
result = subprocess.run(
self.build(output_dir, per_world_timeout, spoiler),
capture_output=True,
text=True,
timeout=timeout,
)
except subprocess.TimeoutExpired:
return self.not_verified("companion verification did not "
f"finish within {timeout}s")
return self.parse(result)

75
src/apworld_tester/pipeline/generation/identifier.py Normal file
View File

@@ -0,0 +1,75 @@
"""Asking an apworld which game it registers as."""
import os
import subprocess
import tempfile
from apworld_tester.core.model.apworld import ApworldFile
from apworld_tester.pipeline.generation.command import ContainerCommand
from apworld_tester.pipeline.generation.report import DriverReport
class ApworldIdentifier:
"""What a file's own World class says it is.
Lighter than a test: no ROMs, no output volume, no generation. Used
right after download to check the content matches the row that
fetched it, before the file is trusted at all.
"""
#: Seconds an import check may take. Not configuration: this only
#: loads the apworld and reads which games it registers, so a file
#: still going after a minute is stuck, not slow.
TIMEOUT = 60
def __init__(self, config, paths, image):
self.paths = paths
self.image = image
self.command = ContainerCommand(config, paths, image)
def build(self, apworld_path):
mount, inside = self.command.apworld_mount(apworld_path)
identify_in = ContainerCommand.IDENTIFY_DRIVER_PATH
return [
*self.command.prefix,
*mount,
"-v", f"{self.paths.identify_driver}:{identify_in}:ro",
*self.command.common_client_mount,
self.image,
identify_in,
"--apworld", inside,
]
def failed(self, detail):
return {"ran": False, "games": [], "detail": detail}
def identify(self, apworld_path, timeout=None):
"""Which games this file registers, and whether the check ran."""
timeout = timeout or self.TIMEOUT
apworld_path = os.path.abspath(apworld_path)
if not os.path.isfile(apworld_path):
return self.failed(f"{apworld_path} does not exist (not a file)")
try:
result = self.run(apworld_path, timeout)
except subprocess.TimeoutExpired:
return self.failed(
f"identify check did not finish within {timeout}s")
data = DriverReport(result).data
if data is not None:
data.setdefault("ran", True)
return data
return self.failed(
f"no JSON output (exit {result.returncode}): "
f"{DriverReport.tail(result.stderr)}")
def run(self, apworld_path, timeout):
"""Run the driver against a prepared copy of the file.
Repaired here as well as at test time, so a backslash-separated
world is not rejected at download for registering no game and
never reaching the test that could repair it.
"""
with tempfile.TemporaryDirectory(prefix="apworld-id-") as workdir:
checked, _ = ApworldFile(apworld_path).prepare(workdir)
return subprocess.run(self.build(checked), capture_output=True,
text=True, timeout=timeout)

54
src/apworld_tester/pipeline/generation/report.py Normal file
View File

@@ -0,0 +1,54 @@
"""Reading what a driver reported."""
import json
class DriverReport:
"""The JSON one driver run wrote to stdout."""
def __init__(self, result):
self.result = result
@staticmethod
def tail(text, lines=20):
return "\n".join(text.splitlines()[-lines:])
@staticmethod
def last_json_object(text):
"""The driver's report, however it ended up on the line.
Not "the last line starting with {": worlds write to raw stdout
during generation, and one that writes without a trailing
newline leaves its output glued to the front of the report -
Super Metroid's randomizer emits a bare "*" per retry, so the
report arrives as "*{...}" and would read as no report at all.
"""
for line in reversed(text.splitlines()):
line = line.strip()
start = line.find("{")
while start != -1:
try:
return json.loads(line[start:])
except json.JSONDecodeError:
start = line.find("{", start + 1)
return None
@property
def data(self):
return self.last_json_object(self.result.stdout)
def parsed(self):
"""The report, or a failure built from what it printed."""
report = self.data
if report is not None:
report.setdefault("stderr_tail", self.tail(self.result.stderr))
return report
return {
"game": None,
"outcome": "failed",
"detail": ("no JSON report on stdout "
f"(exit code {self.result.returncode})"),
"elapsed_seconds": None,
"stdout_tail": self.tail(self.result.stdout),
"stderr_tail": self.tail(self.result.stderr),
}

136
src/apworld_tester/pipeline/generation/tester.py Normal file
View File

@@ -0,0 +1,136 @@
"""Testing one world by generating seeds for it."""
import os
import subprocess
import tempfile
from apworld_tester.core.model.apworld import ApworldFile
from apworld_tester.pipeline.generation.command import ContainerCommand
from apworld_tester.pipeline.generation.report import DriverReport
class WorldTester:
"""Runs every generation mode against one world."""
#: Startup slack added to the container's own ceiling, on top of
#: every mode's generation timeout. Not configuration: it covers
#: docker start and interpreter boot, which no run changes.
HOST_TIMEOUT_BUFFER = 30
def __init__(self, config, paths, image):
self.config = config
self.paths = paths
self.image = image
self.command = ContainerCommand(config, paths, image)
def settings(self, key, default):
return self.config.value("testing", key, default)
def build(self, apworld_path, output_dir, timeout, companion_range,
spoiler, repeats, companion_pool, game, random_repeats):
"""The full `docker run` for one world's whole test."""
mount, inside = self.command.apworld_mount(apworld_path)
selector = ["--apworld", inside] if inside else ["--game", game]
return [
*self.command.prefix,
*mount,
*self.command.driver_mounts(output_dir),
self.image,
ContainerCommand.DRIVER_PATH,
*selector,
"--output-dir", ContainerCommand.OUTPUT_DIRECTORY,
"--timeout", str(timeout),
"--companion-min", str(companion_range[0]),
"--companion-max", str(companion_range[1]),
"--spoiler", str(spoiler),
"--repeats", str(repeats),
"--random-repeats", str(random_repeats),
"--companion-pool", ",".join(companion_pool or ()),
]
def host_timeout(self, timeout, repeats, random_repeats):
"""How long to let the whole container run.
Every mode runs in the one container, each bounded by its own
timeout, so the host allows for all of them plus startup slack.
A ceiling rather than an expected cost.
"""
generations = 2 * repeats + 1 + 2 * random_repeats
return timeout * generations + self.HOST_TIMEOUT_BUFFER
def missing_file(self, path):
"""The failure for a path that is not a file.
Docker silently creates an empty directory at a missing mount
source rather than failing, which turns a not-yet-written
apworld into a bogus directory that breaks the content hash
days later with a confusing "Is a directory".
"""
return {
"game": None,
"outcome": "failed",
"detail": f"{path} does not exist (not a file)",
"elapsed_seconds": None,
}
def test(self, apworld_path=None, game=None, output_dir=None,
timeout=None, companion_range=(2, 5), spoiler=3, repeats=10,
companion_pool=(), random_repeats=3):
"""Run every mode against one world, core or downloaded."""
output_dir = os.path.abspath(output_dir or self.paths.output)
os.makedirs(output_dir, exist_ok=True)
timeout = timeout or self.settings("generation_timeout", 300)
if apworld_path is not None:
apworld_path = os.path.abspath(apworld_path)
if not os.path.isfile(apworld_path):
return self.missing_file(apworld_path)
with tempfile.TemporaryDirectory(prefix="apworld-") as workdir:
tested, repaired = self.prepared(apworld_path, workdir)
report = self.run_once(
apworld_path=tested,
game=game,
output_dir=output_dir,
timeout=timeout,
companion_range=companion_range,
spoiler=spoiler,
repeats=repeats,
companion_pool=companion_pool,
random_repeats=random_repeats,
)
if repaired:
report["repaired"] = True
return report
def prepared(self, apworld_path, workdir):
"""The path to test, and whether it had to be repaired."""
if not apworld_path:
return None, False
return ApworldFile(apworld_path).prepare(workdir)
def run_once(self, apworld_path, game, output_dir, timeout,
companion_range, spoiler, repeats, companion_pool,
random_repeats):
"""One container run, and the report it produced."""
limit = self.host_timeout(timeout, repeats, random_repeats)
command = self.build(
apworld_path=apworld_path,
output_dir=output_dir,
timeout=timeout,
companion_range=companion_range,
spoiler=spoiler,
repeats=repeats,
companion_pool=companion_pool,
game=game,
random_repeats=random_repeats,
)
try:
result = subprocess.run(command, capture_output=True, text=True,
timeout=limit)
except subprocess.TimeoutExpired:
return {
"game": None,
"outcome": "failed",
"detail": f"docker run did not exit within {limit}s",
"elapsed_seconds": limit,
}
return DriverReport(result).parsed()

0
src/apworld_tester/pipeline/sheet/__init__.py Normal file
View File

62
src/apworld_tester/pipeline/sheet/fetcher.py Normal file
View File

@@ -0,0 +1,62 @@
"""Downloading the worlds sheet."""
import io
import os
import zipfile
import requests
class SheetFetcher:
"""The Playable Worlds sheet, as HTML."""
def __init__(self, config, paths):
self.config = config
self.paths = paths
@property
def spreadsheet_id(self):
return self.config.value("sheet", "spreadsheet_id")
@property
def tab_name(self):
return self.config.value("sheet", "tab_name")
@property
def export_url(self):
return ("https://docs.google.com/spreadsheets/d/"
f"{self.spreadsheet_id}/export?format=zip")
def fetch(self):
"""The tab's HTML, as bytes.
Google answers a private sheet with its sign-in page and status
200 rather than an error, so the content type is checked instead
of the status.
"""
response = requests.get(self.export_url, timeout=120)
response.raise_for_status()
if "application/zip" not in response.headers.get("Content-Type", ""):
raise RuntimeError(
"The server did not return a zip archive. The sheet is "
"most likely not shared with 'anyone with the link'.")
with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
wanted = f"{self.tab_name}.html"
if wanted not in archive.namelist():
raise RuntimeError(
f"The export has no tab called '{self.tab_name}'. "
f"Available: {', '.join(archive.namelist())}")
return archive.read(wanted)
def save(self, content=None):
"""Write the tab's HTML, returning where it landed."""
content = self.fetch() if content is None else content
with open(self.paths.sheet_html, "wb") as handle:
handle.write(content)
return self.paths.sheet_html
def ensure(self):
"""Download the sheet only when this machine has no copy."""
if not os.path.exists(self.paths.sheet_html):
self.save()
return self.paths.sheet_html

98
src/apworld_tester/pipeline/sheet/reader.py Normal file
View File

@@ -0,0 +1,98 @@
"""Reading the worlds sheet into Game objects."""
from bs4 import BeautifulSoup
from apworld_tester.core.model.game import Game
class SheetReader:
"""The sheet's HTML export, as rows."""
def __init__(self, config, path):
self.config = config
self.path = path
def column(self, name):
return (self.config.value("sheet", "columns") or {})[name]
def rows(self):
"""Every table row, as lists of cells.
Google repeats each row's frozen-column cells in a spacer, which
would shift every column index by one, so those are dropped.
"""
with open(self.path, "r", encoding="utf-8") as handle:
soup = BeautifulSoup(handle.read(), "html.parser")
found = []
for row in soup.find_all("tr"):
cells = [
cell for cell in row.find_all("td")
if "freezebar-cell" not in cell.get("class", [])
]
if cells:
found.append(cells)
return found
def header(self, rows):
"""Where the header row is, and the headings it holds."""
for index, cells in enumerate(rows):
texts = [cell.get_text(strip=True) for cell in cells]
if self.column("name") in texts:
return index, texts
raise ValueError(
f"No '{self.column('name')}' header found in the export.")
def indexes(self, headers):
"""Where each field sits in a row."""
if self.column("link") not in headers:
raise ValueError(f"No '{self.column('link')}' column found.")
optional = {}
for field in ("stability", "pr_status"):
heading = self.column(field)
optional[field] = (headers.index(heading)
if heading in headers else None)
return (headers.index(self.column("name")),
headers.index(self.column("link")),
optional["stability"],
optional["pr_status"])
@staticmethod
def text_of(cells, index):
"""One cell's text, or None when missing or empty."""
if index is None or index >= len(cells):
return None
return cells[index].get_text(strip=True) or None
@staticmethod
def links_in(cell):
"""Every hyperlink target in one cell, in the order written."""
return [anchor.get("href") for anchor in cell.find_all("a")
if anchor.get("href")]
def build(self, cells, indexes):
"""One row as a Game, or None when it has no name."""
name_index, link_index, stability_index, status_index = indexes
name = self.text_of(cells, name_index)
if name is None:
return None
game = Game(
name=name,
stability=self.text_of(cells, stability_index),
pr_status=self.text_of(cells, status_index),
)
if link_index < len(cells):
for link in self.links_in(cells[link_index]):
game.add_link(link)
return game
def games(self):
"""Every world listed in the export."""
rows = self.rows()
header_index, headers = self.header(rows)
indexes = self.indexes(headers)
found = []
for cells in rows[header_index + 1:]:
game = self.build(cells, indexes)
if game is not None:
found.append(game)
return found

0
src/apworld_tester/pipeline/update/__init__.py Normal file
View File

37
src/apworld_tester/pipeline/update/download_reporter.py Normal file
View File

@@ -0,0 +1,37 @@
"""The download stage's progress line."""
import os
from apworld_tester.core.display.progress import Progress
class DownloadReporter:
"""One line per game, saying what its apworld did.
The download run calls back serialized, so this needs no lock of its
own. What each game's last asset did is remembered so the line can
still name it once the game itself finishes.
"""
#: What each download status means, in words.
STATUS_LABELS = {
"missing": "downloaded (new)",
"stale": "downloaded (updated)",
"verified": "already present (verified up to date)",
"assumed": "already present (not verified)",
}
def __init__(self, config, total):
self.bar = Progress(total, config=config,
labels={"ok": "ok", "none": "none"})
self.last_asset = {}
def on_asset(self, game_name, path, status):
label = self.STATUS_LABELS.get(status, status)
self.last_asset[game_name] = f"{label} {os.path.basename(path)}"
def on_game(self, index, total, game_name, downloaded):
detail = self.last_asset.pop(game_name, "no apworld")
self.bar.update(
index, "ok" if downloaded else "none", game_name,
fallback_line=f"[{index}/{total}] {game_name}: {detail}")

15
src/apworld_tester/pipeline/update/stage_error.py Normal file
View File

@@ -0,0 +1,15 @@
"""A failure, named by the stage it happened in."""
class StageError(RuntimeError):
"""Which stage failed, so the run can say so and stop.
Raised at the stage boundary rather than inside it: a stage can fail
anywhere after it was opened, and an unnamed traceback makes a long
run's failure much harder to place.
"""
def __init__(self, stage, original):
super().__init__(f"failed during '{stage}': {original}")
self.stage = stage
self.original = original

277
src/apworld_tester/pipeline/update/update_run.py Normal file
View File

@@ -0,0 +1,277 @@
"""Everything the testing half does in one pass.
Update Archipelago to the newest release, refresh the worlds sheet,
download whatever changed, and test each world by generating seeds.
What happens to the results afterwards - promoting, publishing,
deploying - is the other half's job and is not known about here.
"""
import contextlib
import requests
from apworld_tester.core.config.config import Config
from apworld_tester.core.config.paths import Paths
from apworld_tester.core.config.run_settings import RunSettings
from apworld_tester.core.config.secrets import Secrets
from apworld_tester.core.model.name import Name
from apworld_tester.core.state.keys import StateKeys
from apworld_tester.core.state.store import StateStore
from apworld_tester.pipeline.batch.batch import Batch
from apworld_tester.pipeline.batch.discontinued import Discontinued
from apworld_tester.pipeline.batch.orphaned_records import (
OrphanedRecords,
)
from apworld_tester.pipeline.batch.download_failures import (
DownloadFailures,
)
from apworld_tester.pipeline.batch.reporter import Reporter
from apworld_tester.pipeline.batch.results import Results
from apworld_tester.pipeline.build.archipelago import ArchipelagoBuild
from apworld_tester.pipeline.download.download_run import DownloadRun
from apworld_tester.pipeline.download.update_history import (
UpdateHistory,
)
from apworld_tester.pipeline.generation.identifier import ApworldIdentifier
from apworld_tester.pipeline.sheet.fetcher import SheetFetcher
from apworld_tester.pipeline.sheet.reader import SheetReader
from apworld_tester.pipeline.update.download_reporter import (
DownloadReporter,
)
from apworld_tester.pipeline.update.stage_error import StageError
class UpdateRun:
"""The four stages, in the order they run.
Takes no lock of its own: a caller that also publishes the results
holds one for the whole thing, so nothing swaps underneath a
generation still in progress.
"""
def __init__(self, config=None, paths=None, stability=None, fetch=None,
jobs=None, force_retest=False):
config = config or Config.load()
paths = paths or Paths(config)
self.config = config
self.paths = paths
self.settings = RunSettings(config)
self.store = StateStore(paths.state)
self.build = ArchipelagoBuild(config, paths)
self.stability = stability
self.fetch = fetch
self.jobs = jobs or self.settings.jobs
self.force_retest = force_retest
self.report = {}
@contextlib.contextmanager
def stage(self, name):
print(f"\n=== {name} ===")
try:
yield
except Exception as error:
raise StageError(name, error) from error
def snapshot_previous_version(self, previous_tag):
"""Keep last version's results before this one overwrites them.
The only source for "did this world's outcome change at the
exact moment Archipelago upgraded": it is the state at the
instant the version changed, which no later re-test can
reconstruct.
"""
state = self.store.load()
state[StateKeys.UPGRADE_SNAPSHOT] = dict(
state.get(StateKeys.APWORLD_TESTS, {}))
state[StateKeys.PREVIOUS_ARCHIPELAGO_TAG] = previous_tag
self.store.save(state)
def update_archipelago(self):
"""Put this machine on the newest release and build its image."""
with self.stage("archipelago_build"):
tag, previous, changed, built = self.build.update_and_build(
self.store)
self.build.ensure_test_image(tag)
self.report.update({
"archipelago_tag": tag,
"archipelago_previous_tag": previous,
"archipelago_tag_changed": changed,
"archipelago_image_built": built,
})
print(f"tag {tag}" + (f" (was {previous})" if previous
else " (first run)"))
if changed and previous:
self.snapshot_previous_version(previous)
return tag, changed
def announce_retired(self, retired):
gone = sorted(key for key, value in retired.items() if value)
back = sorted(key for key, value in retired.items() if not value)
if gone:
print(f"{len(gone)} no longer on the sheet, marked "
f"discontinued: {', '.join(gone)}")
if back:
print(f"{len(back)} back on the sheet: {', '.join(back)}")
def select(self, games):
"""The rows this pass considers.
Applied to the whole sheet, core rows included, so --fetch
really does bound the work: core worlds are tested like every
other row, and leaving them out of the limit made a "cheap,
bounded pass" still test all eighty of them.
"""
if self.stability:
games = [game for game in games
if (game.stability or "").lower()
== self.stability.lower()]
return games[:self.fetch] if self.fetch is not None else games
def read_sheet(self):
"""The sheet's rows, split into core and downloadable.
Discontinued rows are flagged straight after reading, so every
stage below already sees a world taken off the sheet as gone.
Core rows are tested by name but never downloaded: Archipelago
ships them.
"""
with self.stage("sheet_download"):
path = SheetFetcher(self.config, self.paths).save()
games = SheetReader(self.config, path).games()
# Every row, before any filtering: download-time identity
# checks compare against the whole sheet, core rows
# included, and so does the discontinued flag.
sheet_names = [game.name for game in games]
self.report["games_in_sheet"] = len(games)
print(f"{len(games)} games in sheet")
retired = Discontinued(self.store).mark(sheet_names)
self.report["discontinued_changed"] = retired
self.announce_retired(retired)
selected = self.select(games)
core = [game for game in selected if game.is_core]
rest = [game for game in selected if not game.is_core]
self.report["games_core"] = len(core)
self.report["games_selected"] = len(rest)
print(f"{len(core)} core games")
print(f"{len(rest)} games selected")
return rest, core, sheet_names
def session(self):
"""A session carrying the token, if there is one."""
session = requests.Session()
token = Secrets(self.paths).github_token
if token:
session.headers["Authorization"] = f"token {token}"
return session
def downloader(self, image, sheet_names):
return DownloadRun(
config=self.config,
paths=self.paths,
session=self.session(),
identifier=ApworldIdentifier(self.config, self.paths, image),
jobs=self.jobs,
sheet_names=sheet_names,
# Last run's resolved releases, so a game whose attempt
# failed against an unchanged release is not downloaded and
# re-identified all over again.
previous_releases=self.store.load().get(
StateKeys.GAME_RELEASES, {}),
)
def download(self, games, sheet_names, image, tag):
"""Fetch every selected game's newest apworld."""
reporter = DownloadReporter(self.config, len(games))
run = self.downloader(image, sheet_names)
with self.stage("apworld_download"):
total, skipped, releases = run.all(
games, on_asset=reporter.on_asset, on_game=reporter.on_game)
self.report["assets_downloaded"] = total
self.report["games_skipped_download"] = len(skipped)
print(f"{total} assets downloaded, {len(skipped)} games skipped")
run.record_releases(releases)
self.record_versions()
DownloadFailures(self.config, self.store).record(games, skipped, tag)
orphaned = OrphanedRecords(self.store, self.paths.downloads).prune()
self.report["orphaned_records_dropped"] = len(orphaned)
if orphaned:
print(f"{len(orphaned)} result(s) dropped for apworlds no "
f"longer on disk: {', '.join(sorted(orphaned))}")
def record_versions(self):
"""One history entry per game whose apworld content changed.
Recorded here, not by whatever publishes afterwards: this is
the moment a new version is known, and a pass that never
publishes would lose it outright - the next download overwrites
the file and no later stage can reconstruct which version
existed on which date.
"""
history = UpdateHistory(self.paths)
with self.stage("apworld_history"):
recorded = history.record_all(self.store.load(),
self.paths.downloads)
self.report["new_apworld_versions"] = len(recorded)
print(f"{len(recorded)} new apworld version(s) recorded")
def selected_only(self, games, core_games):
"""This pass's rows, named the way the batch names them.
None unless something actually narrowed the sheet. The download
directory holds every apworld ever fetched, so an unnarrowed run
has to consider all of them - and Batch reads an explicit
selection as a reason to retest rather than serve from cache,
which would turn a plain pass into a full re-test of everything.
Downloadable rows are matched on their download folder, core
ones on the bare sheet name, because that is how each is keyed.
"""
if self.fetch is None and not self.stability:
return None
return ([Name(game.name).directory for game in games]
+ [game.name for game in core_games])
def test(self, image, tag, games, core_games, changed):
"""Generation-test the downloaded apworlds, and the core ones.
Core worlds ship with Archipelago, so nothing was downloaded for
them and they are named rather than mounted, but they run the
same five modes and their result is measured rather than
assumed.
A narrowed pass tests what it narrowed to and nothing else:
--fetch bounded the download stage from the start, and leaving
the test stage unbounded meant a "cheap, bounded pass" still
generated against every apworld on disk.
"""
only = self.selected_only(games, core_games)
if only is not None:
print(f"{len(only)} selected for testing")
with self.stage("generation_tests"):
results = Batch(
config=self.config,
paths=self.paths,
settings=self.settings,
image=image,
tag=tag,
force=changed or self.force_retest,
on_progress=Reporter(self.config, bar=True),
jobs=self.jobs,
core_games=[game.name for game in core_games],
only=only,
exact_only=only is not None,
).run()
tested = Results(results, self.config)
self.report["test_counts"] = tested.summary
self.report["test_results"] = results
print(tested.overview)
return results
def run(self):
"""Every stage, in order, returning what each one did."""
tag, changed = self.update_archipelago()
image = self.build.test_image_tag(tag)
games, core_games, sheet_names = self.read_sheet()
self.download(games, sheet_names, image, tag)
self.test(image, tag, games, core_games, changed)
return self.report

1
src/apworld_tester/templates/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""The starter config shipped with the package."""

69
src/apworld_tester/templates/config.yaml Normal file
View File

@@ -0,0 +1,69 @@
# Config file
general:
# Output directory of everything a run generates,
# including the Archipelago source and downloaded APworlds
output_directory: /path/to/your/archipelago-world-tester-data
roms_directory: /path/to/your/base-roms
threads: 4
sheet:
# The Community Spreadsheet
spreadsheet_id: 1iuzDTOAvdoNe8Ne8i461qGNucg5OuEoF-Ikqs8aUQZw
tab_name: Playable Worlds
columns:
name: Game
link: Links & Downloads
stability: Stability
pr_status: PR Status
archipelago:
# Settings regarding the Archipelago repository to use for
# testing worlds.
version: "latest"
repo_url: https://github.com/ArchipelagoMW/Archipelago.git
repository: ArchipelagoMW/Archipelago
# The Docker image basename built from that checkout
image_name: archipelago
# repeats is what makes a verdict a pass rate rather than a single run;
# lowering it makes a run cheaper and its answer weaker, and the cache
# knows the difference.
testing:
generation_timeout: 300
companion_min: 2
companion_max: 5
repeats: 10
random_repeats: 3
# none: no spoiler log
# log: spoiler log, no playthrough
# playthrough: also computes the playthrough
# paths: also computes entrance paths
spoiler: playthrough
# How each verdict is written in the sheet's Stability column.
stability:
labels:
stable: Stable
unstable: Minor issues
flaky: Flaky
solo_only: Solo only
broken: Broken
unknown: Unknown
# Tag prefixes for repositories that publish several games' releases
# side by side, so the right release is picked per game.
release_overrides:
Donkey Kong Country: dkc-
"Donkey Kong Country 2: Diddy's Kong Quest": dkc2-
Mega Man X: mmx-
Mega Man X2: mmx2-
# Worlds not tested.
blacklist:
- Ocarina of Time but it's just Master Quest Water Temple

5
src/archipelago_tester/__init__.py Normal file
View File

@@ -0,0 +1,5 @@
"""Tests Archipelago community worlds by generating seeds with them.
The publishing half - archipelago-world-site - imports this package as a
dependency. Nothing here knows that it exists.
"""

Some files were not shown because too many files have changed in this diff Show More