Skilly/APWorldTester
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Delete the archipelago_tester package

Browse Source 
Left behind by the rename to apworld_tester: the new package was added
but the old tree was never removed, so both were committed and both
were installed. Nothing imported it and 58 of its 85 files had since
diverged, but it stayed importable and shadowed the live code.

Constrain packages.find to apworld_tester* so a stray directory under
src/ cannot be packaged again.
This commit is contained in:
Skilly
2026-09-06 16:19:51 +02:00
parent 61647d7511
commit eba8c0eb8a
86 changed files with 1 additions and 5558 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pyproject.toml
View File

@@ -21,6 +21,7 @@ apworld-tester = "apworld_tester.cli:main"
[tool.setuptools.packages.find]
where = ["src"]
include = ["apworld_tester*"]
[tool.setuptools.package-data]
apworld_tester = [

5
src/archipelago_tester/__init__.py
View File

@@ -1,5 +0,0 @@
"""Tests Archipelago community worlds by generating seeds with them.
The publishing half - archipelago-world-site - imports this package as a
dependency. Nothing here knows that it exists.
"""

0
src/archipelago_tester/core/__init__.py
View File

0
src/archipelago_tester/core/config/__init__.py
View File

39
src/archipelago_tester/core/config/config.py
View File

@@ -1,39 +0,0 @@
"""The project's configuration."""
import json
import os
class Config:
"""config.json, read once and passed to whatever needs it."""
def __init__(self, values, path=None):
self.values = values
self.path = path
@classmethod
def load(cls, path=None):
"""Read config.json from the checkout."""
path = path or cls.default_path()
with open(path, "r", encoding="utf-8") as handle:
return cls(json.load(handle), path)
@staticmethod
def default_path():
"""config.json at the repo root, above the installed package."""
here = os.path.dirname(os.path.abspath(__file__))
root = os.path.dirname(os.path.dirname(os.path.dirname(here)))
return os.path.join(os.path.dirname(root), "config.json")
def get(self, key, default=None):
"""One top-level value."""
value = self.values.get(key)
return default if value is None else value
def section(self, name):
return self.values.get(name) or {}
def value(self, section, key, default=None):
"""One value from a section."""
value = self.section(section).get(key)
return default if value is None else value

134
src/archipelago_tester/core/config/paths.py
View File

@@ -1,134 +0,0 @@
"""Every path the pipeline uses."""
import os
class Paths:
"""Where everything lives, derived from one Config.
The checkout holds code and configuration; the data root holds
everything a run generates, and defaults outside the checkout
because none of that is source. Every path is absolute.
"""
def __init__(self, config):
self.config = config
@property
def project(self):
return os.path.dirname(self.config.path)
@property
def data(self):
"""Where everything a run generates lives."""
configured = self.config.get(
"data_root",
"~/.local/share/archipelago-world-tester",
)
return os.path.abspath(os.path.expanduser(configured))
@property
def state(self):
return os.path.join(self.data, "state.json")
@property
def scratch_state(self):
return os.path.join(self.data, "scratch_state.json")
@property
def lock(self):
return os.path.join(self.data, "pipeline.lock")
@property
def downloads(self):
return os.path.join(self.data, "downloads")
@property
def output(self):
return os.path.join(self.data, "test_output")
@property
def history(self):
"""Every apworld version this pipeline has seen, per game.
Beside the downloads rather than inside them: that directory is
a cache and gets cleared to force a clean re-download, and this
is the one thing that cannot be rebuilt afterwards.
"""
return os.path.join(self.data, "history")
@property
def checkout(self):
return os.path.join(self.data, "archipelago_src")
@property
def core_worlds(self):
return os.path.join(self.checkout, "worlds")
@property
def common_client(self):
return os.path.join(self.checkout, "CommonClient.py")
@property
def failed_report(self):
return os.path.join(self.data, "failed_apworlds.txt")
@property
def sheet_html(self):
return os.path.join(self.data, "playable_worlds.html")
@property
def roms(self):
"""Base ROMs for the worlds that need one."""
configured = self.config.get("roms_directory")
if not configured:
return os.path.join(self.data, "roms")
return os.path.abspath(os.path.expanduser(configured))
@property
def archipelago_location(self):
"""The Archipelago install the approved apworlds are placed in.
Someone's own install rather than anything a run generates, so
it has no default: without archipelago_location there is no
install to speak of, which is a different thing from one at a
path this pipeline picked.
"""
configured = self.config.value("archipelago", "archipelago_location")
if not configured:
return None
return os.path.abspath(os.path.expanduser(configured))
@property
def custom_worlds(self):
"""Where that install loads apworlds from."""
location = self.archipelago_location
return location and os.path.join(location, "custom_worlds")
@property
def env(self):
return os.path.join(self.project, ".env")
@property
def container(self):
return os.path.join(self.project, "container")
@property
def drivers(self):
return os.path.join(self.container, "drivers")
@property
def test_dockerfile(self):
return os.path.join(self.container, "Dockerfile.test")
@property
def driver(self):
return os.path.join(self.drivers, "run_test.py")
@property
def identify_driver(self):
return os.path.join(self.drivers, "identify_apworld.py")
@property
def verify_driver(self):
return os.path.join(self.drivers, "verify_companions.py")

66
src/archipelago_tester/core/config/run_settings.py
View File

@@ -1,66 +0,0 @@
"""How much testing one run asks for.
Companions are drawn at random from the worlds Archipelago ships. Core
worlds are the safe choice: part of the build itself, so always present
and always matching the running version, unlike an apworld that might
be missing or broken for reasons of its own. A fresh draw per attempt
matters - ten runs against one fixed pair only ever test that pair.
"""
class RunSettings:
"""The testing section, as the numbers a run is built from."""
def __init__(self, config):
self.config = config
def number(self, key, default):
return int(self.config.value("testing", key, default))
@property
def companion_range(self):
"""How many core worlds join the world under test.
Redrawn every attempt, so ten repeats sample ten combinations.
Clamped so a mistyped config cannot ask for an empty or
backwards range.
"""
low = max(self.number("companion_min", 2), 1)
return low, max(self.number("companion_max", 5), low)
@property
def repeats(self):
"""How often the single and multi tests are repeated.
Stability is a pass rate across runs, not one verdict: each
repeat gets a different seed, so the fill differs every time.
"""
return self.number("repeats", 10)
@property
def random_repeats(self):
"""How often the two randomized modes are repeated.
Lower on purpose: single and multi re-roll only the seed, so
ten runs measure one thing precisely, while each randomized run
rolls a different point in the option space and three samples
say much more than one did.
"""
return self.number("random_repeats", 3)
@property
def spoiler(self):
"""2 computes the full playthrough, exercising accessibility
and reachability logic that plain generation skips - so a world
whose logic is wrong fails here rather than in a player's seed.
"""
return self.number("spoiler", 2)
@property
def jobs(self):
"""How many apworlds are tested at once.
Each test is one isolated `docker run` and subprocess.run
releases the GIL while it waits, so threads are all this needs.
"""
return self.number("jobs", 4)

41
src/archipelago_tester/core/config/secrets.py
View File

@@ -1,41 +0,0 @@
"""Credentials, kept out of config.json."""
import os
class Secrets:
"""The .env file at the checkout's root.
Kept out of config.json: that file gets opened, edited and shared,
and a token in it leaks the first time it is.
"""
#: The environment variable a GitHub token is read from.
GITHUB_TOKEN = "GITHUB_TOKEN"
def __init__(self, paths):
self.paths = paths
def values(self):
"""NAME -> value for every line in the file."""
if not os.path.exists(self.paths.env):
return {}
values = {}
with open(self.paths.env, "r", encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
name, value = line.split("=", 1)
values[name.strip()] = value.strip().strip("\"'")
return values
def get(self, name, default=None):
"""One secret, or `default` when it is unset or empty."""
value = self.values().get(name)
return default if value in (None, "") else value
@property
def github_token(self):
"""The GitHub token, or None."""
return self.get(self.GITHUB_TOKEN)

0
src/archipelago_tester/core/display/__init__.py
View File

103
src/archipelago_tester/core/display/progress.py
View File

@@ -1,103 +0,0 @@
"""Progress reporting for the long stages."""
import shutil
import sys
import time
class Progress:
"""One stage's progress, tallied by outcome as it goes.
Both the download stage and the generation tests walk ~650 games, so
per-line output scrolls past uselessly when a run is watched. Under
cron the opposite is true - stdout is a log file, where a bar
redrawn with carriage returns is noise - so each caller supplies the
line it would have printed, and that is what a non-terminal gets.
"""
def __init__(self, total, config=None, labels=None, stream=None):
self.total = total
self.config = config
self.labels = dict(labels or self.configured_labels())
self.stream = stream or sys.stdout
self.counts = {}
self.started = None
def configured_labels(self):
"""Short labels for each outcome."""
if self.config is None:
return {}
return self.config.value("display", "outcome_labels", {})
@property
def width(self):
"""How many characters wide the bar is drawn."""
if self.config is None:
return 28
return int(self.config.value("display", "progress_bar_width", 28))
@staticmethod
def duration(seconds):
"""Seconds as m:ss, or h:mm:ss past an hour."""
seconds = int(seconds)
hours, rest = divmod(seconds, 3600)
minutes, seconds = divmod(rest, 60)
if hours:
return f"{hours}:{minutes:02d}:{seconds:02d}"
return f"{minutes:02d}:{seconds:02d}"
@property
def tally(self):
"""The running counts, labelled keys first."""
ordered = [key for key in self.labels if self.counts.get(key)]
ordered += [key for key in self.counts if key not in self.labels]
return " ".join(
f"{self.labels.get(key, key)} {self.counts[key]}"
for key in ordered
)
def bar(self, fraction):
"""The bar itself, filled to `fraction`."""
filled = int(self.width * fraction)
return "#" * filled + "-" * (self.width - filled)
def eta(self, index, elapsed):
"""Time remaining at the current rate, blank once finished."""
if not index or index >= self.total:
return ""
return f" eta {self.duration(elapsed / index * (self.total - index))}"
def line(self, index, text):
"""The whole status line for this update."""
fraction = index / self.total if self.total else 1
elapsed = time.monotonic() - self.started
return (f"[{self.bar(fraction)}] {index}/{self.total} "
f"{fraction * 100:3.0f}% {self.tally} "
f"{self.duration(elapsed)}{self.eta(index, elapsed)} {text}")
def draw(self, index, text):
"""Redraw the bar in place, ending the line when finished.
Padded to overwrite a previously longer line, then trimmed to
the terminal: a wrapped line cannot be overwritten by a carriage
return, and the bar starts spilling down the screen.
"""
columns = shutil.get_terminal_size((100, 24)).columns
padded = self.line(index, text)[:columns - 1].ljust(columns - 1)
self.stream.write("\r" + padded)
self.stream.flush()
if index >= self.total:
self.stream.write("\n")
self.stream.flush()
def update(self, index, key=None, text="", fallback_line=None):
"""Record one finished item and show where the stage is."""
if self.started is None:
self.started = time.monotonic()
if key is not None:
self.counts[key] = self.counts.get(key, 0) + 1
if not self.stream.isatty():
if fallback_line is not None:
print(fallback_line, file=self.stream)
return
self.draw(index, text)

0
src/archipelago_tester/core/model/__init__.py
View File

120
src/archipelago_tester/core/model/apworld.py
View File

@@ -1,120 +0,0 @@
"""The filename an apworld needs before Archipelago will import it."""
import json
import os
import re
import zipfile
class ApworldFile:
"""One downloaded apworld, and what its name has to be.
Archipelago derives the module name from the FILENAME, so a version
in the name reads as package separators and a backslash is just a
character. The right name is the zip's own package folder.
"""
#: The extension an apworld is recognised by.
SUFFIX = ".apworld"
def __init__(self, path):
self.path = path
@property
def entries(self):
"""Every entry name in the zip, empty if it will not open."""
try:
with zipfile.ZipFile(self.path) as archive:
return archive.namelist()
except (OSError, zipfile.BadZipFile):
return []
@property
def uses_backslash_separators(self):
"""Whether the zip stores its paths with backslashes.
Invisible on Windows and fatal on Linux: zipimport swaps "/" for
the platform separator when reading, so on Windows both forms
match and on Linux the backslash entry never matches the
"pkg/__init__.py" it looks for. The author sees a working
apworld and every Linux user sees ModuleNotFoundError.
"""
names = self.entries
if not names:
return False
return (not any("/" in name for name in names)
and any("\\" in name for name in names))
@property
def manifest(self):
"""The apworld's own archipelago.json, or None.
Its self-declared "game" and "world_version", sitting in the
file's content regardless of which release was resolved for it -
or whether one ever was. Older apworlds predate the convention
and ship none, so this is best-effort display information rather
than something worth failing over.
"""
try:
with zipfile.ZipFile(self.path) as archive:
name = next((entry for entry in archive.namelist()
if entry.endswith("/archipelago.json")), None)
if name is None:
return None
with archive.open(name) as handle:
return json.load(handle)
except (OSError, zipfile.BadZipFile, json.JSONDecodeError, KeyError):
return None
@property
def package_folder(self):
"""The single top-level package inside the zip, or None."""
names = [name.replace("\\", "/") for name in self.entries]
packages = {
name.split("/", 1)[0]
for name in names
if name.count("/") == 1 and name.endswith("/__init__.py")
}
return packages.pop() if len(packages) == 1 else None
@property
def importable_name(self):
"""What this file must be called for Archipelago to import it."""
base = os.path.basename(self.path.replace("\\", "/"))
if base.lower().endswith(ApworldFile.SUFFIX):
stem = base[:-len(ApworldFile.SUFFIX)]
else:
stem = base
stem = self.package_folder or self.sanitize(stem)
return stem + ApworldFile.SUFFIX
def sanitize(self, stem):
"""A filename stem reduced to something importable."""
base = os.path.basename(stem.replace("\\", "/"))
cleaned = re.sub(r"[^0-9A-Za-z_]", "_", base).strip("_")
if cleaned and cleaned[0].isdigit():
cleaned = "_" + cleaned
return cleaned or "apworld"
def prepare(self, working_directory):
"""The path to hand Archipelago, and whether it was repaired."""
if not self.uses_backslash_separators:
return self.path, False
os.makedirs(working_directory, exist_ok=True)
destination = os.path.join(working_directory, self.importable_name)
return self.write_normalized(destination), True
def write_normalized(self, destination):
"""Copy the apworld with its separators repaired."""
with zipfile.ZipFile(self.path) as source:
with zipfile.ZipFile(destination, "w",
zipfile.ZIP_DEFLATED) as target:
for item in source.infolist():
repaired = zipfile.ZipInfo(
filename=item.filename.replace("\\", "/"),
date_time=item.date_time,
)
repaired.compress_type = item.compress_type
repaired.external_attr = item.external_attr
target.writestr(repaired, source.read(item.filename))
return destination

34
src/archipelago_tester/core/model/blacklist.py
View File

@@ -1,34 +0,0 @@
"""Worlds this pipeline declines to test."""
from archipelago_tester.core.model.name import Name
class Blacklist:
"""Worlds not tested, and why.
A world that times out is the most expensive thing in a run and the
least informative. Entries are permanent until removed by hand -
"do not retry" is the whole reason one is listed.
"""
#: Recorded verdict for a world that was never tested.
OUTCOME = "unknown"
def __init__(self, config):
self.config = config
@property
def entries(self):
"""Folded name -> reason."""
listed = self.config.get("blacklist", {})
return {
Name(name).slug: reason
for name, reason in listed.items()
if Name(name).slug
}
def reason_for(self, game_name):
return self.entries.get(Name(game_name).slug)
def detail(self, reason):
return f"not tested: {reason}"

30
src/archipelago_tester/core/model/fingerprint.py
View File

@@ -1,30 +0,0 @@
"""Content hashes, used to decide what has changed."""
import hashlib
import os
class Fingerprint:
"""The content of a file or a directory, as one hash."""
@staticmethod
def of_file(path):
"""One file's sha256, read in chunks."""
digest = hashlib.sha256()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(65536), b""):
digest.update(chunk)
return digest.hexdigest()
@classmethod
def of_directory(cls, directory):
"""One hash covering every file directly inside a directory."""
digest = hashlib.sha256()
if not os.path.isdir(directory):
return digest.hexdigest()
for name in sorted(os.listdir(directory)):
path = os.path.join(directory, name)
if os.path.isfile(path):
digest.update(name.encode("utf-8"))
digest.update(cls.of_file(path).encode("utf-8"))
return digest.hexdigest()

47
src/archipelago_tester/core/model/game.py
View File

@@ -1,47 +0,0 @@
"""One game from the worlds sheet."""
import re
class Game:
"""A sheet row: a name, its links, and what the sheet says about it."""
def __init__(self, name, links=None, release=None, stability=None,
pr_status=None):
self.name = name
self.links = list(links or [])
self.release = release
self.stability = stability
self.pr_status = pr_status
for link in self.links:
self.remember_release(link)
@staticmethod
def is_release_link(link):
"""Whether a link points at a GitHub releases page."""
return bool(re.match(
r"^https?://(?:www\.)?github\.com/[^/]+/[^/]+/releases"
r"(?:[/?#]|$)",
link or "",
re.IGNORECASE,
))
@property
def is_core(self):
return (self.pr_status or "").strip().lower() == "core"
def remember_release(self, link):
"""Keep this link as the release URL if none was found yet."""
if self.release is None and self.is_release_link(link):
self.release = link
return self.release
def add_link(self, link):
"""Add a link, ignoring one already recorded."""
if link not in self.links:
self.links.append(link)
self.remember_release(link)
return self.links
def __str__(self):
return f"{self.name} ({len(self.links)} links)"

35
src/archipelago_tester/core/model/instant.py
View File

@@ -1,35 +0,0 @@
"""One timestamp, as the moment it names."""
import datetime
class Instant:
"""An ISO 8601 timestamp parsed into a comparable moment.
Hosts do not agree on how to write the offset: GitHub sends a
trailing "Z", this pipeline's own clock writes "+00:00", and GitLab
and Gitea send a local one ("...19:55:36-04:00" is really 23:55Z).
Compared as text those sort by the numbers on the clock face rather
than by when they happened, so anything comparing two timestamps
from different sources has to parse them first.
"""
@staticmethod
def of(text):
"""One timestamp, or None when it is not a usable one.
A naive timestamp is read as UTC: everything written here is,
and the alternative is a comparison that raises rather than one
that is merely approximate. "Z" is spelled out because
fromisoformat only accepts it from 3.11.
"""
if not text:
return None
try:
parsed = datetime.datetime.fromisoformat(
str(text).replace("Z", "+00:00"))
except (TypeError, ValueError):
return None
if parsed.tzinfo is None:
return parsed.replace(tzinfo=datetime.timezone.utc)
return parsed

22
src/archipelago_tester/core/model/modes.py
View File

@@ -1,22 +0,0 @@
"""The generation modes a complete result carries."""
class Modes:
"""What the driver runs against every world.
Not configuration: each name is a branch in the container driver
and in Verdict, so a mode cannot be added or removed by editing a
list. Changing this would only desynchronise the cache check and
the results table from what actually ran.
"""
SINGLE = "single"
DUPLICATE = "duplicate"
MULTI = "multi"
SINGLE_RANDOM = "single_random"
MULTI_RANDOM = "multi_random"
ALL = (SINGLE, DUPLICATE, MULTI, SINGLE_RANDOM, MULTI_RANDOM)
#: The modes that cannot demote a world on their own.
SECONDARY = (DUPLICATE, SINGLE_RANDOM, MULTI_RANDOM)

42
src/archipelago_tester/core/model/name.py
View File

@@ -1,42 +0,0 @@
"""Folding a game name down to a comparison key."""
import re
import unicodedata
class Name:
"""One game name, and the two ways it gets folded.
One implementation, because five copies once disagreed and a world
silently lost its blacklist entry.
"""
def __init__(self, text):
self.text = text or ""
@property
def slug(self):
"""The key for matching this name against another."""
folded = unicodedata.normalize("NFKD", self.text.lower())
return re.compile(r"[^a-z0-9]").sub("", folded)
@property
def report_slug(self):
return re.compile(r"[^a-z0-9]+").sub("-", self.text.lower()).strip("-")
@property
def directory(self):
return re.sub(r'[<>:"/\\|?*]', "_", self.text).strip().rstrip(".")
def matches(self, other):
"""Whether an internally registered name means this game.
One-directional: the internal name is usually the shorter,
canonical one and the sheet's is often longer, so checking
containment the other way would let a short internal name match
an unrelated row that happens to start the same way.
"""
if not self.text or not other:
return False
internal = Name(other).slug
return internal == self.slug or internal in self.slug

71
src/archipelago_tester/core/model/verdict.py
View File

@@ -1,71 +0,0 @@
"""Where a world's test record lands on the stability ladder."""
from archipelago_tester.core.model.modes import Modes
class Verdict:
"""One world's tested stability.
The ladder: broken, unknown, flaky, solo_only, unstable, stable.
Single and multi decide the first four, because those are what a
player does; the other modes can only stop a world being stable.
"""
#: Outcomes that count as a failure in a secondary mode.
FAILURES = ("failed", "invalid_options", "flaky")
def __init__(self, record, config):
self.record = record or {}
self.config = config
@property
def modes(self):
return self.record.get("tests") or {}
@property
def never_imported(self):
return self.record.get("outcome") == "unverified"
def outcome_of(self, mode):
return (self.modes.get(mode) or {}).get("outcome")
@property
def primary(self):
"""What the single and multi runs decide, or None if they pass."""
single = self.outcome_of(Modes.SINGLE)
multi = self.outcome_of(Modes.MULTI)
if single == "needs_input" or multi == "needs_input":
return "unknown"
if single == "failed":
return "broken"
if single == "flaky" or multi == "flaky":
return "flaky"
if multi == "failed":
return "solo_only"
if single != "passed":
return "unknown"
return None
@property
def secondary_failed(self):
"""Whether a mode that cannot demote alone still failed."""
return any(self.outcome_of(mode) in self.FAILURES
for mode in Modes.SECONDARY)
@property
def name(self):
"""Where this record lands on the ladder."""
if self.never_imported:
return "broken"
if not self.modes or not self.outcome_of(Modes.SINGLE):
return "unknown"
decided = self.primary
if decided is not None:
return decided
return "unstable" if self.secondary_failed else "stable"
@property
def label(self):
labels = self.config.value("stability", "labels", {})
return labels.get(self.name, "Unknown")

0
src/archipelago_tester/core/state/__init__.py
View File

23
src/archipelago_tester/core/state/keys.py
View File

@@ -1,23 +0,0 @@
"""The keys this pipeline writes into state.json."""
class StateKeys:
"""Names of the entries in the state file.
Not configuration: state.json is written and read only by this
pipeline, so renaming a key here would not migrate the file, it
would orphan every record already in it.
"""
APWORLD_TESTS = "apworld_tests"
UPGRADE_SNAPSHOT = "apworld_tests_at_last_upgrade"
ARCHIPELAGO_TAG = "archipelago_tag"
ARCHIPELAGO_TAG_PUBLISHED_AT = "archipelago_tag_published_at"
PREVIOUS_ARCHIPELAGO_TAG = "previous_archipelago_tag"
COMPANION_POOL = "verified_companions"
GAME_RELEASES = "game_releases"
DISCONTINUED = "discontinued"
LAST_CHECKED_AT = "last_checked_at"
FAILED_RELEASE = "failed_release"
FAILED_REASON = "failed_reason"
STABILITY_HISTORY = "stability_history"

37
src/archipelago_tester/core/state/lock.py
View File

@@ -1,37 +0,0 @@
"""The exclusive lock a whole run holds."""
import fcntl
class LockHeldError(RuntimeError):
"""Another run already holds the lock."""
class PipelineLock:
"""Stops two runs, or a run and a promotion, from overlapping.
Non-blocking: a second run fails fast rather than queuing. Queuing
would be worse - a full pass takes hours, so a queued one would
start against an Archipelago version that had moved on.
"""
def __init__(self, path):
self.path = path
self.handle = None
def __enter__(self):
self.handle = open(self.path, "a+")
try:
fcntl.flock(self.handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError as error:
self.handle.close()
raise LockHeldError(
f"Another run already holds {self.path} - refusing to "
"start."
) from error
return self
def __exit__(self, *unused):
fcntl.flock(self.handle, fcntl.LOCK_UN)
self.handle.close()
return False

32
src/archipelago_tester/core/state/scratch.py
View File

@@ -1,32 +0,0 @@
"""Choosing between the live record and a scratch copy of it."""
import os
import shutil
class ScratchState:
"""Where a run's results go, live or not.
A run is an experiment unless it says otherwise: promotion and the
status page read the live file, so a hand-run test that wrote there
would put a world nobody can download into the deployable record.
Seeding the scratch copy from the live one is what makes that cheap
rather than punishing - without it a scratch run has no recorded
Archipelago version to test against and no verified companion pool,
so it would stop for the first and re-check every core world for
the second.
"""
def __init__(self, paths, live=False):
self.paths = paths
self.live = live
@property
def path(self):
if self.live:
return self.paths.state
scratch = self.paths.scratch_state
if not os.path.exists(scratch) and os.path.exists(self.paths.state):
shutil.copy(self.paths.state, scratch)
return scratch

55
src/archipelago_tester/core/state/store.py
View File

@@ -1,55 +0,0 @@
"""Reading and writing the pipeline's JSON state."""
import json
import os
import tempfile
class StateStore:
"""What every world last did, and the companion pool behind it."""
def __init__(self, path):
self.path = path
def load(self):
return self.read(self.path, {})
def save(self, data):
self.write(self.path, data)
@staticmethod
def read(path, default=None):
"""A JSON file's contents, or `default` when it is absent."""
if not os.path.exists(path):
return {} if default is None else default
with open(path, "r", encoding="utf-8") as handle:
return json.load(handle)
@staticmethod
def write(path, data):
"""Write JSON so a reader never sees a partial file.
A kill mid-write used to leave state.json truncated mid-object -
valid up to the cut point and invalid after it - silently
costing every recorded result on the next load. The batch saves
after every single test, so that window was hit often.
os.replace is atomic: the file is always either the old complete
version or the new one.
"""
directory = os.path.dirname(os.path.abspath(path)) or "."
handle_id, temporary = tempfile.mkstemp(
dir=directory, prefix=".state_", suffix=".tmp")
try:
with os.fdopen(handle_id, "w", encoding="utf-8") as handle:
json.dump(data, handle, indent=2, sort_keys=True)
handle.write("\n")
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
except BaseException:
try:
os.remove(temporary)
except OSError:
pass
raise

0
src/archipelago_tester/pipeline/__init__.py
View File

0
src/archipelago_tester/pipeline/batch/__init__.py
View File

67
src/archipelago_tester/pipeline/batch/apworld_finder.py
View File

@@ -1,67 +0,0 @@
"""Finding the apworlds already downloaded."""
import os
class ApworldFinder:
"""The download directory, as the worlds a run can test."""
def __init__(self, root):
self.root = root
@staticmethod
def matches(game_name, only, exact):
if not only:
return True
if exact:
return any(t.casefold() == game_name.casefold() for t in only)
return any(t.lower() in game_name.lower() for t in only)
def in_folder(self, game_dir):
"""Every apworld directly inside one game's folder.
A directory where a file should be is skipped rather than
crashing the batch: that is almost always docker auto-creating
an empty directory at a bind-mount source that did not exist
yet, and a fresh download replaces it next time.
"""
found = []
for entry in sorted(os.listdir(game_dir)):
if not entry.lower().endswith(".apworld"):
continue
path = os.path.join(game_dir, entry)
if not os.path.isfile(path):
print(f"Skipping {path} - not a regular file (directory?)")
continue
found.append(path)
return found
def find(self, only=None, exact=False):
"""Every downloaded apworld, optionally narrowed to `only`.
`only` matches on substring, which is what --only wants. That
is ambiguous when one game's name contains another's - "Hollow
Knight" also selects "Hollow Knight_ Silksong" - so a caller
that already resolved a term to a real folder passes exact.
"""
found = []
if not os.path.isdir(self.root):
return found
for game_name in sorted(os.listdir(self.root)):
if not self.matches(game_name, only, exact):
continue
game_dir = os.path.join(self.root, game_name)
if not os.path.isdir(game_dir):
continue
found += [(game_name, path) for path in self.in_folder(game_dir)]
return found
@property
def available(self):
return sorted({name for name, _ in self.find()})
def resolve(self, term):
"""The folders a name selects, an exact match winning."""
matches = sorted({name for name, _ in self.find(only=[term])})
exact = [n for n in matches if n.casefold() == term.casefold()]
return exact or matches

119
src/archipelago_tester/pipeline/batch/batch.py
View File

@@ -1,119 +0,0 @@
"""Testing every downloaded apworld and recording what each did."""
from archipelago_tester.core.model.fingerprint import Fingerprint
from archipelago_tester.core.state.keys import StateKeys
from archipelago_tester.core.state.store import StateStore
from archipelago_tester.pipeline.batch.companion_pool import CompanionPool
from archipelago_tester.pipeline.batch.failed_report import FailedReport
from archipelago_tester.pipeline.batch.options import Options
from archipelago_tester.pipeline.batch.planner import Planner
from archipelago_tester.pipeline.batch.runner import Runner
from archipelago_tester.pipeline.batch.work_list import WorkList
class Batch:
"""One run: plan it, serve what is cached, test the rest.
An apworld whose file content and Archipelago version both match
its last recorded test is served from cache. The caller forces a
full re-test when the version changes, because last week's passes
no longer apply to anything.
"""
def __init__(self, config, paths, settings, image, tag, force=False,
on_progress=None, **asked):
self.config = config
self.paths = paths
self.settings = settings
self.image = image
self.tag = tag
self.force = force or bool(asked.get("only"))
self.on_progress = on_progress
self.options = Options(paths, settings, **asked).resolve()
self.store = StateStore(self.options["state_path"])
@property
def writes_live_report(self):
"""Whether this run owns the "what is broken" report.
The report is about the deployable, current-version state: a
previous-version comparison writing here would conflate "not
promotable now" with "does not work on a release nobody runs",
and a run pointed at a scratch state file is not the live
record either.
"""
return (self.options["state_key"]
== StateKeys.APWORLD_TESTS
and self.options["state_path"] == self.paths.state)
def pool(self):
return CompanionPool(
config=self.config,
paths=self.paths,
store=self.store,
image=self.image,
tag=self.tag,
roms_directory=self.options["roms_directory"],
).resolve()
def plan(self, tests, fingerprint, state):
"""Everything to consider, what is known, what to run.
Blacklisted rows and the confirmation date on a cached one are
the entries written outside the runner, which is what normally
persists state - without the save here they would exist only
in memory, and a run that tested nothing would write nothing.
"""
work = WorkList(
root=self.options["root_directory"],
only=self.options["only"],
exact=self.options["exact_only"],
core_games=self.options["core_games"],
).items()
planner = Planner(
config=self.config,
settings=self.settings,
tests=tests,
tag=self.tag,
roms_fingerprint=fingerprint,
force=self.force,
repeats=self.options["repeats"],
random_repeats=self.options["random_repeats"],
companion_range=self.options["companion_range"],
# A release published after a world's last result
# invalidates it, whatever the file on disk hashes to.
releases=state.get(StateKeys.GAME_RELEASES, {}),
)
results, to_test = planner.plan(work)
if planner.wrote_state:
self.store.save(state)
return work, results, to_test
def report_cached(self, results, total):
"""Free to produce, so reported without waiting on the pool."""
for index, report in enumerate(results, start=1):
if self.on_progress:
self.on_progress(index, total, report)
def run(self):
self.options["pool"] = self.pool()
state = self.store.load()
tests = state.setdefault(self.options["state_key"], {})
fingerprint = Fingerprint.of_directory(self.options["roms_directory"])
work, results, to_test = self.plan(tests, fingerprint, state)
self.report_cached(results, len(work))
if to_test:
results += Runner(
config=self.config,
paths=self.paths,
store=self.store,
options=self.options,
state=state,
tests=tests,
tag=self.tag,
image=self.image,
roms_fingerprint=fingerprint,
).run(to_test, len(work), len(results), self.on_progress)
if self.writes_live_report:
FailedReport(state, self.paths.failed_report).write()
return results

120
src/archipelago_tester/pipeline/batch/cache_check.py
View File

@@ -1,120 +0,0 @@
"""Whether a previous result still answers this run's question."""
import re
from archipelago_tester.core.model.instant import Instant
from archipelago_tester.core.model.modes import Modes
class CacheCheck:
"""One world's recorded result, against what is being asked now."""
def __init__(self, settings, previous):
self.settings = settings
self.previous = previous
#: Failure text pointing at a base ROM, not the world's logic. A
#: wrong or missing ROM surfaces under many exception types, so the
#: type cannot recognise one and the message is what is left.
#: Word-bounded so "from" does not read as "rom".
ROM_FAILURE = re.compile(
r"\b(rom|md5|sha1|checksum)\b"
r"|\.(sfc|smc|z64|n64|nes|gba|gbc|gb|gen|md|iso|bin)\b",
re.IGNORECASE,
)
def mentions_rom(self, detail):
return bool(detail and self.ROM_FAILURE.search(str(detail)))
def needs_rom_retry(self, roms_fingerprint):
"""Whether a changed ROM folder invalidates this result.
A missing ROM raises FileNotFoundError and lands in
"needs_input", but a WRONG one raises whatever the world throws
after reading it and lands in "failed" - so retrying only
needs_input meant replacing a bad dump could never clear the
result. A "failed" result additionally has to look like a ROM
problem: most failures carrying a stale fingerprint were fill
failures and deprecated APIs that no ROM will fix.
"""
if self.previous is None:
return False
outcome = self.previous.get("outcome")
rom_related = (outcome == "needs_input"
or (outcome == "failed"
and self.mentions_rom(self.previous.get("detail"))))
return (rom_related and self.previous.get("roms_fingerprint")
!= roms_fingerprint)
def released_since(self, released_at):
"""Whether the row's apworld was published after it was tested.
The hash catches a file whose content changed, but not a host
that offered a new one and had it turn out identical - and
"identical" is only ever this pipeline's word for it, resting
on the copy it happened to hold. A publication later than the
answer means the answer was reached before whatever is being
offered now, and testing again is the only thing that settles
it. It settles it for good, too: the new result is dated after
the release, so this stops firing until the release moves
again.
"""
if self.previous is None:
return False
released = Instant.of(released_at)
tested = Instant.of(self.previous.get("tested_at"))
if released is None or tested is None:
return False
return tested < released
@staticmethod
def attempts(recorded_modes):
"""How many attempts a cached record was reached with.
The randomized modes used to run exactly once and carried no
count at all, so a missing value there means "one".
"""
if not recorded_modes:
return None, None
return (
(recorded_modes.get("single") or {}).get("requested_attempts"),
(recorded_modes.get("single_random") or {})
.get("requested_attempts"),
)
def answers(self, repeats, random_repeats, companion_range):
"""Whether the record answers what this run is asking.
It does not when the record predates per-mode results, when the
set of modes has changed, when it was reached with fewer
repeats - "it generated once" is not "it generated ten times
out of ten" - or when the companion range differs. A world that
never imported is exempt: "unverified" is a complete answer
about this exact file, and the file has to change for the
answer to.
"""
if self.previous is None:
return True
if self.previous.get("outcome") == "unverified":
return True
recorded = self.previous.get("tests")
if recorded is None or set(recorded) != set(Modes.ALL):
return False
single, randomized = self.attempts(recorded)
if single != repeats:
return False
if randomized is not None and randomized != random_repeats:
return False
recorded_range = (recorded.get("multi") or {}).get(
"companion_range") or []
return list(recorded_range) == list(companion_range)
def serves(self, current_hash, tag, roms_fingerprint, repeats,
random_repeats, companion_range, released_at=None):
"""Whether this world can be served from its previous result."""
return (self.previous is not None
and self.previous.get("hash") == current_hash
and self.previous.get("tag") == tag
and not self.needs_rom_retry(roms_fingerprint)
and not self.released_since(released_at)
and self.answers(repeats, random_repeats, companion_range))

152
src/archipelago_tester/pipeline/batch/cli.py
View File

@@ -1,152 +0,0 @@
"""The command line for a full batch run."""
import argparse
import json
from archipelago_tester.core.config.config import Config
from archipelago_tester.core.config.paths import Paths
from archipelago_tester.core.config.run_settings import RunSettings
from archipelago_tester.core.state.keys import StateKeys
from archipelago_tester.core.state.lock import LockHeldError, PipelineLock
from archipelago_tester.core.state.scratch import ScratchState
from archipelago_tester.core.state.store import StateStore
from archipelago_tester.pipeline.batch.batch import Batch
from archipelago_tester.pipeline.batch.reporter import Reporter
from archipelago_tester.pipeline.batch.results import Results
from archipelago_tester.pipeline.batch.version import Version
from archipelago_tester.pipeline.build.archipelago import ArchipelagoBuild
class Cli:
"""Test every downloaded apworld, or the ones named with --only."""
def __init__(self):
self.config = Config.load()
self.paths = Paths(self.config)
self.settings = RunSettings(self.config)
self.build = ArchipelagoBuild(self.config, self.paths)
def parser(self):
parser = argparse.ArgumentParser(description=self.__doc__)
parser.add_argument("--root-directory", default=self.paths.downloads)
parser.add_argument("--output-dir", default=self.paths.output)
parser.add_argument(
"--state-path", default=None,
help="Where results are recorded. Defaults to the scratch "
"file, or to the live state.json with --live; naming a "
"path here overrides both.")
parser.add_argument(
"--live", action="store_true",
help="Record into the real state.json, which promotion and "
"the status page read. Off by default: a run started "
"by hand is an experiment until it says otherwise.")
parser.add_argument("--timeout", type=int, default=None)
parser.add_argument(
"--force", action="store_true",
help="Re-test every apworld, ignoring the unchanged-since-"
"last-time skip.")
parser.add_argument(
"--only", action="append",
help="Only check games whose name contains this substring "
"(repeatable). Always retests matches regardless of cache.")
parser.add_argument(
"--update", action="store_true",
help="Resolve the newest Archipelago release, build it if "
"needed, and record it. Off by default: a new version "
"invalidates every cached result.")
parser.add_argument(
"--image",
help="Image to test against; defaults to archipelago-test:"
"<tag> for the version recorded in state.json (which "
"only --update changes).")
parser.add_argument(
"--jobs", "-j", type=int, default=self.settings.jobs,
help=f"Apworlds to test at once (default: {self.settings.jobs}).")
return parser
def state_path(self, args):
"""The file this run records into.
An explicit --state-path wins, so a developer can keep a run
entirely to themselves; otherwise --live decides, and its
default sends results to a scratch copy of the live record.
"""
return args.state_path or ScratchState(self.paths,
live=args.live).path
def announce(self, state_path):
"""Say where results land, because the default is not the
live record and a silent run would not show which it chose."""
if state_path == self.paths.state:
print(f"results {state_path}")
return
print(f"results {state_path} (scratch, not the live "
"record - pass --live to record for real)")
def batch(self, args, tag, state_path):
return Batch(
config=self.config,
paths=self.paths,
settings=self.settings,
image=args.image or self.build.test_image_tag(tag),
tag=tag,
force=args.force,
on_progress=Reporter(self.config, bar=True),
root_directory=args.root_directory,
output_directory=args.output_dir,
state_path=state_path,
timeout=args.timeout,
only=args.only,
jobs=args.jobs,
).run()
def report(self, results):
"""Print the summary a finished run ends on."""
summary = Results(results, self.config).summary
print(f"\n{json.dumps(summary)}")
failed = [r["game"] for r in results if r.get("outcome") == "failed"]
if failed:
print(f"failed: {', '.join(failed)}")
if not results:
print("No apworlds found to test.")
return 0 if summary.get("failed", 0) == 0 else 1
def version(self, args, store):
"""The version to test against, or None with a reason said.
Without --update the recorded version is used as-is: nothing
upgrades behind the caller's back, because a version change
forces every world to be tested again.
"""
if args.update:
tag, source = Version(self.config, self.paths, store).prepare(
update=True)
print(f"archipelago {tag} ({source})")
return tag
tag = store.load().get(StateKeys.ARCHIPELAGO_TAG)
if not tag:
print("No Archipelago version recorded in state.json - run "
"archipelago-test --update to resolve and build one.")
return tag
def run(self):
args = self.parser().parse_args()
state_path = self.state_path(args)
self.announce(state_path)
store = StateStore(state_path)
tag = self.version(args, store)
if not tag:
return 1
if not args.image:
self.build.ensure_test_image(tag)
try:
with PipelineLock(self.paths.lock):
return self.report(self.batch(args, tag, state_path))
except LockHeldError as error:
print(error)
return 1
@classmethod
def main(cls):
"""The console script's entry point."""
return cls().run()

65
src/archipelago_tester/pipeline/batch/companion_pool.py
View File

@@ -1,65 +0,0 @@
"""The core worlds a run may use as companions."""
from datetime import datetime, timezone
from archipelago_tester.core.model.fingerprint import Fingerprint
from archipelago_tester.core.state.keys import StateKeys
from archipelago_tester.pipeline.generation.companion_check import (
CompanionCheck,
)
class CompanionPool:
"""Verified once per run and cached against version and ROMs.
The answer only changes when Archipelago or the ROMs folder does,
and what it rejects would otherwise fail seeds belonging to the
world under test.
"""
def __init__(self, config, paths, store, image, tag, roms_directory):
self.config = config
self.paths = paths
self.store = store
self.image = image
self.tag = tag
self.fingerprint = Fingerprint.of_directory(roms_directory)
@property
def key(self):
return StateKeys.COMPANION_POOL
def cached(self, state):
cached = state.get(self.key) or {}
if (cached.get("tag") == self.tag
and cached.get("roms_fingerprint") == self.fingerprint):
return cached.get("games") or []
return None
def store_result(self, state, result):
state[self.key] = {
"tag": self.tag,
"roms_fingerprint": self.fingerprint,
"games": result["verified"],
"rejected": result["rejected"],
"checked_at": datetime.now(timezone.utc).isoformat(),
}
self.store.save(state)
print(f"{len(result['verified'])} core worlds usable as companions, "
f"{len(result['rejected'])} rejected")
for game, reason in sorted(result["rejected"].items()):
print(f" {game}: {reason[:90]}")
return result["verified"]
def resolve(self):
state = self.store.load()
cached = self.cached(state)
if cached is not None:
return cached
result = CompanionCheck(self.config, self.paths, self.image).run(
self.paths.output)
if not result.get("ran"):
print(f"companion verification failed ({result.get('detail')}) - "
"multi-game tests will draw from every core world")
return []
return self.store_result(state, result)

69
src/archipelago_tester/pipeline/batch/discontinued.py
View File

@@ -1,69 +0,0 @@
"""Flagging test records whose game left the sheet."""
from archipelago_tester.core.model.name import Name
from archipelago_tester.core.state.keys import StateKeys
class Discontinued:
"""A world taken off the sheet stops being one this pipeline
tracks: it should leave the status page and stop being deployed.
The record is kept and labelled rather than deleted, so its history
survives and a row that comes back - or that vanished because of a
bad sheet parse - is one flag away from being live again.
"""
def __init__(self, store):
self.store = store
@staticmethod
def sheet_name(key, record):
"""The sheet row a record's key came from.
A download- or core-stage record is keyed on the bare sheet
name, since nothing was ever fetched for it. Everything else is
keyed "<folder>/<file>.apworld". The record's own "game" is
deliberately not used: that is the name Archipelago registered
the world under, which can differ from the sheet's wording.
"""
if (record or {}).get("stage") in ("download", "core"):
return key
return key.split("/", 1)[0]
def apply(self, tests, live):
"""Set or clear the flag, reporting only what moved.
Comparison is on the folded directory name of both sides, which
is what turned the sheet name into the download folder in the
first place, so punctuation and case cannot make a still-listed
game look discontinued. An unchanged run leaves the state file
byte-identical.
"""
flag = StateKeys.DISCONTINUED
changed = {}
for key, record in tests.items():
name = self.sheet_name(key, record)
gone = Name(name).directory not in live
if gone != bool(record.get(flag)):
changed[key] = gone
if gone:
record[flag] = True
else:
record.pop(flag, None)
return changed
def mark(self, sheet_names):
"""Flag every record whose game is no longer listed.
Does nothing when handed no sheet names: an empty list means
the sheet was not read, not that every world on earth was
retired, and acting on it would empty the page.
"""
if not sheet_names:
return {}
state = self.store.load()
tests = state.get(StateKeys.APWORLD_TESTS, {})
changed = self.apply(
tests, {Name(name).directory for name in sheet_names})
if changed:
self.store.save(state)
return changed

88
src/archipelago_tester/pipeline/batch/download_failures.py
View File

@@ -1,88 +0,0 @@
"""Recording games whose apworld could never be fetched."""
from datetime import datetime, timezone
from archipelago_tester.core.model.name import Name
from archipelago_tester.core.state.keys import StateKeys
from archipelago_tester.pipeline.batch.record import TestRecord
class DownloadFailures:
"""A game with no file has nothing to key a result on the way a
tested world does, so it is keyed on its own sheet name - and still
shows up as an unknown row rather than vanishing between the sheet
and the results. Real result keys always contain "/", so a bare
name cannot collide with one.
"""
def __init__(self, config, store):
self.config = config
self.store = store
@staticmethod
def tested_already(tests, name):
"""Whether a real result already answers for this game.
A download row exists so a game with no file still shows up
rather than vanishing between the sheet and the results. When
an apworld is on disk and was tested, nothing is vanishing -
the link broke, which the download report says, while the file
that world was last tested from is still there and still the
answer. Writing a second row then reports one game twice, and
the two rows collide at promotion because they claim the same
game name.
"""
folded = Name(name).directory
return any(
"/" in key and Name(key.split("/", 1)[0]).directory == folded
for key in tests
)
@staticmethod
def detail_for(reason):
return ("apworld could not be downloaded, so it was never tested: "
f"{reason}")
def clear_stale(self, tests, games, skipped):
"""Drop rows for games that downloaded this time.
Only ever touches entries this class wrote, never a real
generation-test result.
"""
failed = {name for name, _ in skipped}
for game in games:
previous = tests.get(game.name)
if (game.name not in failed and previous is not None
and previous.get("stage") == "download"):
del tests[game.name]
def note(self, tests, name, reason, tag, now):
"""Record one failure, keeping an unchanged row's date.
A row failing the same way as last time keeps the date of the
attempt that reached that answer and only has its last-checked
stamp moved: a test date that advanced every run would claim a
world was looked at when nothing about it was.
"""
detail = self.detail_for(reason)
previous = tests.get(name)
if (previous is not None and previous.get("stage") == "download"
and previous.get("detail") == detail):
previous["tag"] = tag
previous[StateKeys.LAST_CHECKED_AT] = now
return
tests[name] = TestRecord(name).download_failure(
detail, tag)
def record(self, games, skipped, tag):
state = self.store.load()
tests = state.setdefault(StateKeys.APWORLD_TESTS, {})
self.clear_stale(tests, games, skipped)
now = datetime.now(timezone.utc).isoformat()
for name, reason in skipped:
if self.tested_already(tests, name):
tests.pop(name, None)
continue
self.note(tests, name, reason, tag, now)
self.store.save(state)
return state

36
src/archipelago_tester/pipeline/batch/failed_report.py
View File

@@ -1,36 +0,0 @@
"""The list of worlds that were tested and did not pass."""
from archipelago_tester.core.state.keys import StateKeys
class FailedReport:
"""Written after every run, from the whole accumulated state.
Not just this run's results: a targeted run touches a couple of
entries and the report must still reflect every other known
failure. Kept out of state.json, which is bookkeeping rather than a
report, so answering the question does not mean writing a query.
"""
def __init__(self, state, path):
self.state = state
self.path = path
def failed(self):
tests = self.state.get(StateKeys.APWORLD_TESTS, {})
return sorted(
((key, record) for key, record in tests.items()
if record.get("outcome") == "failed"),
key=lambda item: item[1].get("game") or item[0],
)
def write(self):
with open(self.path, "w", encoding="utf-8") as handle:
failed = self.failed()
if not failed:
handle.write("No failed apworlds.\n")
return
for key, record in failed:
detail = (record.get("detail") or "").splitlines()[0]
handle.write(
f"{record.get('game') or key} ({key}): {detail}\n")

92
src/archipelago_tester/pipeline/batch/fetcher.py
View File

@@ -1,92 +0,0 @@
"""Downloading the worlds a run was asked for but does not have."""
import os
import requests
from archipelago_tester.core.config.secrets import Secrets
from archipelago_tester.pipeline.batch.apworld_finder import ApworldFinder
from archipelago_tester.pipeline.download.download_run import DownloadRun
from archipelago_tester.pipeline.sheet.fetcher import SheetFetcher
from archipelago_tester.pipeline.sheet.reader import SheetReader
class Fetcher:
"""Fills the download directory from the sheet, on demand.
That directory is a working directory this pipeline fills itself,
from the same sheet a full pass reads - not somewhere a user
points at their own copy. A world never fetched is work not done
yet rather than an error.
"""
def __init__(self, config, paths, root=None):
self.config = config
self.paths = paths
self.root = root or paths.downloads
self.finder = ApworldFinder(self.root)
def session(self):
session = requests.Session()
token = Secrets(self.paths).get("GITHUB_TOKEN")
if token:
session.headers["Authorization"] = f"token {token}"
return session
def listed(self):
"""The sheet's rows, fetching the sheet if this box has none."""
fetcher = SheetFetcher(self.config, self.paths)
if not os.path.exists(self.paths.sheet_html):
print(f"fetching the {fetcher.tab_name} sheet "
f"-> {self.paths.sheet_html}")
fetcher.ensure()
return SheetReader(self.config, self.paths.sheet_html).games()
@staticmethod
def row_for(term, listed):
"""The one sheet row a name selects.
An exact name wins over a substring, and a term still matching
several rows is refused rather than guessed at - downloading
and testing two worlds because a name was ambiguous is worse
than saying so.
"""
rows = [g for g in listed if term.lower() in g.name.lower()]
exact = [g for g in rows if g.name.casefold() == term.casefold()]
rows = exact or rows
if not rows:
raise LookupError(
f"{term!r} is not on the worlds sheet - nothing to "
"download and nothing to test")
if len(rows) > 1:
names = ", ".join(g.name for g in rows)
raise LookupError(f"{term!r} matches {len(rows)} sheet rows: "
f"{names}. Name one of them exactly.")
return rows[0]
def report(self, terms):
"""Say where each requested world landed, or raise."""
for term in terms:
found = self.finder.find(only=[term])
if not found:
raise LookupError(
f"no apworld could be obtained for {term!r}")
for name, path in found:
print(f" {name}: {path}")
def ensure(self, terms):
"""Download whichever of `terms` is not already here."""
missing = [t for t in terms if not self.finder.find(only=[t])]
if not missing:
return []
listed = self.listed()
wanted = [self.row_for(term, listed) for term in missing]
print(f"downloading {', '.join(g.name for g in wanted)} "
f"-> {self.root}")
run = DownloadRun(self.config, self.paths, self.session(),
root_directory=self.root)
_, skipped, _ = run.all(wanted)
for name, reason in skipped:
print(f" {name} could not be downloaded: {reason}")
self.report(missing)
return [g.name for g in wanted]

48
src/archipelago_tester/pipeline/batch/options.py
View File

@@ -1,48 +0,0 @@
"""What one batch run was asked for, with the gaps filled in."""
from archipelago_tester.core.state.keys import StateKeys
class Options:
"""The caller's choices, over config.json's defaults.
`repeats` of None means "whatever config.json says". An explicit
count trades confidence for time, and the cache check compares
against it, so a cheap run cannot masquerade as a full one.
"""
def __init__(self, paths, settings, **asked):
self.paths = paths
self.settings = settings
self.asked = asked
def chosen(self, key, default):
value = self.asked.get(key)
return default if value is None else value
def counted(self, key, default):
value = self.asked.get(key)
return default if value is None else int(value)
def resolve(self):
return {
"root_directory": self.chosen("root_directory",
self.paths.downloads),
"output_directory": self.chosen("output_directory",
self.paths.output),
"state_path": self.chosen("state_path", self.paths.state),
"roms_directory": self.chosen("roms_directory", self.paths.roms),
"jobs": self.chosen("jobs", self.settings.jobs),
"state_key": self.chosen(
"state_key", StateKeys.APWORLD_TESTS),
"companion_range": self.chosen("companion_range",
self.settings.companion_range),
"repeats": self.counted("repeats", self.settings.repeats),
"random_repeats": self.counted("random_repeats",
self.settings.random_repeats),
"spoiler": self.settings.spoiler,
"timeout": self.asked.get("timeout"),
"only": self.asked.get("only"),
"exact_only": bool(self.asked.get("exact_only")),
"core_games": self.asked.get("core_games") or (),
}

57
src/archipelago_tester/pipeline/batch/orphaned_records.py
View File

@@ -1,57 +0,0 @@
"""Dropping test records whose apworld is no longer on disk."""
import os
from archipelago_tester.core.state.keys import StateKeys
class OrphanedRecords:
"""Results for files that have since gone.
A record is keyed on the path its apworld had under the downloads
root, so a file that is removed - a release that renamed its asset,
a folder cleared by hand - leaves its result behind. Nothing reads
that record as stale: it still names a tag and a verdict, so the
status page goes on showing the world, and where the replacement
file sits beside it the game is reported twice.
Only records with a file behind them are considered. A download- or
core-stage record is keyed on the bare sheet name and never had one,
and Discontinued is what retires those.
"""
def __init__(self, store, downloads):
self.store = store
self.downloads = downloads
def orphaned(self, tests):
"""Every keyed-on-a-path record whose file is missing."""
return [key for key, record in tests.items()
if "/" in key
and (record or {}).get("stage") not in ("download", "core")
and not os.path.isfile(os.path.join(self.downloads, key))]
def safe(self):
"""Whether the downloads folder looks like itself.
An unreadable or empty root would make every record an orphan
and empty the page, which is the one outcome worth refusing:
a genuinely empty downloads directory has nothing to prune for
anyway, so declining costs nothing and guessing costs the lot.
"""
if not os.path.isdir(self.downloads):
return False
return any(os.scandir(self.downloads))
def prune(self):
"""Remove them, returning the keys that went."""
if not self.safe():
return []
state = self.store.load()
tests = state.get(StateKeys.APWORLD_TESTS, {})
gone = self.orphaned(tests)
for key in gone:
del tests[key]
if gone:
self.store.save(state)
return gone

106
src/archipelago_tester/pipeline/batch/planner.py
View File

@@ -1,106 +0,0 @@
"""Splitting the work into known results and tests still to run."""
from archipelago_tester.core.model.blacklist import Blacklist
from archipelago_tester.core.model.name import Name
from archipelago_tester.core.state.keys import StateKeys
from archipelago_tester.pipeline.batch.cache_check import CacheCheck
from archipelago_tester.pipeline.batch.record import TestRecord
class Planner:
"""What is already answered, and what has to be generated.
The blacklist is checked before the cache and is unaffected by a
force flag: "do not retry" is the entire reason a world is listed,
so a flag that retried it would defeat the point.
"""
def __init__(self, config, settings, tests, tag, roms_fingerprint,
force, repeats, random_repeats, companion_range,
releases=None):
self.config = config
self.settings = settings
self.tests = tests
self.tag = tag
self.roms_fingerprint = roms_fingerprint
self.force = force
self.repeats = repeats
self.random_repeats = random_repeats
self.companion_range = companion_range
self.blacklist = Blacklist(config)
# Keyed on the folded name: the work list names a row by
# its download folder, while game_releases keys on the
# sheet's own wording, and the two differ wherever the
# sheet uses a colon or its own capitalisation.
self.releases = {
Name(name).directory: (release or {}).get("published_at")
for name, release in (releases or {}).items()
}
self.wrote_state = False
def record_for(self, item):
game_name, apworld_path, key, _ = item
return TestRecord(game_name, key,
is_core=apworld_path is None)
def decline(self, item, reason):
"""Write and return the result for a declined world."""
_, _, key, current_hash = item
record = self.record_for(item)
self.tests[key] = record.blacklisted(
blacklist=self.blacklist,
current_hash=current_hash,
tag=self.tag,
reason=reason,
roms_fingerprint=self.roms_fingerprint,
)
self.wrote_state = True
return record.as_result(self.tests[key])
def confirm(self, previous):
"""Note that this answer was looked at and still holds.
A cached result is not re-run, so its tested_at stays at the
run that reached it - which on a page refreshed today reads as
a world nobody has looked at in weeks rather than as a result
nothing has been able to change. This is the date that says
which of the two it is, and only a download failure was
recording it, so it was missing from every row that passed.
"""
previous[StateKeys.LAST_CHECKED_AT] = TestRecord.now()
self.wrote_state = True
def known(self, item):
"""This world's answer already, or None if it must be tested."""
game_name, _, key, current_hash = item
reason = self.blacklist.reason_for(game_name)
if reason:
return self.decline(item, reason)
previous = self.tests.get(key)
cache = CacheCheck(self.settings, previous)
served = not self.force and cache.serves(
current_hash=current_hash,
tag=self.tag,
roms_fingerprint=self.roms_fingerprint,
repeats=self.repeats,
random_repeats=self.random_repeats,
companion_range=self.companion_range,
# A core world has no release of its own here, so this
# is None for one and the rule cannot fire: what dates
# it is the Archipelago build, already compared above.
released_at=self.releases.get(game_name),
)
if not served:
return None
self.confirm(previous)
return self.record_for(item).as_result(previous)
def plan(self, work):
results, to_test = [], []
for item in work:
answer = self.known(item)
if answer is None:
to_test.append(item)
else:
results.append(answer)
return results, to_test

111
src/archipelago_tester/pipeline/batch/record.py
View File

@@ -1,111 +0,0 @@
"""The state entry written for one world."""
from datetime import datetime, timezone
from archipelago_tester.core.model.blacklist import Blacklist
from archipelago_tester.core.state.keys import StateKeys
class TestRecord:
"""What a finished, declined or undownloadable world records.
A core world has no file behind it, so it is marked as such and
carries no apworld path.
"""
def __init__(self, game_name, key=None, is_core=False):
self.game_name = game_name
self.key = key
self.is_core = is_core
@staticmethod
def now():
return datetime.now(timezone.utc).isoformat()
def cored(self, record):
if self.is_core:
record["stage"] = "core"
record["apworld"] = None
return record
def from_report(self, report, current_hash, tag, internal_game,
roms_fingerprint):
"""One finished test.
Single and multi verdicts stay separate: different questions.
Every mode's own verdict is kept too, so adding one needs no
change here. "repaired" is true when the file had to be fixed
before it would load - a real result, but for a corrected copy
rather than the file the author published.
"""
return self.cored({
"hash": current_hash,
"tag": tag,
"game": internal_game,
"outcome": report.get("outcome"),
"detail": report.get("detail"),
"elapsed_seconds": report.get("elapsed_seconds"),
"multi_outcome": report.get("multi_outcome"),
"multi_detail": report.get("multi_detail"),
"multi_elapsed_seconds": report.get("multi_elapsed_seconds"),
"multi_companions": report.get("multi_companions"),
"tests": report.get("tests"),
"roms_fingerprint": roms_fingerprint,
"tested_at": self.now(),
"repaired": bool(report.get("repaired")),
})
def blacklisted(self, blacklist, current_hash, tag, reason,
roms_fingerprint):
"""A world this pipeline declines to test."""
return self.cored({
"hash": current_hash,
"tag": tag,
"game": self.game_name,
"outcome": Blacklist.OUTCOME,
"detail": blacklist.detail(reason),
"elapsed_seconds": None,
"multi_outcome": None,
"multi_detail": None,
"multi_elapsed_seconds": None,
"multi_companions": None,
"tests": None,
"roms_fingerprint": roms_fingerprint,
"tested_at": self.now(),
"blacklisted": True,
})
def download_failure(self, detail, tag):
"""A game whose apworld could not be fetched.
"unknown", never "failed": nothing was tested, so nothing is
known to be broken. Whether the link was a Discord message, an
unsupported host, or a release with no apworld in it, the
conclusion is the same - it could not be checked.
"""
return {
"game": self.game_name,
"apworld": None,
"outcome": "unknown",
"detail": detail,
"elapsed_seconds": None,
"stage": "download",
"tag": tag,
"hash": None,
"tested_at": self.now(),
StateKeys.LAST_CHECKED_AT: self.now(),
}
def as_result(self, record):
"""One entry of the caller-facing result list."""
return {
"game": self.game_name,
"apworld": None if self.is_core else self.key,
"outcome": record.get("outcome"),
"detail": record.get("detail"),
"elapsed_seconds": record.get("elapsed_seconds"),
"multi_outcome": record.get("multi_outcome"),
"multi_detail": record.get("multi_detail"),
"tests": record.get("tests"),
"skipped": True,
}

49
src/archipelago_tester/pipeline/batch/reporter.py
View File

@@ -1,49 +0,0 @@
"""Saying what each finished world did."""
from archipelago_tester.core.display.progress import Progress
from archipelago_tester.core.model.verdict import Verdict
class Reporter:
"""One run's progress callback.
Holds its own bar rather than a module variable: two runs in one
process would otherwise share a bar and a tally.
"""
def __init__(self, config, bar=False):
self.config = config
self.wants_bar = bar
self.bar = None
@staticmethod
def marker(report):
return ("cached" if report.get("skipped")
else f"{report.get('elapsed_seconds')}s")
def verdicts(self, report):
text = f"{report['outcome']}"
multi = report.get("multi_outcome")
return f"{text} / multi:{multi}" if multi else text
def drawn(self, index, total, report):
if self.bar is None or self.bar.total != total:
self.bar = Progress(total=total, config=self.config)
self.bar.update(
index=index,
key=report.get("outcome") or "?",
text=report.get("game") or "",
fallback_line=(f"[{index}/{total}] {report['game']}: "
f"{self.verdicts(report)} ({self.marker(report)})"),
)
def printed(self, index, total, report):
print(f"[{index}/{total}] {report.get('game')}: "
f"{Verdict(report, self.config).name} ({self.marker(report)})",
flush=True)
def __call__(self, index, total, report):
if self.wants_bar:
self.drawn(index, total, report)
else:
self.printed(index, total, report)

69
src/archipelago_tester/pipeline/batch/results.py
View File

@@ -1,69 +0,0 @@
"""The reports one run produced."""
from archipelago_tester.core.model.modes import Modes
from archipelago_tester.core.model.verdict import Verdict
class Results(list):
"""A run's reports, printable as a table.
A plain list of the same dicts the batch returns, so anything that
worked on those still works. The repr is the addition, because the
first thing anyone does with these is look at them.
"""
def __init__(self, reports, config):
super().__init__(reports)
self.config = config
def verdict(self, report):
return Verdict(report, self.config).name
@property
def counts(self):
tally = {}
for report in self:
name = self.verdict(report)
tally[name] = tally.get(name, 0) + 1
return dict(sorted(tally.items(), key=lambda item: -item[1]))
@property
def summary(self):
"""Outcome counts, multi verdicts under their own prefix.
They are a separate question, so they are not merged into one
total.
"""
counts = {}
for report in self:
outcome = report.get("outcome")
counts[outcome] = counts.get(outcome, 0) + 1
multi = report.get("multi_outcome")
if multi:
counts[f"multi:{multi}"] = counts.get(f"multi:{multi}", 0) + 1
return counts
def row(self, report, width):
"""One world's line: name, verdict, every mode's outcome."""
mode_results = report.get("tests") or {}
per_mode = " ".join(
f"{mode}={(mode_results.get(mode) or {}).get('outcome') or '-'}"
for mode in Modes.ALL
)
marker = " (cached)" if report.get("skipped") else ""
return (f"{report.get('game') or '?':{width}s} "
f"{Verdict(report, self.config).label:12s} "
f"{per_mode}{marker}")
def __repr__(self):
if not self:
return "no worlds tested"
width = max(len(report.get("game") or "") for report in self)
lines = [
self.row(report, width)
for report in sorted(self, key=lambda r: r.get("game") or "")
]
if len(self) > 1:
lines += ["", " ".join(f"{name}: {count}"
for name, count in self.counts.items())]
return "\n".join(lines)

86
src/archipelago_tester/pipeline/batch/runner.py
View File

@@ -1,86 +0,0 @@
"""Running the tests the cache could not answer."""
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from archipelago_tester.pipeline.batch.record import TestRecord
from archipelago_tester.pipeline.generation.tester import WorldTester
class Runner:
"""Tests each outstanding world and persists what it did."""
def __init__(self, config, paths, store, options, state, tests, tag,
image, roms_fingerprint):
self.config = config
self.paths = paths
self.store = store
self.options = options
self.state = state
self.tests = tests
self.tag = tag
self.tester = WorldTester(config, paths, image)
self.roms_fingerprint = roms_fingerprint
self.lock = threading.Lock()
@staticmethod
def promotion_name(report, game_name):
"""The name Archipelago itself registered this world under.
Recorded for promotion's collision check, while the report's
own "game" becomes the sheet folder name for display. A world
that never loaded has no real name, so it falls back to a
marked sheet name - otherwise every such entry would collide
under one None key and none would promote.
"""
return report.get("game") or f"unverified:{game_name}"
def store_record(self, key, record):
"""Persist after every test, not just at the end, so a killed
run does not lose the tests it already completed.
"""
with self.lock:
self.tests[key] = record
self.store.save(self.state)
def test_one(self, game_name, apworld_path, key, current_hash):
options = self.options
report = self.tester.test(
apworld_path=apworld_path,
game=None if apworld_path else game_name,
output_dir=options["output_directory"],
timeout=options["timeout"],
companion_range=options["companion_range"],
spoiler=options["spoiler"],
repeats=options["repeats"],
companion_pool=options["pool"],
random_repeats=options["random_repeats"],
)
record = TestRecord(game_name, key,
is_core=apworld_path is None)
self.store_record(key, record.from_report(
report=report,
current_hash=current_hash,
tag=self.tag,
internal_game=self.promotion_name(report, game_name),
roms_fingerprint=self.roms_fingerprint,
))
report["game"] = game_name
report["apworld"] = key
report["skipped"] = False
return report
def run(self, to_test, total, completed, on_progress):
"""Run the outstanding tests, reporting as they land."""
results = []
reporting = threading.Lock()
with ThreadPoolExecutor(max_workers=self.options["jobs"]) as pool:
futures = [pool.submit(self.test_one, *args) for args in to_test]
for future in as_completed(futures):
report = future.result()
results.append(report)
with reporting:
completed += 1
if on_progress:
on_progress(completed, total, report)
return results

95
src/archipelago_tester/pipeline/batch/session.py
View File

@@ -1,95 +0,0 @@
"""Testing worlds by hand, from the notebook or a shell."""
from archipelago_tester.core.config.config import Config
from archipelago_tester.core.config.paths import Paths
from archipelago_tester.core.config.run_settings import RunSettings
from archipelago_tester.core.state.lock import PipelineLock
from archipelago_tester.core.state.scratch import ScratchState
from archipelago_tester.core.state.store import StateStore
from archipelago_tester.pipeline.batch.apworld_finder import ApworldFinder
from archipelago_tester.pipeline.batch.batch import Batch
from archipelago_tester.pipeline.batch.fetcher import Fetcher
from archipelago_tester.pipeline.batch.reporter import Reporter
from archipelago_tester.pipeline.batch.results import Results
from archipelago_tester.pipeline.batch.version import Version
from archipelago_tester.pipeline.build.archipelago import ArchipelagoBuild
class TestSession:
"""Test one world, several, or every downloaded one.
`live` records into the real state file - off by default, because a
hand-run test is an experiment. `cached` only affects a whole-set
run; a named world is always retested.
"""
def __init__(self, game=None, repeats=None, random_repeats=None, jobs=2,
live=False, cached=False, root_directory=None, quiet=False,
update=False):
self.config = Config.load()
self.paths = Paths(self.config)
self.settings = RunSettings(self.config)
self.game = game
self.repeats = repeats
self.random_repeats = random_repeats
self.jobs = jobs
self.live = live
self.cached = cached
self.root = root_directory or self.paths.downloads
self.quiet = quiet
self.update = update
@property
def state_path(self):
return ScratchState(self.paths, live=self.live).path
def worlds(self):
"""The folders to test, downloading any that are missing.
Resolved to real folder names and matched exactly from here on,
so the set tested is the set that was just named.
"""
if self.game is None:
return None
terms = [self.game] if isinstance(self.game, str) else list(self.game)
Fetcher(self.config, self.paths, self.root).ensure(terms)
finder = ApworldFinder(self.root)
return [name for term in terms for name in finder.resolve(term)]
def announce(self, tag, source, state_path, only):
print(f"archipelago {tag} ({source})")
print(f"project {self.paths.project}")
print(f"downloads {self.root}")
scratch = "" if self.live else " (scratch, not the live record)"
print(f"results {state_path}{scratch}")
named = ", ".join(only) if only else "every downloaded apworld"
print(f"testing {named}")
def run(self):
"""The reports, printable as a table."""
store = StateStore(self.paths.state)
build = ArchipelagoBuild(self.config, self.paths)
tag, source = Version(self.config, self.paths, store).prepare(
update=self.update)
only = self.worlds()
build.ensure_test_image(tag)
state_path = self.state_path
if not self.quiet:
self.announce(tag, source, state_path, only)
with PipelineLock(self.paths.lock):
return Results(Batch(
config=self.config,
paths=self.paths,
settings=self.settings,
image=build.test_image_tag(tag),
tag=tag,
force=not self.cached,
on_progress=None if self.quiet else Reporter(self.config),
root_directory=self.root,
state_path=state_path,
only=only,
exact_only=only is not None,
jobs=self.jobs,
repeats=self.repeats,
random_repeats=self.random_repeats,
).run(), self.config)

87
src/archipelago_tester/pipeline/batch/version.py
View File

@@ -1,87 +0,0 @@
"""Which Archipelago version a run tests against."""
from archipelago_tester.core.state.keys import StateKeys
from archipelago_tester.pipeline.build.archipelago import ArchipelagoBuild
from archipelago_tester.pipeline.build.docker import Docker
class Version:
"""Resolve the version, and build it if this machine lacks it."""
def __init__(self, config, paths, store):
self.paths = paths
self.store = store
self.build = ArchipelagoBuild(config, paths)
def resolve(self):
"""The version to test, and where that choice came from.
A pin is a decision already made, so nothing else revisits it.
Failing that the recorded version wins, then whichever image is
on this machine - a test runs entirely inside the container, so
an image already here is everything it needs.
"""
pinned = self.build.pinned_version
if pinned:
return pinned, "pinned in config.json"
recorded = self.store.load().get(StateKeys.ARCHIPELAGO_TAG)
if recorded:
return recorded, "from state.json"
local = next(iter(self.build.local_tags()), None)
return (local, "newest image on this machine") if local else (None,
None)
def ensure_built(self, tag):
"""Build what this version needs, if anything is missing.
Building needs the Archipelago source, which is the one job the
checkout has - so this is also the only path that clones.
"""
if tag is None:
print("no Archipelago image on this machine - cloning the "
"newest release to build one (several minutes) ...")
built, _, _, _ = self.build.update_and_build(self.store)
return built, "just built"
if not Docker.image_exists(self.build.image_tag(tag)):
print(f"Archipelago {tag} is not built on this machine - "
"cloning and building it (several minutes) ...")
self.build.build_version(tag)
return tag, None
def note_others(self, tag, source):
"""Say so when an unpinned run is not using the newest image.
An unpinned tag can come from an image that happens to be lying
around, so a run can quietly answer for a version that is not
the current one. A pinned tag was chosen deliberately.
"""
local = self.build.local_tags()
if self.build.pinned_version or not local or tag == local[0]:
return source
return f"{source}, but {local[0]} is also built here"
def update(self):
"""Resolve the newest release, build it, and record it.
Never automatic. A new version invalidates every cached result,
so the full re-test it forces is the caller's decision, not a
side effect of running. A pinned version stays pinned: the
resolution behind this honours the pin, so an update then only
ensures that version is built.
"""
tag, previous, changed, _ = self.build.update_and_build(self.store)
if self.build.pinned_version:
return tag, "pinned in config.json"
if not changed:
return tag, "already the newest release"
if previous is None:
return tag, "newest release, recorded for the first time"
return tag, f"updated from {previous}"
def prepare(self, update=False):
"""The version to test against, built if it is missing."""
if update:
return self.update()
tag, source = self.resolve()
tag, built = self.ensure_built(tag)
return tag, built or self.note_others(tag, source)

41
src/archipelago_tester/pipeline/batch/work_list.py
View File

@@ -1,41 +0,0 @@
"""Everything one run will consider testing."""
import os
from archipelago_tester.core.model.fingerprint import Fingerprint
from archipelago_tester.pipeline.batch.apworld_finder import ApworldFinder
class WorkList:
"""Each world to consider, with its cache key and hash.
A core world is tested by name with no file behind it, so it is
keyed on the bare sheet name - the same "no slash means no apworld"
convention download failures follow - and gets a stand-in hash
whose real content is the Archipelago build beside it.
"""
#: Stands in for a core world's file hash. Its real content is
#: the Archipelago build, so the version compared beside it in the
#: cache check is what decides staleness.
CORE_HASH = "core"
def __init__(self, root, only=None, exact=False, core_games=()):
self.root = root
self.only = only
self.exact = exact
self.core_games = core_games
def items(self):
found = ApworldFinder(self.root).find(only=self.only,
exact=self.exact)
work = [
(name, path, os.path.relpath(path, self.root),
Fingerprint.of_file(path))
for name, path in found
]
return work + [
(name, None, name, WorkList.CORE_HASH)
for name in self.core_games
if self.only is None or name in self.only
]

0
src/archipelago_tester/pipeline/download/__init__.py
View File

75
src/archipelago_tester/pipeline/download/asset_matcher.py
View File

@@ -1,75 +0,0 @@
"""Choosing which asset in a release belongs to this game."""
import re
from archipelago_tester.core.model.name import Name
class AssetMatcher:
"""Matches release assets and release titles to one sheet row."""
def __init__(self, config, game_name):
self.config = config
self.game = Name(game_name)
def named(self, assets, exact):
"""The assets whose filename names this game."""
matched = []
for asset in assets:
name = Name(asset["name"].rsplit(".", 1)[0]).slug
if (name == self.game.slug) if exact else (name in self.game.slug):
matched.append(asset)
return matched
def select(self, assets):
"""The assets to download for this game."""
if len(assets) <= 1:
return assets
for matches in (self.named(assets, True), self.named(assets, False)):
if len(matches) == 1:
return matches
return assets
def title_slug(self, release):
"""A release's own title, reduced to the game it is for."""
title = release.get("name") or release.get("tag_name") or ""
title = re.sub(r"\bv?\d+(?:\.\d+)*\b", " ", title)
title = re.sub(r"\bapworld\b", " ", title, flags=re.IGNORECASE)
return Name(title).slug
def uninformative(self):
"""Titles that name no game at all."""
return set(self.config.value(
"downloads", "uninformative_release_slugs", ()))
def claimed_by(self, title, sheet_names):
"""The other sheet row this title names, or None.
Exact equality only. This is not "the title looks a bit off" -
it is positive evidence that a DIFFERENT row is named, which is
the only thing strong enough to skip a release the link points
at. "Mega Man X2" is exactly a row and is skipped when resolving
"Mega Man X"; "Mega Man X1" is nobody's row and is accepted.
"""
if not sheet_names or not self.game.text:
return None
if not title or title in self.uninformative():
return None
if title == self.game.slug:
return None
for other in sheet_names:
slug = Name(other).slug
if slug != self.game.slug and slug == title:
return other
return None
def release_claimed(self, release, sheet_names):
return self.claimed_by(self.title_slug(release), sheet_names)
def identity_claimed(self, found_games, sheet_names):
"""Whether a downloaded file identifies as another row's game."""
for found in found_games or []:
other = self.claimed_by(Name(found).slug, sheet_names)
if other:
return other
return None

23
src/archipelago_tester/pipeline/download/auth_error.py
View File

@@ -1,23 +0,0 @@
"""A token failure, which is the whole run's problem."""
import requests
class GitHubAuthError(RuntimeError):
"""A bad or expired token affects every remaining request."""
@classmethod
def matching(cls, game, error):
"""This error as an auth failure, or None if it is not one.
Raised loudly so the whole pool stops, rather than quietly
failing game after game against a token that will not work.
"""
if not isinstance(error, requests.HTTPError):
return None
response = error.response
status = response.status_code if response is not None else None
if status not in (401, 403):
return None
return cls(f"GitHub rejected the request for {game.name} with "
f"{status} - check .env / token expiry.")

61
src/archipelago_tester/pipeline/download/context.py
View File

@@ -1,61 +0,0 @@
"""What every download shares."""
import os
from archipelago_tester.core.model.name import Name
from archipelago_tester.pipeline.download.asset_matcher import AssetMatcher
class DownloadContext:
"""Who is downloading what, and where it goes.
One object rather than the same six arguments threaded through
every handler and helper.
"""
def __init__(self, config, paths, session, game, identifier=None,
sheet_names=None, root_directory=None):
self.config = config
self.paths = paths
self.session = session
self.game = game
self.identifier = identifier
self.sheet_names = sheet_names
self.root = root_directory or paths.downloads
self.matcher = AssetMatcher(config, game.name)
@property
def target_directory(self):
return os.path.join(self.root, Name(self.game.name).directory)
def target_path(self, filename):
return os.path.join(self.target_directory, filename)
def setting(self, key, default=None):
return self.config.value("downloads", key, default)
@property
def plain_headers(self):
"""Headers for a host that is not GitHub.
The session carries a GitHub token, which must never be sent to
an unrelated site.
"""
return {"Authorization": None, "Accept": "*/*"}
@property
def json_headers(self):
"""Headers for another forge's JSON API, without the token."""
return {"Authorization": None, "Accept": "application/json"}
def wrong_game(self, path):
"""Why this file is another game's, or None to keep it."""
if self.identifier is None:
return None
identity = self.identifier.identify(path)
if not identity.get("ran"):
return None
found = identity.get("games") or []
if any(Name(self.game.name).matches(name) for name in found):
return None
return found or identity.get("detail")

156
src/archipelago_tester/pipeline/download/download_run.py
View File

@@ -1,156 +0,0 @@
"""Fetching every game's apworld, concurrently."""
import os
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests
from archipelago_tester.core.state.keys import StateKeys
from archipelago_tester.core.state.store import StateStore
from archipelago_tester.pipeline.download.auth_error import GitHubAuthError
from archipelago_tester.pipeline.download.context import DownloadContext
from archipelago_tester.pipeline.download.downloader import Downloader
from archipelago_tester.pipeline.download.superseded_assets import (
SupersededAssets,
)
class DownloadRun:
"""One pass over the sheet's rows.
`sheet_names` should be the FULL sheet: a file can identify as a
core row, which never reaches here.
"""
def __init__(self, config, paths, session, root_directory=None,
identifier=None, jobs=None, sheet_names=None,
previous_releases=None):
self.config = config
self.paths = paths
self.session = session
self.root = root_directory or paths.downloads
self.identifier = identifier
self.jobs = jobs or int(config.value("downloads", "jobs", 8))
self.sheet_names = sheet_names
self.previous_releases = previous_releases or {}
@staticmethod
def split(games):
"""The games worth a network call, and those answered now.
A row with no link has nothing to look up, so it never takes a
slot in the thread pool.
"""
downloadable, skipped = [], []
for game in games:
if not game.release and not game.links:
skipped.append((game.name, "no release link in the sheet"))
else:
downloadable.append(game)
return downloadable, skipped
def context_for(self, game):
return DownloadContext(
config=self.config,
paths=self.paths,
session=self.session,
game=game,
identifier=self.identifier,
sheet_names=self.sheet_names,
root_directory=self.root,
)
def one(self, game):
"""Download one game, which never raises for that game alone.
A network failure is that game's own result, not the run's -
except an auth failure, which is everyone's.
"""
try:
downloaded, reason, info = Downloader(
context=self.context_for(game),
previous=self.previous_releases.get(game.name),
).run()
except requests.RequestException as error:
auth = GitHubAuthError.matching(game, error)
if auth is not None:
raise auth from error
return game, None, str(error), None
return game, downloaded, reason, info
@staticmethod
def source_url(game):
"""Where this apworld came from, for the status page.
game.release is only ever a real releases link, so for anything
resolved through the plain links the first one stands in.
"""
return game.release or (game.links[0] if game.links else None)
def record(self, finished, releases, skipped, completed, total,
on_asset, on_game):
"""Record one finished game and tell the caller about it.
The release is recorded even on failure - it was still found,
so its version is worth showing. on_game is called for failures
too, unlike on_asset: a progress bar has to count every game
that finished, not only the ones that produced a file.
"""
game, downloaded, reason, info = finished
releases[game.name] = {"url": self.source_url(game), **(info or {})}
if downloaded is None:
skipped.append((game.name, reason))
if on_game:
on_game(completed, total, game.name, False)
return 0
paths = [path for path, _ in downloaded]
for path in SupersededAssets(os.path.dirname(paths[0]),
paths).prune():
print(f"{game.name}: removed superseded {os.path.basename(path)}"
f" - the release no longer offers it")
for path, status in downloaded:
if on_asset:
on_asset(game.name, path, status)
if on_game:
on_game(completed, total, game.name, True)
return len(downloaded)
@staticmethod
def result_of(future, executor):
"""One finished download, stopping the pool on auth failure."""
try:
return future.result()
except GitHubAuthError:
executor.shutdown(cancel_futures=True)
raise
def all(self, games, on_asset=None, on_game=None):
"""Every game's apworld: how many files, what was skipped, and
which release each row resolved to.
Recording holds a lock, so on_asset always runs serialized and
a caller's own progress counter needs none of its own.
"""
self.sheet_names = self.sheet_names or [g.name for g in games]
downloadable, skipped = self.split(games)
releases, lock = {}, threading.Lock()
total = completed = 0
with ThreadPoolExecutor(max_workers=self.jobs) as executor:
futures = [executor.submit(self.one, g) for g in downloadable]
for future in as_completed(futures):
finished = self.result_of(future, executor)
with lock:
completed += 1
total += self.record(
finished, releases, skipped, completed,
len(downloadable), on_asset, on_game)
return total, skipped, releases
def record_releases(self, releases, state_path=None):
"""Write each game's resolved release into the state file."""
store = StateStore(state_path or self.paths.state)
state = store.load()
state.setdefault(StateKeys.GAME_RELEASES, {}).update(releases)
store.save(state)
return state

75
src/archipelago_tester/pipeline/download/downloader.py
View File

@@ -1,75 +0,0 @@
"""Fetching one sheet row's apworld."""
from archipelago_tester.pipeline.download.handlers.gitea_release import (
GiteaRelease,
)
from archipelago_tester.pipeline.download.handlers.github_file import (
GitHubFile,
)
from archipelago_tester.pipeline.download.handlers.github_release import (
GitHubRelease,
)
from archipelago_tester.pipeline.download.handlers.github_repo import (
GitHubRepo,
)
from archipelago_tester.pipeline.download.handlers.gitlab_file import (
GitLabFile,
)
from archipelago_tester.pipeline.download.handlers.gitlab_release import (
GitLabRelease,
)
from archipelago_tester.pipeline.download.handlers.plain_file import PlainFile
from archipelago_tester.pipeline.download.unusable_links import UnusableLinks
from archipelago_tester.pipeline.download.url_shapes import UrlShapes
class Downloader:
"""Whatever shape this game's link is, get the apworld.
A releases link is resolved directly; anything else is tried
against every handler in turn, most specific first, so an exact
file beats browsing a whole repo and the plain-URL handler can
never shadow a forge-specific one.
"""
def __init__(self, context, previous=None):
self.context = context
self.shapes = UrlShapes(context.config)
self.previous = previous
def handlers(self):
return (
GitHubFile(self.shapes),
GitHubRepo(self.shapes),
GitLabFile(self.shapes),
GitLabRelease(self.shapes),
GiteaRelease(self.shapes, previous=self.previous),
PlainFile(self.shapes),
)
def from_release_link(self):
handler = GitHubRelease(self.shapes, previous=self.previous)
parsed = handler.parse(self.context.game.release)
if parsed is None:
return None, "not a recognized releases URL", None
return handler.download(parsed, self.context)
def from_link(self, link):
"""This link's result, or None if no handler recognised it."""
for handler in self.handlers():
parsed = handler.parse(link)
if parsed is not None:
return handler.download(parsed, self.context)
return None
def run(self):
"""The files downloaded, why none were, and the release info."""
game = self.context.game
if game.release:
return self.from_release_link()
for link in game.links:
result = self.from_link(link)
if result is not None:
return result
return None, UnusableLinks(self.context.config, game.links).detail(), \
None

0
src/archipelago_tester/pipeline/download/handlers/__init__.py
View File

45
src/archipelago_tester/pipeline/download/handlers/base.py
View File

@@ -1,45 +0,0 @@
"""What every link handler shares."""
import os
from archipelago_tester.pipeline.download.local_file import LocalFile
class LinkHandler:
"""One recognised shape of sheet link, and how to fetch from it.
parse() returning None means "not my shape", which is different from
"mine, and it failed": the caller only moves on to the next link in
the first case.
"""
def __init__(self, shapes):
self.shapes = shapes
def parse(self, link):
raise NotImplementedError
def download(self, parsed, context):
raise NotImplementedError
def file(self, context, filename):
return LocalFile(context.session, context.target_path(filename))
def collect(self, candidates, context, fetch, verify):
"""Download each candidate, dropping any that is another game."""
downloaded, kept, rejected = [], [], []
for candidate in candidates:
path, status = fetch(candidate)
if status in LocalFile.NEEDS_DOWNLOAD and verify:
mismatch = context.wrong_game(path)
if mismatch is not None:
os.remove(path)
rejected.append((candidate["name"], mismatch))
continue
downloaded.append((path, status))
kept.append(candidate)
return downloaded, kept, rejected
def rejected_detail(self, rejected, subject):
details = "; ".join(f"{name} -> {found}" for name, found in rejected)
return f"downloaded {subject} didn't match this game: {details}"

44
src/archipelago_tester/pipeline/download/handlers/gitea_release.py
View File

@@ -1,44 +0,0 @@
"""A releases page on a self-hosted Gitea or Forgejo instance."""
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
from archipelago_tester.pipeline.download.release_downloader import (
ReleaseDownloader,
)
class GiteaRelease(LinkHandler):
"""Codeberg and friends.
The release JSON is close enough to GitHub's that ReleaseDownloader
handles it unchanged, given headers that omit the GitHub token.
"""
def __init__(self, shapes, previous=None):
super().__init__(shapes)
self.previous = previous
def parse(self, link):
return self.shapes.gitea_releases(link)
def latest(self, context, host, owner, repository):
response = context.session.get(
f"https://{host}/api/v1/repos/{owner}/{repository}"
"/releases/latest",
headers=context.json_headers,
)
if response.status_code == 404:
return None
response.raise_for_status()
return response.json()
def download(self, parsed, context):
host, owner, repository = parsed
release = self.latest(context, host, owner, repository)
if release is None:
return None, "no release found on this host", None
return ReleaseDownloader(
context=context,
release=release,
headers=context.plain_headers,
previous_published_at=(self.previous or {}).get("published_at"),
).run()

58
src/archipelago_tester/pipeline/download/handlers/github_file.py
View File

@@ -1,58 +0,0 @@
"""A link straight at an apworld committed in a GitHub repo."""
import os
import urllib.parse
import requests
from archipelago_tester.pipeline.build.archipelago import (
ArchipelagoBuild,
)
from archipelago_tester.pipeline.download.local_file import LocalFile
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
class GitHubFile(LinkHandler):
"""A committed file, named exactly by the sheet.
No identity check: the link names one file, so there is no second
candidate a name comparison could choose between.
"""
def parse(self, link):
return self.shapes.github_file(link)
def download(self, parsed, context):
"""Fetch the file as it currently stands on that branch."""
owner, repository, branch, path = parsed
filename = os.path.basename(urllib.parse.unquote(path))
local = self.file(context, filename)
url = ("https://raw.githubusercontent.com/"
f"{owner}/{repository}/{branch}/{path}")
status = local.status(local.remote_size(url))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url)
published = self.published_at(context, owner, repository, path, branch)
return [(local.path, status)], None, {"published_at": published}
def published_at(self, context, owner, repository, path, branch):
"""When this file was last committed, or None.
Best effort: a blob link has no release to read a date from, and
a download should not fail over display information.
"""
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
"/commits",
params={"path": path, "sha": branch, "per_page": 1},
)
response.raise_for_status()
commits = response.json()
except requests.RequestException:
return None
if not commits:
return None
commit = commits[0].get("commit", {})
return ((commit.get("committer") or {}).get("date")
or (commit.get("author") or {}).get("date"))

81
src/archipelago_tester/pipeline/download/handlers/github_release.py
View File

@@ -1,81 +0,0 @@
"""A GitHub releases page, the sheet's usual link."""
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
from archipelago_tester.pipeline.download.release_downloader import (
ReleaseDownloader,
)
from archipelago_tester.pipeline.download.release_identity import (
ReleaseIdentity,
)
from archipelago_tester.pipeline.download.release_picker import ReleasePicker
from archipelago_tester.pipeline.download.walk_back import WalkBack
class GitHubRelease(LinkHandler):
"""Resolve which release the link means, then fetch its apworld."""
def __init__(self, shapes, previous=None):
super().__init__(shapes)
self.previous = previous
def parse(self, link):
return self.shapes.releases(link)
def resolve(self, parsed, context):
"""The release this link resolves to, or None.
A hand-pinned tag prefix wins, then a link naming one exact
tag, then the search term. Runs every pass: it is the one call
that can report a new release, which is the only thing able to
change a previously failed answer.
"""
owner, repository, term, tag = parsed
picker = ReleasePicker(context, owner, repository, term)
overrides = context.config.get("release_overrides") or {}
prefix = overrides.get(context.game.name)
if prefix:
return picker.by_tag_prefix(prefix)
if tag:
return picker.by_tag(tag)
return picker.pick()
def fetch(self, parsed, context, release):
"""This release's apworld, walking back if it ships none.
Only that one cause is walked back from. "Ships another game's
apworld" must not be, since an older release of a multi-world
repo is precisely the wrong answer there.
"""
owner, repository = parsed[0], parsed[1]
downloader = ReleaseDownloader(
context=context,
release=release,
# What this row's date already says, so a re-upload of
# unchanged bytes keeps it instead of moving it forward.
previous_published_at=(self.previous or {}).get("published_at"),
)
downloaded, reason, info = downloader.run()
if (downloaded is not None or context.identifier is None
or downloader.cause != ReleaseDownloader.NO_APWORLD):
return downloaded, reason, info
fallback, reason = WalkBack(
context=context,
owner=owner,
repository=repository,
release=release,
).run(reason)
return fallback if fallback is not None else (None, reason, info)
def download(self, parsed, context):
release = self.resolve(parsed, context)
if release is None:
return None, "no release matched the search term in the link", None
identity = ReleaseIdentity(release)
repeat = identity.repeated(self.previous)
if repeat is not None:
return None, repeat, identity.failure(
{"published_at": None}, repeat)
downloaded, reason, info = self.fetch(parsed, context, release)
if downloaded is None:
return None, reason, identity.failure(info, reason)
return downloaded, reason, info

115
src/archipelago_tester/pipeline/download/handlers/github_repo.py
View File

@@ -1,115 +0,0 @@
"""A bare repository link, for a world never cut into a release."""
import os
import requests
from archipelago_tester.pipeline.build.archipelago import (
ArchipelagoBuild,
)
from archipelago_tester.pipeline.download.local_file import LocalFile
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
class GitHubRepo(LinkHandler):
"""Browses the repo's own file tree for an apworld."""
def parse(self, link):
return self.shapes.github_repo(link)
def resolve_branch(self, context, owner, repository, branch):
"""The real default branch, so a version reads as a name.
"HEAD" works as a ref, but resolving it means the status page
shows a branch rather than the literal string.
"""
if branch != "HEAD":
return branch
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}")
response.raise_for_status()
return response.json().get("default_branch") or "HEAD"
except requests.RequestException:
return "HEAD"
def tree_entries(self, context, owner, repository, branch):
"""Every apworld in the tree, with its size.
One call: the trees API takes recursive=1 itself, and returns
each blob's size for free.
"""
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
f"/git/trees/{branch}",
params={"recursive": "1"},
)
response.raise_for_status()
except requests.RequestException:
return []
return [
(entry["path"], entry.get("size"))
for entry in response.json().get("tree", [])
if entry.get("type") == "blob"
and entry["path"].lower().endswith(ApworldFile.SUFFIX)
]
def candidates(self, entries):
return [
{"name": os.path.basename(path), "path": path, "size": size}
for path, size in entries
]
def download(self, parsed, context):
owner, repository, branch = parsed
branch = self.resolve_branch(context, owner, repository, branch)
entries = self.tree_entries(context, owner, repository, branch)
if not entries:
return None, ("repo link has no release and no .apworld file "
"found in it"), None
chosen = context.matcher.select(self.candidates(entries))
downloaded, kept, rejected = self.collect(
candidates=chosen,
context=context,
fetch=lambda candidate: self.fetch(
candidate, context, owner, repository, branch),
verify=len(chosen) > 1,
)
if not downloaded:
detail = self.rejected_detail(rejected, "file(s) from repo")
return None, detail, None
published = self.published_at(
context, owner, repository, kept[0]["path"], branch)
return downloaded, None, {"published_at": published}
def fetch(self, candidate, context, owner, repository, branch):
local = self.file(context, candidate["name"])
status = local.status(candidate.get("size"))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(
f"https://raw.githubusercontent.com/{owner}/{repository}"
f"/{branch}/{candidate['path']}")
return local.path, status
def published_at(self, context, owner, repository, path, branch):
"""When the first kept file was last committed.
Only the first: one API call per candidate otherwise.
"""
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
"/commits",
params={"path": path, "sha": branch, "per_page": 1},
)
response.raise_for_status()
commits = response.json()
except requests.RequestException:
return None
if not commits:
return None
commit = commits[0].get("commit", {})
return ((commit.get("committer") or {}).get("date")
or (commit.get("author") or {}).get("date"))

44
src/archipelago_tester/pipeline/download/handlers/gitlab_file.py
View File

@@ -1,44 +0,0 @@
"""A link straight at an apworld committed in a GitLab repo."""
import os
import urllib.parse
import requests
from archipelago_tester.pipeline.download.local_file import LocalFile
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
class GitLabFile(LinkHandler):
"""A committed file on GitLab, named exactly by the sheet."""
def parse(self, link):
return self.shapes.gitlab_file(link)
def download(self, parsed, context):
project_path, branch, path = parsed
filename = os.path.basename(urllib.parse.unquote(path))
local = self.file(context, filename)
url = f"https://gitlab.com/{project_path}/-/raw/{branch}/{path}"
headers = context.json_headers
status = local.status(local.remote_size(url, headers=headers))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url, headers=headers)
published = self.published_at(context, project_path, path, branch)
return [(local.path, status)], None, {"published_at": published}
def published_at(self, context, project_path, path, branch):
"""When this file was last committed, or None."""
encoded = urllib.parse.quote(project_path, safe="")
try:
response = context.session.get(
f"https://gitlab.com/api/v4/projects/{encoded}"
"/repository/commits",
params={"path": path, "ref_name": branch, "per_page": 1},
headers=context.json_headers,
)
response.raise_for_status()
commits = response.json()
except requests.RequestException:
return None
return commits[0].get("committed_date") if commits else None

74
src/archipelago_tester/pipeline/download/handlers/gitlab_release.py
View File

@@ -1,74 +0,0 @@
"""A GitLab releases page."""
import urllib.parse
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
from archipelago_tester.pipeline.download.local_file import LocalFile
class GitLabRelease(LinkHandler):
"""The newest release of a GitLab project.
GitLab's assets are shaped nothing like GitHub's - a release-level
list of {name, url, direct_asset_url} with no per-asset timestamp -
so they are reshaped here rather than reusing the release path.
"""
def parse(self, link):
return self.shapes.gitlab_releases(link)
def latest(self, context, project_path):
"""GitLab's own newest-release shortcut, or None."""
encoded = urllib.parse.quote(project_path, safe="")
response = context.session.get(
f"https://gitlab.com/api/v4/projects/{encoded}"
"/releases/permalink/latest",
headers=context.json_headers,
)
if response.status_code == 404:
return None
response.raise_for_status()
return response.json()
def assets(self, release, context):
"""The release's apworld links, in GitHub's asset shape."""
links = (release.get("assets") or {}).get("links", [])
return [
{
"name": link["name"],
"browser_download_url": (link.get("direct_asset_url")
or link["url"]),
}
for link in links
if (link.get("name") or "").lower().endswith(ApworldFile.SUFFIX)
]
def fetch(self, asset, context):
local = self.file(context, asset["name"])
url = asset["browser_download_url"]
headers = context.plain_headers
status = local.status(local.remote_size(url, headers=headers))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url, headers=headers)
return local.path, status
def download(self, parsed, context):
release = self.latest(context, parsed)
if release is None:
return None, "no GitLab release found for this project", None
info = {"published_at": (release.get("released_at")
or release.get("created_at"))}
assets = self.assets(release, context)
if not assets:
return None, "latest GitLab release has no .apworld asset", info
chosen = context.matcher.select(assets)
downloaded, _, rejected = self.collect(
candidates=chosen,
context=context,
fetch=lambda asset: self.fetch(asset, context),
verify=len(chosen) > 1,
)
if not downloaded:
return None, self.rejected_detail(rejected, "asset(s)"), info
return downloaded, None, info

48
src/archipelago_tester/pipeline/download/handlers/plain_file.py
View File

@@ -1,48 +0,0 @@
"""A plain URL ending in .apworld, on any host."""
import email.utils
import requests
from archipelago_tester.pipeline.download.local_file import LocalFile
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
class PlainFile(LinkHandler):
"""Any other host. Tried last, so a forge handler always wins."""
def parse(self, link):
return self.shapes.plain_file(link)
def download(self, parsed, context):
url, filename = parsed
local = self.file(context, filename)
size, published = self.remote_facts(context, url)
status = local.status(size)
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url, headers=context.plain_headers)
return [(local.path, status)], None, {"published_at": published}
def remote_facts(self, context, url):
"""The size and date one HEAD request can offer.
A plain URL has no version to compare, so size is the whole
staleness check. Neither is required.
"""
try:
response = context.session.head(
url, headers=context.plain_headers, allow_redirects=True)
response.raise_for_status()
except requests.RequestException:
return None, None
length = response.headers.get("Content-Length")
size = int(length) if length and length.isdigit() else None
return size, self.modified_at(response.headers.get("Last-Modified"))
def modified_at(self, modified):
if not modified:
return None
try:
return email.utils.parsedate_to_datetime(modified).isoformat()
except (TypeError, ValueError):
return None

83
src/archipelago_tester/pipeline/download/local_file.py
View File

@@ -1,83 +0,0 @@
"""Deciding whether a local copy is still the current one."""
import datetime
import os
import requests
from archipelago_tester.core.model.instant import Instant
class LocalFile:
"""One file on disk, against what the remote currently offers.
"It exists" is not "it is current": an author can replace an
asset's content in place, same filename, same version. Two signals
catch that - the asset's own timestamp, and its size - and both are
needed. A replacement that happens to be the same length is
invisible to size alone, and a remote that reports no length at all
makes size no signal whatsoever.
"""
#: The statuses that mean the file has to be fetched. The other
#: two - "verified" and "assumed" - mean the copy on disk stands.
NEEDS_DOWNLOAD = ("missing", "stale")
def __init__(self, session, path):
self.session = session
self.path = path
def outdated(self, remote_updated_at):
"""Whether the remote was touched after this copy arrived.
mtime is when download() wrote this file, so a remote timestamp
later than it means the copy on disk predates what is being
offered now. Without this, a release that replaced its asset
with one of the same length was never fetched, and the recorded
test result went on describing content that no longer exists -
while the page showed the new release's date beside it.
"""
stamp = Instant.of(remote_updated_at)
if stamp is None:
return False
arrived = datetime.datetime.fromtimestamp(
os.path.getmtime(self.path), datetime.timezone.utc)
return arrived < stamp
def status(self, remote_size, remote_updated_at=None):
"""Whether this file is missing, stale, verified or assumed.
The timestamp is checked before the size, and before a missing
size falls through to "assumed": it is the stronger signal, and
the two cases it catches are exactly the ones size cannot.
"""
if not os.path.exists(self.path):
return "missing"
if self.outdated(remote_updated_at):
return "stale"
if remote_size is None:
return "assumed"
if os.path.getsize(self.path) == remote_size:
return "verified"
return "stale"
def remote_size(self, url, headers=None):
"""The size a HEAD request reports, or None."""
try:
response = self.session.head(url, headers=headers,
allow_redirects=True)
response.raise_for_status()
except requests.RequestException:
return None
length = response.headers.get("Content-Length")
return int(length) if length and length.isdigit() else None
def download(self, url, headers=None):
"""Stream a URL into this path, creating its directory."""
response = self.session.get(url, stream=True, headers=headers)
response.raise_for_status()
os.makedirs(os.path.dirname(self.path), exist_ok=True)
with open(self.path, "wb") as handle:
for chunk in response.iter_content(chunk_size=65536):
handle.write(chunk)
return self.path

179
src/archipelago_tester/pipeline/download/release_downloader.py
View File

@@ -1,179 +0,0 @@
"""Fetching the apworld a release carries."""
import os
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.core.model.fingerprint import Fingerprint
from archipelago_tester.pipeline.download.local_file import LocalFile
from archipelago_tester.pipeline.download.zip_search import ZipSearch
class ReleaseDownloader:
"""A GitHub-shaped release, and the apworld it ships.
Shared with Gitea and Forgejo, whose release JSON is close enough
that the same logic applies unchanged.
"""
#: Why run() came back empty. The caller decides what to do about
#: it from this, never from the message beside it: a release that
#: ships no apworld can be walked back from, and one that ships
#: another game's must not be.
NO_APWORLD = "no_apworld"
WRONG_GAME = "wrong_game"
NO_APWORLD_DETAIL = "latest release has no .apworld asset"
def __init__(self, context, release, headers=None,
require_identity=False, previous_published_at=None):
self.context = context
self.release = release
self.headers = headers
self.require_identity = require_identity
self.previous_published_at = previous_published_at
self.cause = None
def named_assets(self):
return [
asset for asset in self.release.get("assets", [])
if asset["name"].lower().endswith(ApworldFile.SUFFIX)
]
def missing_detail(self):
detail = self.NO_APWORLD_DETAIL
if any(asset["name"].lower().endswith(ZipSearch.SUFFIX)
for asset in self.release.get("assets", [])):
detail += " (checked inside its .zip asset(s) too, none found)"
return detail
def assets(self):
"""Every apworld this release offers, zips included."""
found = self.named_assets()
if found:
return found
return ZipSearch(self.context, self.release, self.headers).assets()
def verifies(self, assets):
"""Whether a file has to prove which game it is.
With one candidate there is nothing to choose between, and the
sheet's link points at this release - so that file is the row's
apworld whatever its World class calls itself. A walk-back
release overrides that, since the link no longer points at it.
"""
return len(assets) > 1 or self.require_identity
@staticmethod
def touched_at(asset):
"""When this asset was last written, as the remote reports it.
The same value published_at() reports for the row, read here so
the two cannot disagree: a file whose recorded release date is
newer than the copy on disk is a file that was never fetched.
"""
return asset.get("updated_at") or asset.get("created_at")
def fetch(self, asset):
"""The asset's path, how it got there, and whether it changed.
An asset whose timestamp moved is fetched again - the copy on
disk can no longer be trusted to be current - and only then can
the two be compared. Identical bytes mean the author re-uploaded
the same file, so nothing about this row actually changed. A
file lifted out of a zip is taken as changed: ZipSearch does its
own fetching, and there is no earlier copy here to compare to.
"""
if "extracted_path" in asset:
return asset["extracted_path"], asset["status"], True
local = LocalFile(self.context.session,
self.context.target_path(asset["name"]))
url = asset["browser_download_url"]
status = local.status(
local.remote_size(url, headers=self.headers),
remote_updated_at=self.touched_at(asset),
)
if status not in LocalFile.NEEDS_DOWNLOAD:
return local.path, status, False
before = (Fingerprint.of_file(local.path)
if os.path.exists(local.path) else None)
local.download(url, headers=self.headers)
return local.path, status, Fingerprint.of_file(local.path) != before
def rejection(self, path, assets):
"""Why this file is not ours, or None to keep it.
A walk-back file that registered no world at all is kept:
nothing was proven either way, and generation testing reports
the import failure far more usefully than a silent drop.
"""
identity = self.context.identifier.identify(path)
if not identity.get("ran"):
return None
found = identity.get("games") or []
if self.require_identity and not found and len(assets) == 1:
return None
other = self.context.matcher.identity_claimed(
found, self.context.sheet_names)
matches = bool(found) and self.context.matcher.game.matches(
found[0])
if other:
return f'{found} - belongs to the "{other}" row'
if self.verifies(assets) and not matches:
return found or identity.get("detail")
return None
def checks(self, status, assets):
return (status in LocalFile.NEEDS_DOWNLOAD
and self.context.identifier is not None
and (self.verifies(assets) or self.context.sheet_names))
def published_at(self, kept):
"""When the content this row now holds was published.
Each asset carries its own timestamp; a release's own stays
fixed at creation even when its files are replaced later, so
the assets' is the one to read. But that timestamp moves for a
re-upload of identical bytes too, and then it dates the upload
rather than the world: the page would show a release newer than
the test beside it while the tested file never changed, and the
Compatibility column would call a world current on the strength
of an upload that added nothing. So only an asset whose content
actually changed sets this date, and otherwise the one already
recorded stands. With nothing recorded there is nothing to
hold, and the asset's own timestamp is the best available.
"""
stamps = [(self.touched_at(asset), changed)
for asset, changed in kept]
stamps = [pair for pair in stamps if pair[0]]
replaced = [stamp for stamp, changed in stamps if changed]
if replaced:
return max(replaced)
if self.previous_published_at:
return self.previous_published_at
return max((stamp for stamp, _ in stamps), default=None)
def run(self):
"""The files downloaded, why none were, and the release info."""
assets = self.assets()
if not assets:
self.cause = self.NO_APWORLD
return None, self.missing_detail(), {"published_at": None}
assets = self.context.matcher.select(assets)
downloaded, kept, rejected = [], [], []
for asset in assets:
path, status, changed = self.fetch(asset)
reason = (self.rejection(path, assets)
if self.checks(status, assets) else None)
if reason is not None:
os.remove(path)
rejected.append((asset["name"], reason))
continue
downloaded.append((path, status))
kept.append((asset, changed))
if not downloaded:
self.cause = self.WRONG_GAME
details = "; ".join(f"{name} -> {found}"
for name, found in rejected)
return None, (f"downloaded asset(s) didn't match this game: "
f"{details}"), {"published_at": None}
return downloaded, None, {"published_at": self.published_at(kept)}

55
src/archipelago_tester/pipeline/download/release_identity.py
View File

@@ -1,55 +0,0 @@
"""Remembering which release a failure was decided on."""
from archipelago_tester.core.state.keys import StateKeys
class ReleaseIdentity:
"""What makes a resolved release the same one as last run.
Its tag and publication date, plus every asset's name and size. The
sizes matter because an author can replace an asset in place
without cutting a new tag.
"""
def __init__(self, release):
self.release = release
@property
def value(self):
assets = sorted(
(asset.get("name"), asset.get("size"))
for asset in self.release.get("assets") or []
)
return {
"tag": self.release.get("tag_name"),
"published_at": self.release.get("published_at"),
"assets": [{"name": name, "size": size} for name, size in assets],
}
def repeated(self, previous):
"""The recorded reason, if this exact release failed before.
A release that ships no apworld, or only another row's, fails
the same way every run - and finding that out costs a download
of every zip asset plus a container run to identify what came
back. None of that can reach a different answer while the
release is unchanged. Keyed on the release rather than the
link, because the link is usually /releases/latest and a newly
published release is exactly what would fix a world in this
state. Only ever holds a verdict about content: a network error
escapes as an exception and a container that could not start
keeps the file, so neither is cached here.
"""
if not previous:
return None
if previous.get(StateKeys.FAILED_RELEASE) != self.value:
return None
return previous.get(StateKeys.FAILED_REASON) or None
def failure(self, info, reason):
"""A failure recorded against the release it was decided on."""
return {
**(info or {}),
StateKeys.FAILED_RELEASE: self.value,
StateKeys.FAILED_REASON: reason,
}

151
src/archipelago_tester/pipeline/download/release_picker.py
View File

@@ -1,151 +0,0 @@
"""Which release a sheet link means."""
import re
import urllib.parse
from archipelago_tester.pipeline.build.archipelago import (
ArchipelagoBuild,
)
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.core.model.name import Name
class ReleasePicker:
"""One repository's releases, narrowed to this game's.
A repo dedicated to one world can answer with its latest release. A
repo hosting several has to be matched on the link's search term and
the release's own title, and must never answer with a release that
another sheet row already claims.
"""
def __init__(self, context, owner, repository, term=None):
self.context = context
self.owner = owner
self.repository = repository
self.term = term
self.matcher = context.matcher
@property
def repo_url(self):
return (f"{ArchipelagoBuild.API_ROOT}/repos/"
f"{self.owner}/{self.repository}")
def all(self, max_pages=10):
"""Every release, newest first, across every page.
GitHub pages at 30 by default and several source repos hold
more than that, which would silently hide real releases from
every lookup below.
"""
releases = []
for page in range(1, max_pages + 1):
response = self.context.session.get(
f"{self.repo_url}/releases",
params={"per_page": 100, "page": page},
)
response.raise_for_status()
batch = response.json()
releases.extend(batch)
if len(batch) < 100:
break
return releases
def by_tag(self, tag):
"""The release for exactly this tag, or None."""
encoded = urllib.parse.quote(tag, safe="")
response = self.context.session.get(
f"{self.repo_url}/releases/tags/{encoded}")
if response.status_code == 404:
return None
response.raise_for_status()
return response.json()
def by_tag_prefix(self, prefix):
"""The newest release whose tag starts with this prefix."""
for release in self.all():
if (release.get("tag_name") or "").startswith(prefix):
return release
return None
def claimed(self, release):
return self.matcher.release_claimed(release, self.context.sheet_names)
def newest(self):
"""GitHub's own newest full release.
Right whenever it resolves, since it skips in-progress
prereleases. The list's first entry is the fallback for a repo
where every release is flagged prerelease, which /latest
excludes rather than relaxing.
"""
response = self.context.session.get(f"{self.repo_url}/releases/latest")
if response.status_code != 404:
response.raise_for_status()
return response.json()
releases = self.all()
return releases[0] if releases else None
def titled_for_this_row(self):
for release in self.all():
if self.matcher.title_slug(release) == self.matcher.game.slug:
return release
return None
def without_a_term(self):
"""The release to use when the link named none.
"Newest in the repo" is a fine answer for a repo holding one
world and a coin flip for one holding twelve, so a newest that
is titled for somebody else's row gives way to the one titled
for this row.
"""
latest = self.newest()
if latest is None or not self.claimed(latest):
return latest
return self.titled_for_this_row() or latest
def by_title(self, releases):
"""The first release whose title holds every word of the term.
The sheet's term is built for GitHub's own search box, which
matches per word: requiring the whole phrase rejects real terms
like "Jurassic Park Randomizer (SNES) Jurassic Park", where the
link generator repeats the game name. That looseness is what
lets a term land on a sibling world, so a release titled for
another row is skipped and the search continues.
"""
words = re.findall(r"[a-z0-9]+", self.term.lower())
if not words:
return None
for release in releases:
title = (f"{release.get('name') or ''} "
f"{release.get('tag_name') or ''}").lower()
if all(word in title for word in words) \
and not self.claimed(release):
return release
return None
def by_asset_name(self, releases):
"""The release carrying an asset named for this game.
A fork mirroring many worlds often tags every release by build
date alone, so the game's name appears only in which apworlds
the release bundles.
"""
term = Name(self.term).slug
for release in releases:
for asset in release.get("assets", []):
name = asset["name"]
if not name.lower().endswith(ApworldFile.SUFFIX):
continue
if Name(name.rsplit(".", 1)[0]).slug == term:
return release
return None
def pick(self):
"""The release this link means, or None."""
if self.term is None:
return self.without_a_term()
releases = self.all()
return self.by_title(releases) or self.by_asset_name(releases)

58
src/archipelago_tester/pipeline/download/superseded_assets.py
View File

@@ -1,58 +0,0 @@
"""Clearing apworlds a game's release no longer offers."""
import os
class SupersededAssets:
"""The files left behind when an author renames their asset.
A download writes each asset under its own filename, so a release
that renamed one leaves the previous name sitting beside it. Both
are then real files in the game's folder, and everything downstream
keys on the path rather than the game - two work items, two test
records, two rows on the status page, and a promotion that refuses
both because they claim one game name between them.
Only ever run for a game whose download succeeded: with nothing
fetched there is no current set to compare against, and pruning on
a failed network call would delete the copy that still works.
"""
#: What this class considers. A game's folder holds nothing else,
#: but naming it here keeps a stray file from being deleted on the
#: strength of living in the wrong directory.
SUFFIX = ".apworld"
def __init__(self, directory, kept):
self.directory = directory
self.kept = {os.path.abspath(path) for path in kept}
def stale(self):
"""Every apworld in the folder that this download did not write."""
if not self.kept or not os.path.isdir(self.directory):
return []
found = []
for entry in sorted(os.listdir(self.directory)):
if not entry.lower().endswith(self.SUFFIX):
continue
path = os.path.join(self.directory, entry)
if os.path.isfile(path) and os.path.abspath(path) not in self.kept:
found.append(path)
return found
def prune(self):
"""Delete them, returning what went.
A file that cannot be removed is not worth failing a download
over - it means the next run sees a duplicate, which is what
was already happening.
"""
removed = []
for path in self.stale():
try:
os.remove(path)
except OSError as error:
print(f"could not remove superseded {path}: {error}")
continue
removed.append(path)
return removed

54
src/archipelago_tester/pipeline/download/unusable_links.py
View File

@@ -1,54 +0,0 @@
"""Why none of a game's links could be downloaded from."""
import urllib.parse
class UnusableLinks:
"""Categorise links no handler recognised.
Reaching here means every recognised shape was tried against every
link, so the status page can say something concrete instead of a
generic "failed".
"""
def __init__(self, config, links):
self.config = config
self.links = links
@property
def unreachable(self):
"""Hosts known in advance never to be downloadable.
A Discord message link is not a public file however it is
fetched, unlike an unknown host that merely has no handler yet.
"""
return self.config.value("downloads", "unreachable_link_hosts") or {}
def domains(self):
"""Each link's host, or a reason it has none."""
found = []
for link in self.links:
parsed = urllib.parse.urlparse(link)
if parsed.scheme not in ("http", "https"):
protocol = parsed.scheme or "(none)"
return None, f"unsupported link protocol {protocol!r}: {link}"
if not parsed.netloc:
return None, f"not a valid URL: {link}"
netloc = parsed.netloc.lower()
found.append(netloc[4:] if netloc.startswith("www.") else netloc)
return found, None
def detail(self):
domains, problem = self.domains()
if problem is not None:
return problem
for domain in domains:
if domain in self.unreachable:
return self.unreachable[domain]
if "github.com" in domains:
return ("GitHub link found, but not a recognized "
"releases/repo/file URL")
if "gitlab.com" in domains:
return "GitLab link found, but not a recognized releases/blob URL"
hosts = ", ".join(sorted(set(domains)))
return f"no supported download source (custom host: {hosts})"

168
src/archipelago_tester/pipeline/download/update_history.py
View File

@@ -1,168 +0,0 @@
"""Every distinct apworld file this pipeline has seen, per game.
One file per game, written only when that game gains content it has not
seen before - so a run touches the handful of games that actually
changed rather than rewriting one shared blob. Deliberately NOT under
downloads/: that directory is a cache and gets cleared to force a clean
re-download, which is harmless for everything else in it. History is the
one thing that cannot be rebuilt afterwards - which version existed on
which date has no other source once it is gone.
"""
import os
from datetime import datetime, timezone
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.core.model.fingerprint import Fingerprint
from archipelago_tester.core.model.name import Name
from archipelago_tester.core.state.keys import StateKeys
from archipelago_tester.core.state.store import StateStore
class UpdateHistory:
"""What each game's apworld was, and when.
Identity is the file's sha256, so a re-download of unchanged content
adds nothing and recording is safe to repeat within a run. The
game's sheet name is stored inside the file as well as encoded in
its name, because the filename is a folded directory name and a
sheet rename would otherwise leave a file nothing could identify.
"""
def __init__(self, paths, directory=None):
self.directory = directory or paths.history
def path_for(self, game_name):
folded = Name(game_name).directory
return os.path.join(self.directory, f"{folded}.json")
def load(self, game_name):
return StateStore.read(
self.path_for(game_name),
{"game": game_name, "watching_since": None,
"baseline_at": None, "files": []})
def save(self, game_name, history):
os.makedirs(self.directory, exist_ok=True)
StateStore(self.path_for(game_name)).save(history)
#: Everything recorded before this field existed was gathered from
#: whatever the hosts happened to say, at whatever moment the file
#: was first downloaded, and none of it measures how often a world
#: changes under observation. The first pass to touch a game draws
#: a line: that instant is the game's one starting point, and only
#: changes seen after it count. See UpdateCadence, which reads it.
@staticmethod
def watching_since(history, stamp):
"""When this pipeline began watching this game.
The earliest content it has already recorded, so a file written
before this field existed still gets an honest answer rather
than a window that starts the day the field was added. Only a
game with no history at all starts watching now.
"""
seen = [item.get("first_seen_at") for item in history["files"]
if item.get("first_seen_at")]
return min(seen) if seen else stamp
def entry_for(self, apworld_path, sha256, released_at, source_url, now):
"""One file's record.
`released_at` is the upload date the download stage saw;
first_seen_at is when this pipeline observed it, which is the
only honest timestamp for a file whose host reports no date.
"""
manifest = ApworldFile(apworld_path).manifest or {}
return {
"version": manifest.get("world_version"),
"released_at": released_at,
"first_seen_at": (now or datetime.now(timezone.utc)).isoformat(),
"sha256": sha256,
"filename": os.path.basename(apworld_path),
"source_url": source_url,
}
def record_file(self, game_name, apworld_path, sha256, released_at,
source_url=None, now=None):
"""Add this file to the game's history if its content is new.
A game whose content has not changed still records that it was
looked at. How often a world updates is only meaningful against
how long it has been watched, and without that a game seen once
and a game seen all year are indistinguishable.
"""
history = self.load(game_name)
now = now or datetime.now(timezone.utc)
opened = not history.get("watching_since")
if opened:
history["watching_since"] = self.watching_since(
history, now.isoformat())
if not history.get("baseline_at"):
opened = True
history["baseline_at"] = now.isoformat()
if any(item.get("sha256") == sha256 for item in history["files"]):
if opened:
self.save(game_name, history)
return None
entry = self.entry_for(apworld_path, sha256, released_at,
source_url, now)
history["game"] = game_name
history["files"].append(entry)
# Oldest first. released_at is missing often enough that it
# cannot be the only key - first_seen_at always exists and
# breaks those ties in the order the pipeline saw them.
history["files"].sort(
key=lambda item: (item.get("released_at") or "",
item.get("first_seen_at") or ""))
self.save(game_name, history)
return entry
def files_in(self, game_directory):
return [
name for name in sorted(os.listdir(game_directory))
if name.lower().endswith(ApworldFile.SUFFIX)
and os.path.isfile(os.path.join(game_directory, name))
]
def record_all(self, state, root_directory, now=None):
"""Record every apworld on disk that is not yet in history.
Hashes the files directly rather than reading apworld_tests:
that dict only gains a hash once a world has been TESTED, so a
file downloaded this run would not appear until the next one -
and this runs right after downloading, which is the point at
which a new version is known.
"""
releases = state.get(StateKeys.GAME_RELEASES) or {}
# game_releases is keyed by the sheet name while the download
# folder is that name folded - map one to the other rather than
# assuming they match as written.
by_folder = {Name(name).directory: info
for name, info in releases.items()}
added = []
if not os.path.isdir(root_directory):
return added
for folder in sorted(os.listdir(root_directory)):
game_directory = os.path.join(root_directory, folder)
if not os.path.isdir(game_directory):
continue
added += self.record_folder(
folder, game_directory, by_folder.get(folder) or {}, now)
return added
def record_folder(self, folder, game_directory, release, now):
"""Every new apworld in one game's download folder."""
added = []
for filename in self.files_in(game_directory):
path = os.path.join(game_directory, filename)
entry = self.record_file(
game_name=folder,
apworld_path=path,
sha256=Fingerprint.of_file(path),
released_at=release.get("published_at"),
source_url=release.get("url"),
now=now,
)
if entry:
added.append((folder, entry))
return added

98
src/archipelago_tester/pipeline/download/url_shapes.py
View File

@@ -1,98 +0,0 @@
"""Recognising the shapes a sheet link can take."""
import os
import re
import urllib.parse
from archipelago_tester.core.model.apworld import ApworldFile
class UrlShapes:
"""The link shapes this pipeline knows how to fetch from."""
def __init__(self, config):
self.config = config
def releases(self, url):
"""Owner, repo, search term and tag from a releases link.
A link naming one exact tag is read as such: without it, a repo
hosting several worlds would fall back to "latest", which drifts
to whichever world was tagged most recently.
"""
match = re.match(
r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+)/releases",
url or "", re.IGNORECASE)
if match is None:
return None
parsed = urllib.parse.urlparse(url)
terms = urllib.parse.parse_qs(parsed.query).get("q", [])
tag_match = re.match(
r"^/[^/]+/[^/]+/releases/tag/([^/?#]+)", parsed.path,
re.IGNORECASE)
return (
match.group(1),
match.group(2),
terms[0].strip('"') if terms else None,
urllib.parse.unquote(tag_match.group(1)) if tag_match else None,
)
def github_file(self, url):
"""Owner, repo, branch and path from a committed-file link."""
match = re.match(
r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+)"
r"/(?:blob|raw)/([^/]+)/(.+\.apworld)$",
url or "", re.IGNORECASE)
return match.groups() if match else None
def github_repo(self, url):
"""Owner, repo and branch from a bare repository link.
Deliberately strict, so it does not swallow issues or wiki
links: those should keep falling through to the generic "not a
releases URL" message rather than triggering a tree search that
could never succeed.
"""
match = re.match(
r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+?)"
r"(?:/tree/([^/?#]+))?/?$",
url or "", re.IGNORECASE)
if match is None:
return None
owner, repository, branch = match.groups()
return owner, repository, branch or "HEAD"
def gitlab_file(self, url):
"""Project path, branch and file path from a GitLab blob link."""
match = re.match(
r"^https?://(?:www\.)?gitlab\.com/([^/]+(?:/[^/]+)*?)"
r"/-/blob/([^/]+)/(.+\.apworld)$",
url or "", re.IGNORECASE)
return match.groups() if match else None
def gitlab_releases(self, url):
"""The project path from a GitLab releases link."""
match = re.match(
r"^https?://(?:www\.)?gitlab\.com/([^/]+(?:/[^/]+)*?)"
r"/-/releases(?:[/?#]|$)",
url or "", re.IGNORECASE)
return match.group(1) if match else None
def gitea_releases(self, url):
"""Host, owner and repo from a Gitea-style releases link."""
match = re.match(
r"^https?://(?!(?:www\.)?(?:github|gitlab)\.com)"
r"([^/]+)/([^/]+)/([^/]+)/releases(?:[/?#]|$)",
url or "", re.IGNORECASE)
return match.groups() if match else None
def plain_file(self, url):
"""A URL ending in .apworld on any host, and its filename."""
parsed = urllib.parse.urlparse(url or "")
if parsed.scheme not in ("http", "https") or not parsed.netloc:
return None
path = urllib.parse.unquote(parsed.path)
if not path.lower().endswith(ApworldFile.SUFFIX):
return None
filename = os.path.basename(path)
return (url, filename) if filename != ApworldFile.SUFFIX else None

77
src/archipelago_tester/pipeline/download/walk_back.py
View File

@@ -1,77 +0,0 @@
"""Falling back to an older release that still ships an apworld."""
import requests
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.pipeline.download.release_downloader import (
ReleaseDownloader,
)
from archipelago_tester.pipeline.download.release_picker import ReleasePicker
class WalkBack:
"""One narrow rescue: the newest release forgot the apworld.
An author cuts a release and re-attaches only a mod zip, leaving
the world perfectly downloadable one release back. Every other
failure is left alone on purpose - in particular "did not match
this game", where an older release holding a different game's
apworld is precisely the wrong answer.
"""
def __init__(self, context, owner, repository, release):
self.context = context
self.owner = owner
self.repository = repository
self.release = release
def candidate(self, release):
"""Whether this is an older release that ships an apworld.
Matches on asset filename alone: this runs over a repo's whole
history, and opening every zip in it would cost far more than
the problem is worth.
"""
cutoff = self.release.get("published_at")
published_at = release.get("published_at")
return bool(
not release.get("draft")
and release.get("tag_name") != self.release.get("tag_name")
and published_at
and (not cutoff or published_at < cutoff)
and any(asset["name"].lower().endswith(ApworldFile.SUFFIX)
for asset in release.get("assets", []))
)
def older(self):
"""The newest earlier release with an apworld, by date.
Ordering comes from published_at rather than the order GitHub
returns, which follows the underlying tag and is not the same
thing.
"""
picker = ReleasePicker(self.context, self.owner, self.repository)
try:
releases = picker.all()
except requests.RequestException:
return None
candidates = [r for r in releases if self.candidate(r)]
if not candidates:
return None
return max(candidates, key=lambda release: release["published_at"])
def run(self, reason):
"""The fallback's result, or None and a combined reason."""
older = self.older()
if older is None:
return None, reason
fallback = ReleaseDownloader(
context=self.context,
release=older,
require_identity=True,
).run()
if fallback[0] is not None:
fallback[2]["fallback_release_tag"] = older.get("tag_name")
return fallback, None
return None, (f"{reason}; older release {older.get('tag_name')!r} "
f"was tried too: {fallback[1]}")

96
src/archipelago_tester/pipeline/download/zip_search.py
View File

@@ -1,96 +0,0 @@
"""Looking for an apworld inside a release's zip assets."""
import os
import tempfile
import zipfile
import requests
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.pipeline.download.local_file import LocalFile
class ZipSearch:
"""For a release that publishes a zip instead of an apworld.
Some repos attach a source snapshot with the built apworld nested
inside. Every zip is re-downloaded to scratch space each time, since
the member's own filename - what a normal download caches on - is
not known until the zip is opened.
"""
#: The extension a release's archive asset is recognised by.
SUFFIX = ".zip"
def __init__(self, context, release, headers=None):
self.context = context
self.release = release
self.headers = headers
def zip_assets(self):
return [
asset for asset in self.release.get("assets", [])
if asset["name"].lower().endswith(ZipSearch.SUFFIX)
]
def members(self, zip_path):
"""Every apworld inside the zip, with its size."""
try:
with zipfile.ZipFile(zip_path) as archive:
return [
(info.filename, info.file_size)
for info in archive.infolist()
if not info.filename.endswith("/")
and info.filename.lower().endswith(ApworldFile.SUFFIX)
]
except zipfile.BadZipFile:
return []
def extract(self, zip_path, member):
"""Unpack one member into the game's folder."""
target = self.context.target_path(os.path.basename(member))
os.makedirs(os.path.dirname(target), exist_ok=True)
with zipfile.ZipFile(zip_path) as archive:
with archive.open(member) as source:
with open(target, "wb") as handle:
handle.write(source.read())
return target
def unpack(self, zip_path, members, zip_asset):
"""The members as assets, already on disk."""
results = []
for member, size in members:
target = self.context.target_path(os.path.basename(member))
status = LocalFile(self.context.session, target).status(size)
if status in LocalFile.NEEDS_DOWNLOAD:
self.extract(zip_path, member)
results.append({
"name": os.path.basename(member),
"updated_at": zip_asset.get("updated_at"),
"created_at": zip_asset.get("created_at"),
"extracted_path": target,
"status": status,
})
return results
def search_one(self, zip_asset):
with tempfile.TemporaryDirectory() as scratch:
zip_path = os.path.join(scratch, zip_asset["name"])
local = LocalFile(self.context.session, zip_path)
try:
local.download(zip_asset["browser_download_url"],
headers=self.headers)
except requests.RequestException:
return []
members = self.members(zip_path)
if not members:
return []
return self.unpack(zip_path, members, zip_asset)
def assets(self):
"""The first zip that holds an apworld wins."""
for zip_asset in self.zip_assets():
found = self.search_one(zip_asset)
if found:
return found
return []

0
src/archipelago_tester/pipeline/generation/__init__.py
View File

102
src/archipelago_tester/pipeline/generation/command.py
View File

@@ -1,102 +0,0 @@
"""Building the docker invocations the drivers run in."""
import os
from archipelago_tester.core.model.apworld import ApworldFile
class ContainerCommand:
"""The `docker run` for one driver, and the mounts it needs.
The paths below are where this pipeline mounts things inside its
own container. Both ends of that contract live in this repository -
these mount arguments and the drivers that receive them - so they
are part of the code rather than something to configure.
"""
#: Where each driver script is mounted.
DRIVER_PATH = "/app/run_test.py"
IDENTIFY_DRIVER_PATH = "/app/identify_apworld.py"
VERIFY_DRIVER_PATH = "/app/verify_companions.py"
#: Where the apworld under test, the seeds and the ROMs go.
APWORLD_DIRECTORY = "/app/custom_worlds"
OUTPUT_DIRECTORY = "/data/output"
DATA_DIRECTORY = "/app/data"
#: A writable cache directory. The image runs as uid 1000 but
#: leaves HOME as "/", which is root-owned, so anything resolving
#: "~/.cache" fails with PermissionError before generation starts.
CACHE_ENVIRONMENT = {"HOME": "/tmp", "XDG_CACHE_HOME": "/tmp/.cache"}
def __init__(self, config, paths, image):
self.config = config
self.paths = paths
self.image = image
def timeout(self, name, default):
return int(self.config.value("testing", name, default))
@property
def prefix(self):
"""The part of every invocation that never varies."""
return ["docker", "run", "--rm", "--network", "none",
"--entrypoint", "python", *self.cache_environment]
@property
def cache_environment(self):
args = []
for name, value in self.CACHE_ENVIRONMENT.items():
args += ["-e", f"{name}={value}"]
return args
@property
def common_client_mount(self):
"""CommonClient.py, mounted back in from the checkout.
The image excludes every client file, correctly - but a few
apworlds import their own during generation and fail with "No
module named 'CommonClient'" purely because it is absent.
Absent source means no mount: it exists only after a clone.
"""
if not os.path.isfile(self.paths.common_client):
return []
return ["-v", f"{self.paths.common_client}:/app/CommonClient.py:ro"]
@property
def rom_mounts(self):
"""Every base ROM, mounted individually.
Per file rather than mounting the directory over /app/data,
which would shadow the image's own shipped data. Worlds declare
rom_file inconsistently - some embed "data/", some do not - so
each is mounted at both locations.
"""
directory = self.paths.roms
if not os.path.isdir(directory):
return []
data_in = self.DATA_DIRECTORY
args = []
for name in sorted(os.listdir(directory)):
path = os.path.join(directory, name)
if os.path.isfile(path):
args += ["-v", f"{path}:{data_in}/{name}:ro"]
args += ["-v", f"{path}:/app/{name}:ro"]
return args
def apworld_mount(self, apworld_path):
"""The mount and container path for the world under test."""
if not apworld_path:
return [], None
name = ApworldFile(apworld_path).importable_name
inside = f"{self.APWORLD_DIRECTORY}/{name}"
return ["-v", f"{apworld_path}:{inside}:ro"], inside
def driver_mounts(self, output_dir):
"""The driver, the output directory, the ROMs and the client."""
return [
"-v", f"{self.paths.driver}:{self.DRIVER_PATH}:ro",
"-v", f"{output_dir}:{self.OUTPUT_DIRECTORY}",
*self.rom_mounts,
*self.common_client_mount,
]

77
src/archipelago_tester/pipeline/generation/companion_check.py
View File

@@ -1,77 +0,0 @@
"""Which core worlds can actually generate in this container."""
import os
import subprocess
from archipelago_tester.core.config.run_settings import RunSettings
from archipelago_tester.pipeline.generation.command import ContainerCommand
from archipelago_tester.pipeline.generation.report import DriverReport
class CompanionCheck:
"""Tries every core world once, so the rest can trust the pool.
Several core worlds need a base ROM this container has no copy of,
and drawing one fails the seed - with the failure landing on the
apworld under test rather than on the companion that caused it.
"""
def __init__(self, config, paths, image):
self.config = config
self.paths = paths
self.image = image
self.command = ContainerCommand(config, paths, image)
def build(self, output_dir, per_world_timeout, spoiler):
verify_in = ContainerCommand.VERIFY_DRIVER_PATH
return [
*self.command.prefix,
"-v", f"{self.paths.verify_driver}:{verify_in}:ro",
"-v", (f"{self.paths.driver}:"
f"{ContainerCommand.DRIVER_PATH}:ro"),
"-v", (f"{output_dir}:"
f"{ContainerCommand.OUTPUT_DIRECTORY}"),
*self.command.rom_mounts,
*self.command.common_client_mount,
self.image,
verify_in,
"--output-dir", ContainerCommand.OUTPUT_DIRECTORY,
"--timeout", str(per_world_timeout),
"--spoiler", str(spoiler),
]
@staticmethod
def not_verified(detail):
return {"ran": False, "verified": [], "rejected": {},
"detail": detail}
def parse(self, result):
data = DriverReport(result).data
if data is not None:
data.setdefault("ran", True)
return data
return self.not_verified(
f"no JSON output (exit {result.returncode}): "
f"{DriverReport.tail(result.stderr)}")
def run(self, output_dir, timeout=None, per_world_timeout=None,
spoiler=None):
"""The pool of usable companions, or why there is none."""
timeout = timeout or self.command.timeout("verify_timeout", 1800)
per_world_timeout = per_world_timeout or self.command.timeout(
"verify_per_world_timeout", 120)
if spoiler is None:
spoiler = RunSettings(self.config).spoiler
output_dir = os.path.abspath(output_dir)
os.makedirs(output_dir, exist_ok=True)
try:
result = subprocess.run(
self.build(output_dir, per_world_timeout, spoiler),
capture_output=True,
text=True,
timeout=timeout,
)
except subprocess.TimeoutExpired:
return self.not_verified("companion verification did not "
f"finish within {timeout}s")
return self.parse(result)

70
src/archipelago_tester/pipeline/generation/identifier.py
View File

@@ -1,70 +0,0 @@
"""Asking an apworld which game it registers as."""
import os
import subprocess
import tempfile
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.pipeline.generation.command import ContainerCommand
from archipelago_tester.pipeline.generation.report import DriverReport
class ApworldIdentifier:
"""What a file's own World class says it is.
Lighter than a test: no ROMs, no output volume, no generation. Used
right after download to check the content matches the row that
fetched it, before the file is trusted at all.
"""
def __init__(self, config, paths, image):
self.paths = paths
self.image = image
self.command = ContainerCommand(config, paths, image)
def build(self, apworld_path):
mount, inside = self.command.apworld_mount(apworld_path)
identify_in = ContainerCommand.IDENTIFY_DRIVER_PATH
return [
*self.command.prefix,
*mount,
"-v", f"{self.paths.identify_driver}:{identify_in}:ro",
*self.command.common_client_mount,
self.image,
identify_in,
"--apworld", inside,
]
def failed(self, detail):
return {"ran": False, "games": [], "detail": detail}
def identify(self, apworld_path, timeout=None):
"""Which games this file registers, and whether the check ran."""
timeout = timeout or self.command.timeout("identify_timeout", 60)
apworld_path = os.path.abspath(apworld_path)
if not os.path.isfile(apworld_path):
return self.failed(f"{apworld_path} does not exist (not a file)")
try:
result = self.run(apworld_path, timeout)
except subprocess.TimeoutExpired:
return self.failed(
f"identify check did not finish within {timeout}s")
data = DriverReport(result).data
if data is not None:
data.setdefault("ran", True)
return data
return self.failed(
f"no JSON output (exit {result.returncode}): "
f"{DriverReport.tail(result.stderr)}")
def run(self, apworld_path, timeout):
"""Run the driver against a prepared copy of the file.
Repaired here as well as at test time, so a backslash-separated
world is not rejected at download for registering no game and
never reaching the test that could repair it.
"""
with tempfile.TemporaryDirectory(prefix="apworld-id-") as workdir:
checked, _ = ApworldFile(apworld_path).prepare(workdir)
return subprocess.run(self.build(checked), capture_output=True,
text=True, timeout=timeout)

54
src/archipelago_tester/pipeline/generation/report.py
View File

@@ -1,54 +0,0 @@
"""Reading what a driver reported."""
import json
class DriverReport:
"""The JSON one driver run wrote to stdout."""
def __init__(self, result):
self.result = result
@staticmethod
def tail(text, lines=20):
return "\n".join(text.splitlines()[-lines:])
@staticmethod
def last_json_object(text):
"""The driver's report, however it ended up on the line.
Not "the last line starting with {": worlds write to raw stdout
during generation, and one that writes without a trailing
newline leaves its output glued to the front of the report -
Super Metroid's randomizer emits a bare "*" per retry, so the
report arrives as "*{...}" and would read as no report at all.
"""
for line in reversed(text.splitlines()):
line = line.strip()
start = line.find("{")
while start != -1:
try:
return json.loads(line[start:])
except json.JSONDecodeError:
start = line.find("{", start + 1)
return None
@property
def data(self):
return self.last_json_object(self.result.stdout)
def parsed(self):
"""The report, or a failure built from what it printed."""
report = self.data
if report is not None:
report.setdefault("stderr_tail", self.tail(self.result.stderr))
return report
return {
"game": None,
"outcome": "failed",
"detail": ("no JSON report on stdout "
f"(exit code {self.result.returncode})"),
"elapsed_seconds": None,
"stdout_tail": self.tail(self.result.stdout),
"stderr_tail": self.tail(self.result.stderr),
}

132
src/archipelago_tester/pipeline/generation/tester.py
View File

@@ -1,132 +0,0 @@
"""Testing one world by generating seeds for it."""
import os
import subprocess
import tempfile
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.pipeline.generation.command import ContainerCommand
from archipelago_tester.pipeline.generation.report import DriverReport
class WorldTester:
"""Runs every generation mode against one world."""
def __init__(self, config, paths, image):
self.config = config
self.paths = paths
self.image = image
self.command = ContainerCommand(config, paths, image)
def settings(self, key, default):
return self.config.value("testing", key, default)
def build(self, apworld_path, output_dir, timeout, companion_range,
spoiler, repeats, companion_pool, game, random_repeats):
"""The full `docker run` for one world's whole test."""
mount, inside = self.command.apworld_mount(apworld_path)
selector = ["--apworld", inside] if inside else ["--game", game]
return [
*self.command.prefix,
*mount,
*self.command.driver_mounts(output_dir),
self.image,
ContainerCommand.DRIVER_PATH,
*selector,
"--output-dir", ContainerCommand.OUTPUT_DIRECTORY,
"--timeout", str(timeout),
"--companion-min", str(companion_range[0]),
"--companion-max", str(companion_range[1]),
"--spoiler", str(spoiler),
"--repeats", str(repeats),
"--random-repeats", str(random_repeats),
"--companion-pool", ",".join(companion_pool or ()),
]
def host_timeout(self, timeout, repeats, random_repeats):
"""How long to let the whole container run.
Every mode runs in the one container, each bounded by its own
timeout, so the host allows for all of them plus startup slack.
A ceiling rather than an expected cost.
"""
generations = 2 * repeats + 1 + 2 * random_repeats
buffer = self.settings("host_timeout_buffer", 30)
return timeout * generations + buffer
def missing_file(self, path):
"""The failure for a path that is not a file.
Docker silently creates an empty directory at a missing mount
source rather than failing, which turns a not-yet-written
apworld into a bogus directory that breaks the content hash
days later with a confusing "Is a directory".
"""
return {
"game": None,
"outcome": "failed",
"detail": f"{path} does not exist (not a file)",
"elapsed_seconds": None,
}
def test(self, apworld_path=None, game=None, output_dir=None,
timeout=None, companion_range=(2, 5), spoiler=3, repeats=10,
companion_pool=(), random_repeats=3):
"""Run every mode against one world, core or downloaded."""
output_dir = os.path.abspath(output_dir or self.paths.output)
os.makedirs(output_dir, exist_ok=True)
timeout = timeout or self.settings("generation_timeout", 300)
if apworld_path is not None:
apworld_path = os.path.abspath(apworld_path)
if not os.path.isfile(apworld_path):
return self.missing_file(apworld_path)
with tempfile.TemporaryDirectory(prefix="apworld-") as workdir:
tested, repaired = self.prepared(apworld_path, workdir)
report = self.run_once(
apworld_path=tested,
game=game,
output_dir=output_dir,
timeout=timeout,
companion_range=companion_range,
spoiler=spoiler,
repeats=repeats,
companion_pool=companion_pool,
random_repeats=random_repeats,
)
if repaired:
report["repaired"] = True
return report
def prepared(self, apworld_path, workdir):
"""The path to test, and whether it had to be repaired."""
if not apworld_path:
return None, False
return ApworldFile(apworld_path).prepare(workdir)
def run_once(self, apworld_path, game, output_dir, timeout,
companion_range, spoiler, repeats, companion_pool,
random_repeats):
"""One container run, and the report it produced."""
limit = self.host_timeout(timeout, repeats, random_repeats)
command = self.build(
apworld_path=apworld_path,
output_dir=output_dir,
timeout=timeout,
companion_range=companion_range,
spoiler=spoiler,
repeats=repeats,
companion_pool=companion_pool,
game=game,
random_repeats=random_repeats,
)
try:
result = subprocess.run(command, capture_output=True, text=True,
timeout=limit)
except subprocess.TimeoutExpired:
return {
"game": None,
"outcome": "failed",
"detail": f"docker run did not exit within {limit}s",
"elapsed_seconds": limit,
}
return DriverReport(result).parsed()

0
src/archipelago_tester/pipeline/host/__init__.py
View File

134
src/archipelago_tester/pipeline/host/custom_worlds.py
View File

@@ -1,134 +0,0 @@
"""Placing the approved apworlds where an Archipelago install finds them."""
import os
import shutil
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.core.model.verdict import Verdict
from archipelago_tester.core.state.keys import StateKeys
class CustomWorlds:
"""The install's custom_worlds, kept to the worlds that passed.
Only apworlds are placed. Nothing else in the install is read or
written - not host.yaml, not options, not the checkout - because the
install is someone else's and this pipeline only decides which
worlds are good enough to be in it.
update_archipelago says whether to touch it at all and
archipelago_location says which install: with either missing,
nothing is changed and the reason is reported rather than guessed
at.
"""
def __init__(self, config, paths, store, tag):
self.config = config
self.paths = paths
self.store = store
self.tag = tag
def setting(self, key, default=None):
return self.config.value("archipelago", key, default)
@property
def enabled(self):
return bool(self.setting("update_archipelago", False))
@property
def directory(self):
return self.paths.custom_worlds
@property
def approved_labels(self):
"""The labels good enough to place, matched case-insensitively.
Written as the labels read rather than as the verdicts behind
them, so promote_labels and stability.labels are the same
wording and a rename in one is visible in the other.
"""
return {str(label).casefold()
for label in self.setting("promote_labels", [])}
def approved(self, record):
"""Whether this record's world belongs in the install.
Judged on the verdict its own test produced, for the version
being run now: a result from an older Archipelago says nothing
about whether the world loads in this one.
"""
if record.get(StateKeys.DISCONTINUED):
return False
if record.get("tag") != self.tag:
return False
label = Verdict(record, self.config).label
return label.casefold() in self.approved_labels
def candidates(self):
"""Each approved apworld on disk, by the name it imports as.
Two apworlds can declare the same importable name - a stray
download beside the real one - and one would silently overwrite
the other in a flat directory, so both are held back and
reported instead of picking a winner here.
"""
tests = self.store.load().get(StateKeys.APWORLD_TESTS, {})
by_name = {}
for key, record in tests.items():
if not self.approved(record):
continue
path = os.path.join(self.paths.downloads, key)
if os.path.exists(path):
name = ApworldFile(path).importable_name
by_name.setdefault(name, []).append(path)
placeable = {name: paths[0] for name, paths in by_name.items()
if len(paths) == 1}
collisions = {name: paths for name, paths in by_name.items()
if len(paths) > 1}
return placeable, collisions
def place(self, placeable):
"""Copy each approved apworld in under its importable name.
Copied rather than linked: Archipelago loads these directly,
and a link into this pipeline's download cache would break the
moment that cache is cleared.
"""
os.makedirs(self.directory, exist_ok=True)
for name, path in placeable.items():
shutil.copy2(path, os.path.join(self.directory, name))
return sorted(placeable)
def withdraw(self, placeable):
"""Delete the apworlds that are no longer approved.
A world that has stopped passing has to leave the install, not
merely stop being refreshed: leaving it there would keep
offering a world this pipeline no longer vouches for. Only
apworlds are ever removed, so anything else a person put in
that directory stays.
"""
present = {name for name in os.listdir(self.directory)
if name.endswith(ApworldFile.SUFFIX)}
withdrawn = present - set(placeable)
for name in withdrawn:
os.remove(os.path.join(self.directory, name))
return sorted(withdrawn)
def update(self):
"""Bring the install's custom_worlds to the approved set."""
if not self.enabled:
return {"updated": False, "reason": "update_archipelago is off"}
if not self.directory:
return {"updated": False,
"reason": "archipelago_location is not set"}
placeable, collisions = self.candidates()
placed = self.place(placeable)
withdrawn = self.withdraw(placeable)
return {
"updated": True,
"directory": self.directory,
"placed": placed,
"withdrawn": withdrawn,
"collisions": collisions,
}

0
src/archipelago_tester/pipeline/sheet/__init__.py
View File

62
src/archipelago_tester/pipeline/sheet/fetcher.py
View File

@@ -1,62 +0,0 @@
"""Downloading the worlds sheet."""
import io
import os
import zipfile
import requests
class SheetFetcher:
"""The Playable Worlds sheet, as HTML."""
def __init__(self, config, paths):
self.config = config
self.paths = paths
@property
def spreadsheet_id(self):
return self.config.value("sheet", "spreadsheet_id")
@property
def tab_name(self):
return self.config.value("sheet", "tab_name")
@property
def export_url(self):
return ("https://docs.google.com/spreadsheets/d/"
f"{self.spreadsheet_id}/export?format=zip")
def fetch(self):
"""The tab's HTML, as bytes.
Google answers a private sheet with its sign-in page and status
200 rather than an error, so the content type is checked instead
of the status.
"""
response = requests.get(self.export_url, timeout=120)
response.raise_for_status()
if "application/zip" not in response.headers.get("Content-Type", ""):
raise RuntimeError(
"The server did not return a zip archive. The sheet is "
"most likely not shared with 'anyone with the link'.")
with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
wanted = f"{self.tab_name}.html"
if wanted not in archive.namelist():
raise RuntimeError(
f"The export has no tab called '{self.tab_name}'. "
f"Available: {', '.join(archive.namelist())}")
return archive.read(wanted)
def save(self, content=None):
"""Write the tab's HTML, returning where it landed."""
content = self.fetch() if content is None else content
with open(self.paths.sheet_html, "wb") as handle:
handle.write(content)
return self.paths.sheet_html
def ensure(self):
"""Download the sheet only when this machine has no copy."""
if not os.path.exists(self.paths.sheet_html):
self.save()
return self.paths.sheet_html

98
src/archipelago_tester/pipeline/sheet/reader.py
View File

@@ -1,98 +0,0 @@
"""Reading the worlds sheet into Game objects."""
from bs4 import BeautifulSoup
from archipelago_tester.core.model.game import Game
class SheetReader:
"""The sheet's HTML export, as rows."""
def __init__(self, config, path):
self.config = config
self.path = path
def column(self, name):
return (self.config.value("sheet", "columns") or {})[name]
def rows(self):
"""Every table row, as lists of cells.
Google repeats each row's frozen-column cells in a spacer, which
would shift every column index by one, so those are dropped.
"""
with open(self.path, "r", encoding="utf-8") as handle:
soup = BeautifulSoup(handle.read(), "html.parser")
found = []
for row in soup.find_all("tr"):
cells = [
cell for cell in row.find_all("td")
if "freezebar-cell" not in cell.get("class", [])
]
if cells:
found.append(cells)
return found
def header(self, rows):
"""Where the header row is, and the headings it holds."""
for index, cells in enumerate(rows):
texts = [cell.get_text(strip=True) for cell in cells]
if self.column("name") in texts:
return index, texts
raise ValueError(
f"No '{self.column('name')}' header found in the export.")
def indexes(self, headers):
"""Where each field sits in a row."""
if self.column("link") not in headers:
raise ValueError(f"No '{self.column('link')}' column found.")
optional = {}
for field in ("stability", "pr_status"):
heading = self.column(field)
optional[field] = (headers.index(heading)
if heading in headers else None)
return (headers.index(self.column("name")),
headers.index(self.column("link")),
optional["stability"],
optional["pr_status"])
@staticmethod
def text_of(cells, index):
"""One cell's text, or None when missing or empty."""
if index is None or index >= len(cells):
return None
return cells[index].get_text(strip=True) or None
@staticmethod
def links_in(cell):
"""Every hyperlink target in one cell, in the order written."""
return [anchor.get("href") for anchor in cell.find_all("a")
if anchor.get("href")]
def build(self, cells, indexes):
"""One row as a Game, or None when it has no name."""
name_index, link_index, stability_index, status_index = indexes
name = self.text_of(cells, name_index)
if name is None:
return None
game = Game(
name=name,
stability=self.text_of(cells, stability_index),
pr_status=self.text_of(cells, status_index),
)
if link_index < len(cells):
for link in self.links_in(cells[link_index]):
game.add_link(link)
return game
def games(self):
"""Every world listed in the export."""
rows = self.rows()
header_index, headers = self.header(rows)
indexes = self.indexes(headers)
found = []
for cells in rows[header_index + 1:]:
game = self.build(cells, indexes)
if game is not None:
found.append(game)
return found

0
src/archipelago_tester/pipeline/update/__init__.py
View File

68
src/archipelago_tester/pipeline/update/cli.py
View File

@@ -1,68 +0,0 @@
"""The command line for a full update pass."""
import argparse
import json
from archipelago_tester.core.config.config import Config
from archipelago_tester.core.config.paths import Paths
from archipelago_tester.core.config.run_settings import RunSettings
from archipelago_tester.core.state.lock import LockHeldError, PipelineLock
from archipelago_tester.pipeline.update.stage_error import StageError
from archipelago_tester.pipeline.update.update_run import UpdateRun
class UpdateCli:
"""Update Archipelago, the sheet and every apworld, then test."""
def __init__(self):
self.config = Config.load()
self.paths = Paths(self.config)
self.settings = RunSettings(self.config)
def parser(self):
parser = argparse.ArgumentParser(description=self.__doc__)
parser.add_argument(
"--stability",
help="Only consider games with this Stability column value "
"(e.g. 'stable', 'broken on main'). Case-insensitive.")
parser.add_argument(
"--fetch", type=int,
help="Only consider the first N games, after --stability "
"filtering - for a cheap, bounded pass.")
parser.add_argument(
"--jobs", "-j", type=int, default=None,
help="Apworlds to generation-test at once "
f"(default: {self.settings.jobs}).")
parser.add_argument(
"--force-retest", action="store_true",
help="Re-test every apworld, ignoring the cache. Expensive.")
return parser
def run(self):
args = self.parser().parse_args()
run = UpdateRun(
config=self.config,
paths=self.paths,
stability=args.stability,
fetch=args.fetch,
jobs=args.jobs,
force_retest=args.force_retest,
)
try:
with PipelineLock(self.paths.lock):
report = run.run()
except LockHeldError as error:
print(error)
return 1
except StageError as error:
print(f"Update {error}")
return 1
print("\n=== summary ===")
summary = {key: value for key, value in report.items()
if key != "test_results"}
print(json.dumps(summary, indent=2, default=str))
return 0
@classmethod
def main(cls):
return cls().run()

37
src/archipelago_tester/pipeline/update/download_reporter.py
View File

@@ -1,37 +0,0 @@
"""The download stage's progress line."""
import os
from archipelago_tester.core.display.progress import Progress
class DownloadReporter:
"""One line per game, saying what its apworld did.
The download run calls back serialized, so this needs no lock of its
own. What each game's last asset did is remembered so the line can
still name it once the game itself finishes.
"""
#: What each download status means, in words.
STATUS_LABELS = {
"missing": "downloaded (new)",
"stale": "downloaded (updated)",
"verified": "already present (verified up to date)",
"assumed": "already present (not verified)",
}
def __init__(self, config, total):
self.bar = Progress(total, config=config,
labels={"ok": "ok", "none": "none"})
self.last_asset = {}
def on_asset(self, game_name, path, status):
label = self.STATUS_LABELS.get(status, status)
self.last_asset[game_name] = f"{label} {os.path.basename(path)}"
def on_game(self, index, total, game_name, downloaded):
detail = self.last_asset.pop(game_name, "no apworld")
self.bar.update(
index, "ok" if downloaded else "none", game_name,
fallback_line=f"[{index}/{total}] {game_name}: {detail}")

15
src/archipelago_tester/pipeline/update/stage_error.py
View File

@@ -1,15 +0,0 @@
"""A failure, named by the stage it happened in."""
class StageError(RuntimeError):
"""Which stage failed, so the run can say so and stop.
Raised at the stage boundary rather than inside it: a stage can fail
anywhere after it was opened, and an unnamed traceback makes a long
run's failure much harder to place.
"""
def __init__(self, stage, original):
super().__init__(f"failed during '{stage}': {original}")
self.stage = stage
self.original = original

298
src/archipelago_tester/pipeline/update/update_run.py
View File

@@ -1,298 +0,0 @@
"""Everything the testing half does in one pass.
Update Archipelago to the newest release, refresh the worlds sheet,
download whatever changed, and test each world by generating seeds.
What happens to the results afterwards - promoting, publishing,
deploying - is the other half's job and is not known about here.
"""
import contextlib
import requests
from archipelago_tester.core.config.run_settings import RunSettings
from archipelago_tester.core.config.secrets import Secrets
from archipelago_tester.core.model.name import Name
from archipelago_tester.core.state.keys import StateKeys
from archipelago_tester.core.state.store import StateStore
from archipelago_tester.pipeline.batch.batch import Batch
from archipelago_tester.pipeline.batch.discontinued import Discontinued
from archipelago_tester.pipeline.batch.orphaned_records import (
OrphanedRecords,
)
from archipelago_tester.pipeline.batch.download_failures import (
DownloadFailures,
)
from archipelago_tester.pipeline.batch.reporter import Reporter
from archipelago_tester.pipeline.batch.results import Results
from archipelago_tester.pipeline.build.archipelago import ArchipelagoBuild
from archipelago_tester.pipeline.download.download_run import DownloadRun
from archipelago_tester.pipeline.download.update_history import (
UpdateHistory,
)
from archipelago_tester.pipeline.generation.identifier import ApworldIdentifier
from archipelago_tester.pipeline.host.custom_worlds import CustomWorlds
from archipelago_tester.pipeline.sheet.fetcher import SheetFetcher
from archipelago_tester.pipeline.sheet.reader import SheetReader
from archipelago_tester.pipeline.update.download_reporter import (
DownloadReporter,
)
from archipelago_tester.pipeline.update.stage_error import StageError
class UpdateRun:
"""The four stages, in the order they run.
Takes no lock of its own: a caller that also publishes the results
holds one for the whole thing, so nothing swaps underneath a
generation still in progress.
"""
def __init__(self, config, paths, stability=None, fetch=None, jobs=None,
force_retest=False):
self.config = config
self.paths = paths
self.settings = RunSettings(config)
self.store = StateStore(paths.state)
self.build = ArchipelagoBuild(config, paths)
self.stability = stability
self.fetch = fetch
self.jobs = jobs or self.settings.jobs
self.force_retest = force_retest
self.report = {}
@contextlib.contextmanager
def stage(self, name):
print(f"\n=== {name} ===")
try:
yield
except Exception as error:
raise StageError(name, error) from error
def snapshot_previous_version(self, previous_tag):
"""Keep last version's results before this one overwrites them.
The only source for "did this world's outcome change at the
exact moment Archipelago upgraded": it is the state at the
instant the version changed, which no later re-test can
reconstruct.
"""
state = self.store.load()
state[StateKeys.UPGRADE_SNAPSHOT] = dict(
state.get(StateKeys.APWORLD_TESTS, {}))
state[StateKeys.PREVIOUS_ARCHIPELAGO_TAG] = previous_tag
self.store.save(state)
def update_archipelago(self):
"""Put this machine on the newest release and build its image."""
with self.stage("archipelago_build"):
tag, previous, changed, built = self.build.update_and_build(
self.store)
self.build.ensure_test_image(tag)
self.report.update({
"archipelago_tag": tag,
"archipelago_previous_tag": previous,
"archipelago_tag_changed": changed,
"archipelago_image_built": built,
})
print(f"tag {tag}" + (f" (was {previous})" if previous
else " (first run)"))
if changed and previous:
self.snapshot_previous_version(previous)
return tag, changed
def announce_retired(self, retired):
gone = sorted(key for key, value in retired.items() if value)
back = sorted(key for key, value in retired.items() if not value)
if gone:
print(f"{len(gone)} no longer on the sheet, marked "
f"discontinued: {', '.join(gone)}")
if back:
print(f"{len(back)} back on the sheet: {', '.join(back)}")
def select(self, games):
"""The rows this pass considers.
Applied to the whole sheet, core rows included, so --fetch
really does bound the work: core worlds are tested like every
other row, and leaving them out of the limit made a "cheap,
bounded pass" still test all eighty of them.
"""
if self.stability:
games = [game for game in games
if (game.stability or "").lower()
== self.stability.lower()]
return games[:self.fetch] if self.fetch is not None else games
def read_sheet(self):
"""The sheet's rows, split into core and downloadable.
Discontinued rows are flagged straight after reading, so every
stage below already sees a world taken off the sheet as gone.
Core rows are tested by name but never downloaded: Archipelago
ships them.
"""
with self.stage("sheet_download"):
path = SheetFetcher(self.config, self.paths).save()
games = SheetReader(self.config, path).games()
# Every row, before any filtering: download-time identity
# checks compare against the whole sheet, core rows
# included, and so does the discontinued flag.
sheet_names = [game.name for game in games]
self.report["games_in_sheet"] = len(games)
print(f"{len(games)} games in sheet")
retired = Discontinued(self.store).mark(sheet_names)
self.report["discontinued_changed"] = retired
self.announce_retired(retired)
selected = self.select(games)
core = [game for game in selected if game.is_core]
rest = [game for game in selected if not game.is_core]
self.report["games_core"] = len(core)
self.report["games_selected"] = len(rest)
print(f"{len(core)} core games (tested by name, nothing to download)")
print(f"{len(rest)} games selected (stability={self.stability!r}, "
f"fetch={self.fetch!r})")
return rest, core, sheet_names
def session(self):
"""A session carrying the token, if there is one."""
session = requests.Session()
token = Secrets(self.paths).github_token
if token:
session.headers["Authorization"] = f"token {token}"
return session
def downloader(self, image, sheet_names):
return DownloadRun(
config=self.config,
paths=self.paths,
session=self.session(),
identifier=ApworldIdentifier(self.config, self.paths, image),
jobs=self.jobs,
sheet_names=sheet_names,
# Last run's resolved releases, so a game whose attempt
# failed against an unchanged release is not downloaded and
# re-identified all over again.
previous_releases=self.store.load().get(
StateKeys.GAME_RELEASES, {}),
)
def download(self, games, sheet_names, image, tag):
"""Fetch every selected game's newest apworld."""
reporter = DownloadReporter(self.config, len(games))
run = self.downloader(image, sheet_names)
with self.stage("apworld_download"):
total, skipped, releases = run.all(
games, on_asset=reporter.on_asset, on_game=reporter.on_game)
self.report["assets_downloaded"] = total
self.report["games_skipped_download"] = len(skipped)
print(f"{total} assets downloaded, {len(skipped)} games skipped")
run.record_releases(releases)
self.record_versions()
DownloadFailures(self.config, self.store).record(games, skipped, tag)
orphaned = OrphanedRecords(self.store, self.paths.downloads).prune()
self.report["orphaned_records_dropped"] = len(orphaned)
if orphaned:
print(f"{len(orphaned)} result(s) dropped for apworlds no "
f"longer on disk: {', '.join(sorted(orphaned))}")
def record_versions(self):
"""One history entry per game whose apworld content changed.
Recorded here, not by whatever publishes afterwards: this is
the moment a new version is known, and a pass that never
publishes would lose it outright - the next download overwrites
the file and no later stage can reconstruct which version
existed on which date.
"""
history = UpdateHistory(self.paths)
with self.stage("apworld_history"):
recorded = history.record_all(self.store.load(),
self.paths.downloads)
self.report["new_apworld_versions"] = len(recorded)
print(f"{len(recorded)} new apworld version(s) recorded")
def selected_only(self, games, core_games):
"""This pass's rows, named the way the batch names them.
None unless something actually narrowed the sheet. The download
directory holds every apworld ever fetched, so an unnarrowed run
has to consider all of them - and Batch reads an explicit
selection as a reason to retest rather than serve from cache,
which would turn a plain pass into a full re-test of everything.
Downloadable rows are matched on their download folder, core
ones on the bare sheet name, because that is how each is keyed.
"""
if self.fetch is None and not self.stability:
return None
return ([Name(game.name).directory for game in games]
+ [game.name for game in core_games])
def test(self, image, tag, games, core_games, changed):
"""Generation-test the downloaded apworlds, and the core ones.
Core worlds ship with Archipelago, so nothing was downloaded for
them and they are named rather than mounted, but they run the
same five modes and their result is measured rather than
assumed.
A narrowed pass tests what it narrowed to and nothing else:
--fetch bounded the download stage from the start, and leaving
the test stage unbounded meant a "cheap, bounded pass" still
generated against every apworld on disk.
"""
only = self.selected_only(games, core_games)
if only is not None:
print(f"{len(only)} selected for testing "
f"(stability={self.stability!r}, fetch={self.fetch!r})")
with self.stage("generation_tests"):
results = Batch(
config=self.config,
paths=self.paths,
settings=self.settings,
image=image,
tag=tag,
force=changed or self.force_retest,
on_progress=Reporter(self.config, bar=True),
jobs=self.jobs,
core_games=[game.name for game in core_games],
only=only,
exact_only=only is not None,
).run()
summary = Results(results, self.config).summary
self.report["test_counts"] = summary
self.report["test_results"] = results
print(summary)
return results
def place_apworlds(self, tag):
"""Put the worlds that passed into the Archipelago install.
Last, because it acts on what the tests just found. Off by
default: an install is someone's own, so it is only written to
when update_archipelago says so, and then only its
custom_worlds directory.
"""
worlds = CustomWorlds(self.config, self.paths, self.store, tag)
with self.stage("custom_worlds"):
result = worlds.update()
self.report["custom_worlds"] = result
if not result["updated"]:
print(f"not updating any install - {result['reason']}")
return
print(f"{len(result['placed'])} apworlds in {result['directory']}, "
f"{len(result['withdrawn'])} withdrawn")
if result["collisions"]:
print(f"{len(result['collisions'])} name collision(s) held "
f"back: {', '.join(sorted(result['collisions']))}")
def run(self):
"""Every stage, in order, returning what each one did."""
tag, changed = self.update_archipelago()
image = self.build.test_image_tag(tag)
games, core_games, sheet_names = self.read_sheet()
self.download(games, sheet_names, image, tag)
self.test(image, tag, games, core_games, changed)
self.place_apworlds(tag)
return self.report