Skilly/APWorldTester
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

initial commit

Browse Source
This commit is contained in:
Skilly
2026-09-06 15:12:38 +02:00
parent 4caf096780
commit 0d77a07291
185 changed files with 15465 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
container/Dockerfile.test Normal file
View File

@@ -0,0 +1,20 @@
# Unprivileged wrapper for the generation tests.
#
# Upstream's image runs as root and leaves /.local root-owned, so running
# it as the host user fails before generation starts (PermissionError on
# /.local) and running it as root leaves root-owned seeds in the mounted
# output directory. Two lines fix both, and they are the only difference
# from the base image.
#
# Built once per Archipelago tag, not per apworld test, so the cost of
# the recursive chown is paid once and amortised over a few hundred
# generations.
ARG BASE_IMAGE=archipelago:latest
FROM ${BASE_IMAGE}
ARG RUNNER_UID=1000
ARG RUNNER_GID=1000
RUN chown -R ${RUNNER_UID}:${RUNNER_GID} /app /.local 2>/dev/null || chown -R ${RUNNER_UID}:${RUNNER_GID} /app
USER ${RUNNER_UID}:${RUNNER_GID}

49
container/drivers/identify_apworld.py Normal file
View File

@@ -0,0 +1,49 @@
"""Runs inside the archipelago Docker image. Reports which game (if any)
a single mounted .apworld registers itself as under Archipelago's own
World registry - the import/registration step only, no seed generation.
Used to verify a freshly downloaded .apworld's actual content matches
the sheet row that fetched it, rather than trusting release/asset
name-matching alone (see AssetMatcher): the filename
and release title are just text an author chose, but AutoWorldRegister
is filled from the World class's own "game" attribute, which is what
Archipelago itself will treat this file as - the authoritative source.
"""
import argparse
import json
import sys
import warnings
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--apworld", required=True)
args = parser.parse_args()
warnings.simplefilter("ignore")
try:
import worlds
from worlds.AutoWorld import AutoWorldRegister
except BaseException as error:
# The apworld (or something it imports) crashed outright - that's
# itself a meaningful signal (this file is broken/wrong), not an
# infrastructure failure, so it's still reported as a clean JSON
# result rather than a non-zero exit / stack trace.
print(json.dumps({"games": [], "detail": f"{type(error).__name__}: {error}"}))
return 0
games = [
name for name, world_type in AutoWorldRegister.world_types.items()
if str(getattr(world_type, "zip_path", "") or "") == args.apworld
]
print(json.dumps({
"games": games,
"detail": None if games else f"failed_world_loads={worlds.failed_world_loads}",
}))
return 0
if __name__ == "__main__":
sys.exit(main())

827
container/drivers/run_test.py Normal file
View File

@@ -0,0 +1,827 @@
"""Runs inside the archipelago Docker image. Generates one seed for a
single installed apworld and reports the outcome as one JSON line on
stdout: passed, needs_input (missing base ROM or similar), or failed.
The logic mirrors Archipelago's own test/hosting/generate.py, but is
copied here rather than imported from it: the project's .dockerignore
excludes test/ from the image, so that module isn't present at runtime.
"""
import argparse
import io
import json
import logging
import random
import re
import shutil
import sys
import unicodedata
import time
import warnings
from multiprocessing import Manager, Process, set_start_method
from pathlib import Path
def ensure_tutorials_default():
# Monkeypatch applied AFTER whichever Archipelago version's own real
# worlds/AutoWorld.py has already loaded normally - defaults
# WebWorld.tutorials to [] if that core doesn't already declare it,
# without needing to know or duplicate anything else about that
# core's own AutoWorld.py content. Previously this was a full-file
# bind-mount replacement (container/AutoWorld.py (absent - see config.py)) built against one
# specific version's source; that broke outright when tested against
# an older Archipelago release whose real AutoWorld.py imports
# differently (e.g. no rule_builder module before it was added
# upstream) - replacing the whole file assumes it stays
# source-compatible with everything else that release's core
# expects, which isn't true across versions. This achieves the same
# fix (see the original container/AutoWorld.py (absent - see config.py) comment for why it's
# needed - WebHost.py's invalid_worlds filter vs.
# network_data_package snapshot timing) without that assumption.
from worlds.AutoWorld import WebWorld
if "tutorials" not in vars(WebWorld):
WebWorld.tutorials = []
class WebHostIncompatibleError(RuntimeError):
"""Generation succeeds, but this world would crash WebHost's own
startup - a bug class Generate.py never exercises (see
check_webhost_compatibility below)."""
def check_webhost_compatibility(game):
# Reproduces WebHost's own generate_yaml_templates() (Options.py),
# scoped to just this one world, so a bug like Option.visibility
# being set to a bare int instead of a Visibility flag - which
# crashes generate_yaml_templates() for the ENTIRE deployed site,
# not just the offending world, since it iterates every visible
# world in one loop - is caught here as a normal test failure
# instead of only in production. (A hand-run diagnostic that located
# the offending worlds used to live in container/tools/; removed.)
# which found this bug class live on the deployed instance.
from inspect import cleandoc
import yaml
from jinja2 import Template
import Options
from Utils import local_path, __version__
from worlds.AutoWorld import AutoWorldRegister
world = AutoWorldRegister.world_types[game]
if not hasattr(world.web, "tutorials"):
# WebHost.py strips any world missing this from AutoWorldRegister
# entirely at startup (its invalid_worlds set) - so it can't
# crash generate_yaml_templates (never part of that loop), but
# the world is now gone from WebHost's registry altogether. That
# breaks something worse than templates: MultiServer can't find
# the game's data (item_name_groups etc.) when starting a room,
# so *hosting* any seed generated with this world crashes with
# a KeyError - even though generation itself succeeded fine.
raise WebHostIncompatibleError(
"passes generation but has no WebWorld.tutorials - WebHost "
"strips it from its world registry entirely at startup, so "
"hosting a room for a seed generated with it crashes "
"MultiServer with a KeyError on the game name"
)
def dictify_range(option, option_val):
data = {option_val: 50}
for sub_option in ["random", "random-low", "random-high",
f"random-range-{option.range_start}-{option.range_end}"]:
if sub_option != option_val:
data[sub_option] = 0
notes = {}
for name, number in getattr(option, "special_range_names", {}).items():
notes[name] = f"equivalent to {number}"
if number in data:
data[name] = data[number]
del data[number]
elif name in data:
pass
else:
data[name] = 0
return data, notes
def yaml_dump_scalar(scalar):
return yaml.dump(scalar).replace("...\n", "").strip()
with open(local_path("data", "options.yaml")) as f:
template = Template(f.read())
try:
option_groups = Options.get_option_groups(world)
presets = world.web.options_presets.copy()
presets.update({"": {}})
for name, preset in presets.items():
template.render(
option_groups=option_groups,
__version__=__version__,
game=game,
world_version=world.world_version.as_simple_string(),
yaml_dump=yaml_dump_scalar,
dictify_range=dictify_range,
cleandoc=cleandoc,
preset_name=name,
preset=preset,
)
except Exception as error:
raise WebHostIncompatibleError(
f"passes generation but would crash WebHost's own startup "
f"(generate_yaml_templates): {type(error).__name__}: {error}"
) from error
def core_worlds(exclude):
"""Every world Archipelago ships, minus the one under test.
Core worlds are the safe companions: they are part of the build
itself, so they are always present and always match the running
version - unlike an apworld, which might be missing or broken for
reasons that have nothing to do with the world being tested.
"""
from worlds.AutoWorld import AutoWorldRegister
return sorted(
name for name, world_type in AutoWorldRegister.world_types.items()
if not str(getattr(world_type, "zip_path", "") or "") and name != exclude
)
def random_options(game):
"""One concrete random value per option of `game`.
Only the option types with a defined value space are randomized -
toggles, choices and ranges. Free text, option lists/sets and item
dicts (start inventory, plando, exclusions...) have no meaningful
"random", and forcing one would fail generation for reasons that have
nothing to do with the world being tested.
The values are RESOLVED here rather than written into the yaml as the
string "random". Both randomized modes then run the exact same option
values, so a difference between them is attributable to the extra
players and nothing else - writing "random" would have each mode roll
its own values from its own seed, and SR-passes-while-MR-fails would
say nothing about the multiworld.
Resolution goes through the option's own from_text("random"), so the
values are exactly the ones Archipelago itself would roll: weighted
ranges, named-range specials and all.
"""
from worlds.AutoWorld import AutoWorldRegister
import Options
randomizable = (Options.Toggle, Options.Choice, Options.Range)
world_type = AutoWorldRegister.world_types[game]
chosen = {}
for name, option in getattr(world_type.options_dataclass, "type_hints", {}).items():
if not issubclass(option, randomizable):
continue
try:
chosen[name] = option.from_text("random").value
except Exception:
# A world can define an option whose randomization raises;
# leaving it out means it keeps its default rather than
# failing the whole mode for one unrollable option.
continue
return chosen
def generate_one(games, dest, results, options_per_game=None, seed=None, spoiler=3):
# games is the full player list: the world under test first, then any
# companions (see RunSettings). One entry is the single-player
# test, the same game twice is the duplicate test, and the world plus
# companions is the multi-game test - all the same code path, differing
# only in who is in the seed and what options they roll.
warnings.simplefilter("ignore")
try:
from tempfile import TemporaryDirectory
import Generate
import Main
ensure_tutorials_default()
with TemporaryDirectory() as players_dir, TemporaryDirectory() as output_dir:
for number, game in enumerate(games, start=1):
player_path = Path(players_dir) / f"{number}.yaml"
# {} means every option at its default - the only point in
# option space the pipeline used to cover.
game_options = (options_per_game or {}).get(game, {})
player_path.write_text(json.dumps({
"name": f"Tester{number}",
"game": game,
game: game_options,
"description": f"ArchiUpdater compatibility test: {game}",
}), encoding="utf-8")
sys.argv = [
sys.argv[0],
"--player_files_path", players_dir,
"--outputpath", output_dir,
# Spoiler 3 computes the full playthrough, which exercises
# accessibility/reachability logic that plain generation
# never reaches - "a seed came out" becomes "the seed is
# actually completable".
"--spoiler", str(spoiler),
]
if seed is not None:
sys.argv += ["--seed", str(seed)]
Main.main(*Generate.main())
check_webhost_compatibility(games[0])
output_files = list(Path(output_dir).glob("*.zip"))
if len(output_files) != 1:
raise RuntimeError(
f"expected exactly one output file, found {len(output_files)}")
final_file = Path(dest) / output_files[0].name
# rename() can't cross the tmpdir -> bind-mounted output volume
# boundary, so move (copy + delete) instead.
shutil.move(str(output_files[0]), str(final_file))
results.append(str(final_file))
except BaseException as error:
results.append(error)
raise
def describe_load_failure(log_text, apworld_path):
"""The real reason an apworld did not load, out of the swallowed log.
worlds/__init__.py catches an import failure per world, formats the
traceback, and hands it to logging.exception - so by default it goes
nowhere and the pipeline could only report "found 0 worlds", which
says nothing an author could act on. Untitled Goose Game, for
instance, defines `option_random` on a Choice, which Archipelago
reserves; the assert fires at class-definition time during import.
Returns (summary, traceback) - summary being the exception and the
line in the world's own code that raised it.
"""
marker = "Could not load world"
TRACEBACK_HEADER = "Traceback (most recent call last):"
blocks = [block for block in log_text.split(marker) if apworld_path in block]
if not blocks:
return None, None
block = marker + blocks[-1]
lines = [line for line in block.splitlines() if line.strip()]
# The exception itself is the last line of a traceback.
exception = next((line.strip() for line in reversed(lines)
if line.strip() and not line.startswith((" ", "\t"))
and "Traceback" not in line and marker not in line), None)
# The deepest frame inside the apworld itself - core frames above it
# are just the import machinery and tell an author nothing.
where = None
for line in lines:
match = re.search(r'File "([^"]*%s[^"]*)", line (\d+)' % re.escape(apworld_path), line)
if match:
inside = match.group(1).split(apworld_path + "/", 1)[-1]
where = f"{inside}, line {match.group(2)}"
summary = exception or "import failed"
if where:
summary += f" (in {where})"
# logging.exception both formats the traceback into the message AND
# appends exc_info's own copy, so the block holds it twice - keep one.
first = block.find(TRACEBACK_HEADER)
if first != -1:
second = block.find(TRACEBACK_HEADER, first + len(TRACEBACK_HEADER))
if second != -1:
block = block[:second]
# Drop the import machinery. Every one of these frames is identical
# for every failing world and tells an author nothing; what is left is
# their own code and the line that actually raised.
# A frame is its File line plus the indented source and caret lines
# under it, so drop the whole frame - removing only the File line
# leaves orphaned source fragments behind.
noise = ("<frozen importlib", "importlib/__init__.py", "worlds/__init__.py", "_bootstrap")
kept = []
skipping = False
for line in block.splitlines():
stripped = line.lstrip()
if stripped.startswith("File "):
skipping = any(n in line for n in noise)
if skipping:
continue
elif skipping:
# Continuation of a dropped frame: still indented, and not the
# start of the next frame.
if line.startswith(" ") or not line.strip():
continue
skipping = False
kept.append(line)
return summary, "\n".join(kept)
def rooted_in(error, exception_type):
seen = set()
while error is not None and id(error) not in seen:
seen.add(id(error))
if isinstance(error, exception_type):
return True
error = error.__cause__ or error.__context__
return False
def rooted_in_named(error, class_name):
# Some apworlds define their own exception class inside their own
# zipimported package (e.g. CTJoT's InvalidYamlException) rather
# than a stable top-level module - there's no fixed path to import
# and isinstance-check it the way Fill.FillError works, so match by
# class name instead.
seen = set()
while error is not None and id(error) not in seen:
seen.add(id(error))
if type(error).__name__ == class_name:
return True
error = error.__cause__ or error.__context__
return False
def run_with_timeout(games, dest, timeout, options_per_game=None, seed=None, spoiler=3):
# fork, not spawn. Every mode runs in its own process so a generation
# can be killed on timeout and cannot leak state into the next one -
# but spawn re-imports the whole of Archipelago in each child, five
# times per apworld, which measured as the single largest cost in a
# run. Forking inherits the import the parent already did.
#
# The isolation that matters is unchanged: the child still gets its
# own copy-on-write memory, so whatever generation mutates dies with
# it and the parent's registry stays pristine for the next mode.
#
# Safe here specifically because the forking process is
# single-threaded: main() starts no threads of its own, and the
# Manager below is a separate process rather than a thread in this
# one. Forking a multi-threaded parent is the case to avoid, and this
# is not one.
try:
set_start_method("fork")
except RuntimeError:
pass
manager = Manager()
results = manager.list()
process = Process(target=generate_one,
args=(games, dest, results, options_per_game, seed, spoiler))
started = time.monotonic()
process.start()
process.join(timeout)
timed_out = process.is_alive()
if timed_out:
process.terminate()
process.join(5)
if process.is_alive():
process.kill()
process.join()
elapsed = time.monotonic() - started
return list(results), timed_out, elapsed
def classify(results, timed_out, elapsed, single_player=True, randomized=False):
if timed_out:
return "failed", f"timed out after {elapsed:.0f}s"
if not results:
return "failed", "generation process exited without a result"
result = results[0]
if isinstance(result, BaseException):
if rooted_in(result, FileNotFoundError):
return "needs_input", str(result)
if randomized and rooted_in_named(result, "OptionError"):
# The world rejected the option combination we rolled - not a
# crash and not a fill failure, but the world validating its
# own input. Archipelago rolls every option independently, so
# a world with interdependent options (Blender's min/max
# similarity percent, for example) can always be handed a
# combination it considers invalid. A player writing
# "random" for both hits exactly this, so it is worth
# reporting - but as its own thing, not as a defect.
#
# Only when the options were randomized: a world that rejects
# its OWN DEFAULTS is genuinely broken and stays "failed".
return "invalid_options", f"rejected the rolled options: {result}"
from Fill import FillError
if rooted_in(result, FillError):
# A fill failure is a failure in both modes. This used to be
# excused in single-player on the theory that some games'
# item/location balance "only works out in a real multiworld"
# - the multi-game test exists precisely to check that theory,
# and it does not hold (Dead Cells fails both ways), so the
# excuse is gone: a seed that cannot be filled is a seed that
# cannot be generated, whoever else is in it.
where = "a multi-game seed" if not single_player else "a single-player seed"
return "failed", f"fill failed in {where}: {result}"
if rooted_in_named(result, "InvalidYamlException"):
# Some worlds (e.g. CTJoT) require a yaml pre-generated by an
# external tool and reject our generic single-player yaml
# outright - not a defect, just incompatible with this
# testing methodology.
return "passed", f"requires an externally-generated yaml (not a defect): {result}"
return "failed", f"{type(result).__name__}: {result}"
return "passed", str(result)
def resolve_core_game(name, world_types):
"""Match a sheet name to a world Archipelago actually registers.
The sheet and the worlds disagree about wording often enough that an
exact comparison finds only 72 of the 81 core rows. The rest differ
by an accent ("Pokemon Emerald"), a hyphen ("Choo-Choo Charles"),
capitalisation ("EarthBound"), or a series prefix the sheet adds and
the world does not ("The Legend of Zelda: Ocarina of Time").
Tried in order: the name as written, then folded to letters and
digits, then either side of a colon folded the same way. Either side,
because the sheet puts the distinguishing part before the colon as
often as after it - "Super Mario Land 2: The Golden Coins" is
registered as "Super Mario Land 2", while "The Legend of Zelda:
Ocarina of Time" is registered as "Ocarina of Time".
"""
if name in world_types:
return name
by_slug = {}
for registered in world_types:
by_slug.setdefault(slug(registered), registered)
candidates = [name]
if ":" in name:
head, tail = name.split(":", 1)
candidates += [tail, head]
for candidate in candidates:
registered = by_slug.get(slug(candidate))
if registered:
return registered
return None
# Deliberate copy of Name.slug in core/model/name.py. This file is mounted
# alone into the container, with no access to this checkout's
# packages, so it cannot import it - but the two must stay identical
# or a world folds to one key here and a different one on the host.
def slug(text):
return re.sub(r"[^a-z0-9]", "", unicodedata.normalize("NFKD", (text or "").lower()))
def main():
parser = argparse.ArgumentParser(description=__doc__)
# Exactly one of these. --apworld is the normal case: a downloaded
# file, identified by which world claims it as its zip_path. --game
# names a world that is already part of the Archipelago build, which
# has no file to point at and so cannot be identified that way.
parser.add_argument("--apworld")
parser.add_argument("--game", help="Test a world that ships with Archipelago, by name, "
"instead of an apworld file. Mutually exclusive with --apworld.")
parser.add_argument("--output-dir", required=True)
parser.add_argument("--timeout", type=int, default=300)
parser.add_argument("--companion-pool", default="",
help="Comma-separated core games to draw companions from. Empty "
"means every core world, which risks drawing one that cannot "
"generate here (see container/verify_companions.py).")
parser.add_argument("--companion-min", type=int, default=2,
help="Fewest core games to draw as companions for a multi-game seed.")
parser.add_argument("--companion-max", type=int, default=5,
help="Most core games to draw as companions for a multi-game seed.")
parser.add_argument("--random-repeats", type=int, default=3,
help="How many times the randomized-option modes run. Each attempt "
"rolls a different option combination.")
parser.add_argument("--repeats", type=int, default=10,
help="How many times to run the single-game and multi-game tests. "
"Stability is a pass rate, not a single verdict.")
parser.add_argument("--spoiler", type=int, default=3,
help="Spoiler level passed to Generate. 3 computes the full "
"playthrough, exercising accessibility logic.")
args = parser.parse_args()
if bool(args.apworld) == bool(args.game):
parser.error("pass exactly one of --apworld or --game")
# Determine the game name from the loaded World class itself rather
# than the apworld's archipelago.json manifest: older apworlds don't
# ship a manifest at all, and the core (<0.7.0 here) tolerates that
# and loads them anyway - a manifest-only lookup would misreport a
# perfectly loadable world as failed. World.zip_path is set only for
# worlds loaded from a .apworld zip (worlds/AutoWorld.py), and equals
# the apworld's own path, so it identifies our one mounted world
# exactly regardless of manifest presence.
# worlds/__init__.py logs each failed world's traceback and carries
# on; without a handler here that detail is lost and all the pipeline
# can say is "found 0 worlds".
load_log = io.StringIO()
log_handler = logging.StreamHandler(load_log)
log_handler.setLevel(logging.ERROR)
logging.getLogger().addHandler(log_handler)
try:
import worlds
from worlds.AutoWorld import AutoWorldRegister
finally:
logging.getLogger().removeHandler(log_handler)
ensure_tutorials_default()
if args.game:
# A core world is already loaded - there is no file to identify
# it by, and none of the load-failure reporting below applies. It
# either exists in this build or it does not, and "does not" is a
# naming problem on the sheet rather than a defect in a world.
resolved = resolve_core_game(args.game, AutoWorldRegister.world_types)
if resolved:
args.game = resolved
else:
# "unknown", not "unverified". Nothing about this world is
# broken - this Archipelago build simply has no world under
# that name, which is a naming mismatch on the sheet (or a
# world that has since been removed from core), and calling
# it Broken would blame a world for the sheet's wording.
print(json.dumps({
"game": None,
"outcome": "unknown",
"detail": (f"this Archipelago build registers no world named {args.game!r}, "
f"so there was nothing to test. The sheet lists this row as Core, "
f"so either its name differs from the world's own or the world is "
f"no longer part of Archipelago."),
"elapsed_seconds": 0,
}))
return 1
custom_games = [args.game]
else:
custom_games = [
name for name, world_type in AutoWorldRegister.world_types.items()
if str(getattr(world_type, "zip_path", "") or "") == args.apworld
]
if len(custom_games) != 1:
# Its own outcome, NOT "passed". This covers several distinct
# causes (a duplicate game name against a now-built-in world, an
# import-time crash, a file that isn't a valid apworld at all,
# a genuine collision between two unrelated apworlds) - none of
# them necessarily an apworld defect, but none of them a world
# this pipeline has verified either. Calling it "passed" promoted
# files that never loaded and crash-looped real WebHost startup
# Factorio Platformer did exactly that: not a valid zip, zero
# worlds loaded, recorded as a pass, promoted, and crash-looping
# real WebHost startup - it needed a hand-written blacklist entry
# to undo. promote.py takes "passed" alone, so this verdict now
# keeps such a file out of worlds_live/ on its own.
summary, traceback_text = describe_load_failure(load_log.getvalue(), args.apworld)
if len(custom_games) > 1:
detail = (f"this file registers {len(custom_games)} worlds, not one: "
f"{', '.join(custom_games)}. Only a single-world apworld can be "
f"promoted, since the pipeline cannot tell which one this row means.")
elif summary:
detail = f"the world failed to import: {summary}"
if traceback_text:
detail += f"\n\n{traceback_text.strip()}"
else:
# No captured traceback: the file is not a loadable apworld at
# all (not a zip, wrong layout), so nothing was even attempted.
detail = (f"no world loaded from this file and no import was attempted - it is "
f"probably not a valid .apworld (failed_world_loads="
f"{worlds.failed_world_loads})")
print(json.dumps({
"game": None,
"outcome": "unverified",
"detail": detail,
"elapsed_seconds": 0,
}))
return 1
game = custom_games[0]
Path(args.output_dir).mkdir(parents=True, exist_ok=True)
def run_mode(games, options_per_game=None, seed=None, single_player=None, randomized=False):
if single_player is None:
single_player = len(games) == 1
mode_results, timed_out, elapsed = run_with_timeout(
games, args.output_dir, args.timeout,
options_per_game=options_per_game, seed=seed, spoiler=args.spoiler)
outcome, detail = classify(mode_results, timed_out, elapsed,
single_player=single_player, randomized=randomized)
return {"outcome": outcome, "detail": detail, "elapsed_seconds": round(elapsed, 1),
"timed_out": timed_out}
# A timeout anywhere abandons the rest of this apworld. One world that
# hangs would otherwise cost timeout x repeats x modes - by far the
# largest thing in a run - and a world that cannot finish one seed has
# nothing useful to say about its stability across ten.
state = {"timed_out": False}
def skipped(reason):
return {"outcome": "unknown", "detail": reason, "elapsed_seconds": None}
def run_repeated(players, repeats, single_player=None, options=None, randomized=False):
"""The same test several times, reported as a pass rate.
Stability is not something a single generation can tell you: S, D
and M each get a different Archipelago seed, so the fill differs
every time, and a world that places items successfully four times
in five is genuinely unstable. Running once records whichever
result we happened to get.
Each repeat uses an explicit seed so a failure among ten passes is
reproducible - without it the interesting case is the one that
cannot be re-examined.
"""
attempts = []
for _ in range(max(repeats, 1)):
seed = random.randint(0, 2 ** 31 - 1)
# players is a callable so the multi-game test can draw a
# different set of companions every attempt - ten runs against
# one fixed pair only ever tests that pair.
games = players() if callable(players) else players
# A callable, so the randomized modes can roll a different
# point in the option space for every attempt - repeating one
# roll would only re-measure the same combination.
per_game = options() if callable(options) else options
result = run_mode(games, options_per_game=per_game, seed=seed,
single_player=single_player, randomized=randomized)
result["seed"] = seed
if len(games) > 1:
result["companions"] = games[1:]
attempts.append(result)
if result["timed_out"]:
state["timed_out"] = True
break
passed = [a for a in attempts if a["outcome"] == "passed"]
failed = [a for a in attempts if a["outcome"] != "passed"]
if not failed:
outcome, detail = "passed", f"generated {len(passed)}/{len(attempts)} times"
elif not passed:
# Every attempt agreeing on a non-failure reason keeps that
# reason. A world that needs a base ROM reports needs_input
# once and "failed all 10 attempts" ten times over, and
# flattening those to "failed" blames the world for a file
# this pipeline does not have - which matters most for core
# worlds, where roughly seven of them need a ROM and would
# otherwise all read as Broken.
outcomes = {attempt["outcome"] for attempt in attempts}
outcome = outcomes.pop() if len(outcomes) == 1 else "failed"
detail = f"failed all {len(attempts)} attempts. {failed[0]['detail']}"
if outcome != "failed":
detail = (f"{outcome} on all {len(attempts)} attempts. "
f"{failed[0]['detail']}")
else:
# The whole point of repeating: a world that works sometimes
# is worse for a player than one that never does, because
# nobody knows to avoid it.
outcome = "flaky"
detail = (f"generated {len(passed)}/{len(attempts)} times. "
f"First failure (seed {failed[0]['seed']}): {failed[0]['detail']}")
return {
"outcome": outcome,
"detail": detail,
"elapsed_seconds": round(sum(a["elapsed_seconds"] or 0 for a in attempts), 1),
"attempts": len(attempts),
# What was asked for, as opposed to what ran - a timeout cuts
# a mode short, and the retest cache compares against this so
# such a record is not re-run forever.
"requested_attempts": max(repeats, 1),
"passed_attempts": len(passed),
"failed_seeds": [a["seed"] for a in failed],
# Which companions each failing attempt drew, so a failure
# caused by one particular combination can be reproduced.
"failed_companions": [a.get("companions") for a in failed if a.get("companions")],
}
tests = {}
# 1. the world on its own, every option at its default.
tests["single"] = run_repeated([game], args.repeats)
tests["single"]["label"] = "Single game"
# 2. the same world twice. A world that keeps state on the class
# rather than the instance works alone and corrupts itself here -
# a bug class no other mode can see.
tests["duplicate"] = (skipped("skipped - an earlier test timed out") if state["timed_out"]
else run_mode([game, game], single_player=False))
tests["duplicate"]["label"] = "Two players, same game"
# 3. the world alongside other games. Each attempt draws a fresh
# random selection of core worlds rather than reusing one fixed
# pair: ten runs against the same two companions only ever tell you
# about those two, while a different combination each time samples
# what a real multiworld looks like.
configured_pool = [name.strip() for name in args.companion_pool.split(",") if name.strip()]
if configured_pool:
# Verified by the host once per run: every one of these generated
# a seed on its own in this exact environment, so a multi-game
# failure is attributable to the world under test rather than to
# a companion that was never going to work.
pool = [name for name in configured_pool if name != game]
else:
pool = core_worlds(exclude=game)
low = max(args.companion_min, 1)
high = max(args.companion_max, low)
def draw_companions():
count = min(random.randint(low, high), len(pool))
return random.sample(pool, count)
if state["timed_out"]:
tests["multi"] = skipped("skipped - an earlier test timed out")
elif pool:
tests["multi"] = run_repeated(
lambda: [game] + draw_companions(), args.repeats, single_player=False)
tests["multi"]["companion_pool"] = len(pool)
tests["multi"]["companion_range"] = [low, high]
tests["multi"]["detail"] += f" (against {low}-{high} core games drawn from {len(pool)})"
else:
tests["multi"] = skipped("no core worlds available to pair with")
tests["multi"]["label"] = "Multi game"
# 4 and 5. The same two shapes as above - the world alone, and the
# world alongside the companions - but with its options randomized
# instead of left at their defaults. The default option set is a
# single point in a world's option space and most option-dependent
# crashes are nowhere near it.
#
# One roll each, and every roll that does not produce a seed is a
# failure, including one the world rejects with OptionError. That
# last point is the whole reason these modes exist: the yaml a
# player writes is accepted as valid - nothing in the template or
# the upload path objects to "random" on two interdependent
# options - so the conflict only surfaces when generation is
# actually attempted, possibly after everyone has submitted, and it
# is intermittent. Re-rolling until the world accepts something
# would hide exactly the failure a real player hits.
#
# Each randomized run is gated on its own default-options
# counterpart: SR only when the world generates alone, MR only when
# it generates alongside the companions. A randomized run whose
# plain equivalent already fails would fail for the same underlying
# reason and cost a generation to learn nothing.
rolls = None
if state["timed_out"]:
aborted = "skipped - an earlier test timed out"
tests["single_random"] = skipped(aborted)
tests["multi_random"] = skipped(aborted)
elif tests["single"]["outcome"] == "passed" or tests["multi"]["outcome"] == "passed":
try:
# One roll per attempt, drawn up front so SR and MR run the
# SAME combinations as each other. Pairing them is what makes
# "SR passed but MR failed" attributable to the extra players
# rather than to two different option sets.
rolls = [random_options(game) for _ in range(max(args.random_repeats, 1))]
except BaseException as error:
rolls = None
unreadable = f"could not read this world's options: {type(error).__name__}: {error}"
tests["single_random"] = skipped(unreadable)
tests["multi_random"] = skipped(unreadable)
if rolls:
def roller():
# Walks the pre-drawn rolls in order, so attempt N of SR and
# attempt N of MR use the same option values.
supply = iter(rolls)
def next_roll():
return {game: next(supply)}
return next_roll
# "flaky" counts as generating: the world produced a seed at
# least once, so a randomized run can still tell us something.
if tests["single"]["outcome"] in ("passed", "flaky"):
result = run_repeated([game], len(rolls), single_player=True,
options=roller(), randomized=True)
result["randomized_options"] = len(rolls[0])
tests["single_random"] = result
else:
tests["single_random"] = skipped(
"skipped - the single-game test did not pass")
if tests["multi"]["outcome"] in ("passed", "flaky"):
result = run_repeated(lambda: [game] + draw_companions(), len(rolls),
single_player=False, options=roller(), randomized=True)
result["randomized_options"] = len(rolls[0])
tests["multi_random"] = result
else:
tests["multi_random"] = skipped(
"skipped - the multi-game test did not pass")
elif "single_random" not in tests:
neither = "skipped - neither the single-game nor the multi-game test passed"
tests["single_random"] = skipped(neither)
tests["multi_random"] = skipped(neither)
tests["single_random"]["label"] = "Random options, alone"
tests["multi_random"]["label"] = "Random options, multi game"
# The top-level fields stay exactly what they were - the single-player
# verdict - because that is what promotion and every existing consumer
# reads. The per-mode results live alongside it.
report = {
"game": game,
"outcome": tests["single"]["outcome"],
"detail": tests["single"]["detail"],
"elapsed_seconds": tests["single"]["elapsed_seconds"],
"multi_outcome": tests["multi"]["outcome"],
"multi_detail": tests["multi"]["detail"],
"multi_elapsed_seconds": tests["multi"]["elapsed_seconds"],
"multi_companions": tests["multi"].get("companions", []),
"tests": tests,
}
print(json.dumps(report))
# Exit code still reflects the single-player result alone: it is what
# gates promotion, and the other modes are reported rather than
# blocking.
return {"passed": 0, "needs_input": 2}.get(tests["single"]["outcome"], 1)
if __name__ == "__main__":
sys.exit(main())

65
container/drivers/verify_companions.py Normal file
View File

@@ -0,0 +1,65 @@
"""Runs inside the archipelago Docker image. Reports which core worlds
can actually generate a seed in THIS environment, so the multi-game tests
only ever draw companions that work.
Core worlds are shipped with Archipelago, but that does not make them all
usable here: some need a base ROM this container has no copy of, and at
least one (Final Fantasy) needs a settings yaml generated by an external
website and refuses a generic one. Drawing such a world as a companion
fails the whole seed, and the failure gets recorded against the apworld
under test rather than the companion that caused it - Blender was marked
flaky 7/10 purely because three of its ten draws included worlds like
these.
Rather than maintain an exclusion list by hand, this generates one solo
seed per core world and keeps the ones that succeed. A world drops out
by itself when its ROM is missing, and comes back by itself once the ROM
is added.
Shares run_test.py's generation machinery rather than repeating it, so
"can generate" means exactly the same thing in both places.
"""
import argparse
import json
import sys
import warnings
sys.path.insert(0, "/app")
import run_test # noqa: E402 - mounted alongside this script
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output-dir", required=True)
parser.add_argument("--timeout", type=int, default=120,
help="Per-world timeout. Deliberately shorter than the apworld "
"timeout: a companion that is slow to generate would multiply "
"across every multi-game attempt that draws it.")
parser.add_argument("--spoiler", type=int, default=2)
args = parser.parse_args()
warnings.simplefilter("ignore")
run_test.ensure_tutorials_default()
from pathlib import Path
Path(args.output_dir).mkdir(parents=True, exist_ok=True)
verified = []
rejected = {}
for game in run_test.core_worlds(exclude=None):
results, timed_out, elapsed = run_test.run_with_timeout(
[game], args.output_dir, args.timeout, spoiler=args.spoiler)
outcome, detail = run_test.classify(results, timed_out, elapsed)
if outcome == "passed":
verified.append(game)
else:
rejected[game] = f"{outcome}: {detail}"[:300]
print(json.dumps({"verified": verified, "rejected": rejected}))
return 0
if __name__ == "__main__":
sys.exit(main())