828 lines
39 KiB
Python
828 lines
39 KiB
Python
"""Runs inside the archipelago Docker image. Generates one seed for a
|
|
single installed apworld and reports the outcome as one JSON line on
|
|
stdout: passed, needs_input (missing base ROM or similar), or failed.
|
|
|
|
The logic mirrors Archipelago's own test/hosting/generate.py, but is
|
|
copied here rather than imported from it: the project's .dockerignore
|
|
excludes test/ from the image, so that module isn't present at runtime.
|
|
"""
|
|
|
|
import argparse
|
|
import io
|
|
import json
|
|
import logging
|
|
import random
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import unicodedata
|
|
import time
|
|
import warnings
|
|
from multiprocessing import Manager, Process, set_start_method
|
|
from pathlib import Path
|
|
|
|
def ensure_tutorials_default():
|
|
# Monkeypatch applied AFTER whichever Archipelago version's own real
|
|
# worlds/AutoWorld.py has already loaded normally - defaults
|
|
# WebWorld.tutorials to [] if that core doesn't already declare it,
|
|
# without needing to know or duplicate anything else about that
|
|
# core's own AutoWorld.py content. Previously this was a full-file
|
|
# bind-mount replacement (container/AutoWorld.py (absent - see config.py)) built against one
|
|
# specific version's source; that broke outright when tested against
|
|
# an older Archipelago release whose real AutoWorld.py imports
|
|
# differently (e.g. no rule_builder module before it was added
|
|
# upstream) - replacing the whole file assumes it stays
|
|
# source-compatible with everything else that release's core
|
|
# expects, which isn't true across versions. This achieves the same
|
|
# fix (see the original container/AutoWorld.py (absent - see config.py) comment for why it's
|
|
# needed - WebHost.py's invalid_worlds filter vs.
|
|
# network_data_package snapshot timing) without that assumption.
|
|
from worlds.AutoWorld import WebWorld
|
|
if "tutorials" not in vars(WebWorld):
|
|
WebWorld.tutorials = []
|
|
|
|
|
|
class WebHostIncompatibleError(RuntimeError):
|
|
"""Generation succeeds, but this world would crash WebHost's own
|
|
startup - a bug class Generate.py never exercises (see
|
|
check_webhost_compatibility below)."""
|
|
|
|
|
|
def check_webhost_compatibility(game):
|
|
# Reproduces WebHost's own generate_yaml_templates() (Options.py),
|
|
# scoped to just this one world, so a bug like Option.visibility
|
|
# being set to a bare int instead of a Visibility flag - which
|
|
# crashes generate_yaml_templates() for the ENTIRE deployed site,
|
|
# not just the offending world, since it iterates every visible
|
|
# world in one loop - is caught here as a normal test failure
|
|
# instead of only in production. (A hand-run diagnostic that located
|
|
# the offending worlds used to live in container/tools/; removed.)
|
|
# which found this bug class live on the deployed instance.
|
|
from inspect import cleandoc
|
|
|
|
import yaml
|
|
from jinja2 import Template
|
|
|
|
import Options
|
|
from Utils import local_path, __version__
|
|
from worlds.AutoWorld import AutoWorldRegister
|
|
|
|
world = AutoWorldRegister.world_types[game]
|
|
if not hasattr(world.web, "tutorials"):
|
|
# WebHost.py strips any world missing this from AutoWorldRegister
|
|
# entirely at startup (its invalid_worlds set) - so it can't
|
|
# crash generate_yaml_templates (never part of that loop), but
|
|
# the world is now gone from WebHost's registry altogether. That
|
|
# breaks something worse than templates: MultiServer can't find
|
|
# the game's data (item_name_groups etc.) when starting a room,
|
|
# so *hosting* any seed generated with this world crashes with
|
|
# a KeyError - even though generation itself succeeded fine.
|
|
raise WebHostIncompatibleError(
|
|
"passes generation but has no WebWorld.tutorials - WebHost "
|
|
"strips it from its world registry entirely at startup, so "
|
|
"hosting a room for a seed generated with it crashes "
|
|
"MultiServer with a KeyError on the game name"
|
|
)
|
|
|
|
def dictify_range(option, option_val):
|
|
data = {option_val: 50}
|
|
for sub_option in ["random", "random-low", "random-high",
|
|
f"random-range-{option.range_start}-{option.range_end}"]:
|
|
if sub_option != option_val:
|
|
data[sub_option] = 0
|
|
notes = {}
|
|
for name, number in getattr(option, "special_range_names", {}).items():
|
|
notes[name] = f"equivalent to {number}"
|
|
if number in data:
|
|
data[name] = data[number]
|
|
del data[number]
|
|
elif name in data:
|
|
pass
|
|
else:
|
|
data[name] = 0
|
|
return data, notes
|
|
|
|
def yaml_dump_scalar(scalar):
|
|
return yaml.dump(scalar).replace("...\n", "").strip()
|
|
|
|
with open(local_path("data", "options.yaml")) as f:
|
|
template = Template(f.read())
|
|
|
|
try:
|
|
option_groups = Options.get_option_groups(world)
|
|
presets = world.web.options_presets.copy()
|
|
presets.update({"": {}})
|
|
for name, preset in presets.items():
|
|
template.render(
|
|
option_groups=option_groups,
|
|
__version__=__version__,
|
|
game=game,
|
|
world_version=world.world_version.as_simple_string(),
|
|
yaml_dump=yaml_dump_scalar,
|
|
dictify_range=dictify_range,
|
|
cleandoc=cleandoc,
|
|
preset_name=name,
|
|
preset=preset,
|
|
)
|
|
except Exception as error:
|
|
raise WebHostIncompatibleError(
|
|
f"passes generation but would crash WebHost's own startup "
|
|
f"(generate_yaml_templates): {type(error).__name__}: {error}"
|
|
) from error
|
|
|
|
|
|
def core_worlds(exclude):
|
|
"""Every world Archipelago ships, minus the one under test.
|
|
|
|
Core worlds are the safe companions: they are part of the build
|
|
itself, so they are always present and always match the running
|
|
version - unlike an apworld, which might be missing or broken for
|
|
reasons that have nothing to do with the world being tested.
|
|
"""
|
|
from worlds.AutoWorld import AutoWorldRegister
|
|
return sorted(
|
|
name for name, world_type in AutoWorldRegister.world_types.items()
|
|
if not str(getattr(world_type, "zip_path", "") or "") and name != exclude
|
|
)
|
|
|
|
|
|
def random_options(game):
|
|
"""One concrete random value per option of `game`.
|
|
|
|
Only the option types with a defined value space are randomized -
|
|
toggles, choices and ranges. Free text, option lists/sets and item
|
|
dicts (start inventory, plando, exclusions...) have no meaningful
|
|
"random", and forcing one would fail generation for reasons that have
|
|
nothing to do with the world being tested.
|
|
|
|
The values are RESOLVED here rather than written into the yaml as the
|
|
string "random". Both randomized modes then run the exact same option
|
|
values, so a difference between them is attributable to the extra
|
|
players and nothing else - writing "random" would have each mode roll
|
|
its own values from its own seed, and SR-passes-while-MR-fails would
|
|
say nothing about the multiworld.
|
|
|
|
Resolution goes through the option's own from_text("random"), so the
|
|
values are exactly the ones Archipelago itself would roll: weighted
|
|
ranges, named-range specials and all.
|
|
"""
|
|
from worlds.AutoWorld import AutoWorldRegister
|
|
import Options
|
|
|
|
randomizable = (Options.Toggle, Options.Choice, Options.Range)
|
|
world_type = AutoWorldRegister.world_types[game]
|
|
chosen = {}
|
|
for name, option in getattr(world_type.options_dataclass, "type_hints", {}).items():
|
|
if not issubclass(option, randomizable):
|
|
continue
|
|
try:
|
|
chosen[name] = option.from_text("random").value
|
|
except Exception:
|
|
# A world can define an option whose randomization raises;
|
|
# leaving it out means it keeps its default rather than
|
|
# failing the whole mode for one unrollable option.
|
|
continue
|
|
return chosen
|
|
|
|
|
|
def generate_one(games, dest, results, options_per_game=None, seed=None, spoiler=3):
|
|
# games is the full player list: the world under test first, then any
|
|
# companions (see RunSettings). One entry is the single-player
|
|
# test, the same game twice is the duplicate test, and the world plus
|
|
# companions is the multi-game test - all the same code path, differing
|
|
# only in who is in the seed and what options they roll.
|
|
warnings.simplefilter("ignore")
|
|
try:
|
|
from tempfile import TemporaryDirectory
|
|
|
|
import Generate
|
|
import Main
|
|
|
|
ensure_tutorials_default()
|
|
|
|
with TemporaryDirectory() as players_dir, TemporaryDirectory() as output_dir:
|
|
for number, game in enumerate(games, start=1):
|
|
player_path = Path(players_dir) / f"{number}.yaml"
|
|
# {} means every option at its default - the only point in
|
|
# option space the pipeline used to cover.
|
|
game_options = (options_per_game or {}).get(game, {})
|
|
player_path.write_text(json.dumps({
|
|
"name": f"Tester{number}",
|
|
"game": game,
|
|
game: game_options,
|
|
"description": f"ArchiUpdater compatibility test: {game}",
|
|
}), encoding="utf-8")
|
|
|
|
sys.argv = [
|
|
sys.argv[0],
|
|
"--player_files_path", players_dir,
|
|
"--outputpath", output_dir,
|
|
# Spoiler 3 computes the full playthrough, which exercises
|
|
# accessibility/reachability logic that plain generation
|
|
# never reaches - "a seed came out" becomes "the seed is
|
|
# actually completable".
|
|
"--spoiler", str(spoiler),
|
|
]
|
|
if seed is not None:
|
|
sys.argv += ["--seed", str(seed)]
|
|
Main.main(*Generate.main())
|
|
check_webhost_compatibility(games[0])
|
|
|
|
output_files = list(Path(output_dir).glob("*.zip"))
|
|
if len(output_files) != 1:
|
|
raise RuntimeError(
|
|
f"expected exactly one output file, found {len(output_files)}")
|
|
final_file = Path(dest) / output_files[0].name
|
|
# rename() can't cross the tmpdir -> bind-mounted output volume
|
|
# boundary, so move (copy + delete) instead.
|
|
shutil.move(str(output_files[0]), str(final_file))
|
|
results.append(str(final_file))
|
|
except BaseException as error:
|
|
results.append(error)
|
|
raise
|
|
|
|
|
|
def describe_load_failure(log_text, apworld_path):
|
|
"""The real reason an apworld did not load, out of the swallowed log.
|
|
|
|
worlds/__init__.py catches an import failure per world, formats the
|
|
traceback, and hands it to logging.exception - so by default it goes
|
|
nowhere and the pipeline could only report "found 0 worlds", which
|
|
says nothing an author could act on. Untitled Goose Game, for
|
|
instance, defines `option_random` on a Choice, which Archipelago
|
|
reserves; the assert fires at class-definition time during import.
|
|
|
|
Returns (summary, traceback) - summary being the exception and the
|
|
line in the world's own code that raised it.
|
|
"""
|
|
marker = "Could not load world"
|
|
TRACEBACK_HEADER = "Traceback (most recent call last):"
|
|
blocks = [block for block in log_text.split(marker) if apworld_path in block]
|
|
if not blocks:
|
|
return None, None
|
|
block = marker + blocks[-1]
|
|
lines = [line for line in block.splitlines() if line.strip()]
|
|
|
|
# The exception itself is the last line of a traceback.
|
|
exception = next((line.strip() for line in reversed(lines)
|
|
if line.strip() and not line.startswith((" ", "\t"))
|
|
and "Traceback" not in line and marker not in line), None)
|
|
|
|
# The deepest frame inside the apworld itself - core frames above it
|
|
# are just the import machinery and tell an author nothing.
|
|
where = None
|
|
for line in lines:
|
|
match = re.search(r'File "([^"]*%s[^"]*)", line (\d+)' % re.escape(apworld_path), line)
|
|
if match:
|
|
inside = match.group(1).split(apworld_path + "/", 1)[-1]
|
|
where = f"{inside}, line {match.group(2)}"
|
|
|
|
summary = exception or "import failed"
|
|
if where:
|
|
summary += f" (in {where})"
|
|
|
|
# logging.exception both formats the traceback into the message AND
|
|
# appends exc_info's own copy, so the block holds it twice - keep one.
|
|
first = block.find(TRACEBACK_HEADER)
|
|
if first != -1:
|
|
second = block.find(TRACEBACK_HEADER, first + len(TRACEBACK_HEADER))
|
|
if second != -1:
|
|
block = block[:second]
|
|
|
|
# Drop the import machinery. Every one of these frames is identical
|
|
# for every failing world and tells an author nothing; what is left is
|
|
# their own code and the line that actually raised.
|
|
# A frame is its File line plus the indented source and caret lines
|
|
# under it, so drop the whole frame - removing only the File line
|
|
# leaves orphaned source fragments behind.
|
|
noise = ("<frozen importlib", "importlib/__init__.py", "worlds/__init__.py", "_bootstrap")
|
|
kept = []
|
|
skipping = False
|
|
for line in block.splitlines():
|
|
stripped = line.lstrip()
|
|
if stripped.startswith("File "):
|
|
skipping = any(n in line for n in noise)
|
|
if skipping:
|
|
continue
|
|
elif skipping:
|
|
# Continuation of a dropped frame: still indented, and not the
|
|
# start of the next frame.
|
|
if line.startswith(" ") or not line.strip():
|
|
continue
|
|
skipping = False
|
|
kept.append(line)
|
|
return summary, "\n".join(kept)
|
|
|
|
|
|
def rooted_in(error, exception_type):
|
|
seen = set()
|
|
while error is not None and id(error) not in seen:
|
|
seen.add(id(error))
|
|
if isinstance(error, exception_type):
|
|
return True
|
|
error = error.__cause__ or error.__context__
|
|
return False
|
|
|
|
|
|
def rooted_in_named(error, class_name):
|
|
# Some apworlds define their own exception class inside their own
|
|
# zipimported package (e.g. CTJoT's InvalidYamlException) rather
|
|
# than a stable top-level module - there's no fixed path to import
|
|
# and isinstance-check it the way Fill.FillError works, so match by
|
|
# class name instead.
|
|
seen = set()
|
|
while error is not None and id(error) not in seen:
|
|
seen.add(id(error))
|
|
if type(error).__name__ == class_name:
|
|
return True
|
|
error = error.__cause__ or error.__context__
|
|
return False
|
|
|
|
|
|
def run_with_timeout(games, dest, timeout, options_per_game=None, seed=None, spoiler=3):
|
|
# fork, not spawn. Every mode runs in its own process so a generation
|
|
# can be killed on timeout and cannot leak state into the next one -
|
|
# but spawn re-imports the whole of Archipelago in each child, five
|
|
# times per apworld, which measured as the single largest cost in a
|
|
# run. Forking inherits the import the parent already did.
|
|
#
|
|
# The isolation that matters is unchanged: the child still gets its
|
|
# own copy-on-write memory, so whatever generation mutates dies with
|
|
# it and the parent's registry stays pristine for the next mode.
|
|
#
|
|
# Safe here specifically because the forking process is
|
|
# single-threaded: main() starts no threads of its own, and the
|
|
# Manager below is a separate process rather than a thread in this
|
|
# one. Forking a multi-threaded parent is the case to avoid, and this
|
|
# is not one.
|
|
try:
|
|
set_start_method("fork")
|
|
except RuntimeError:
|
|
pass
|
|
manager = Manager()
|
|
results = manager.list()
|
|
process = Process(target=generate_one,
|
|
args=(games, dest, results, options_per_game, seed, spoiler))
|
|
started = time.monotonic()
|
|
process.start()
|
|
process.join(timeout)
|
|
timed_out = process.is_alive()
|
|
if timed_out:
|
|
process.terminate()
|
|
process.join(5)
|
|
if process.is_alive():
|
|
process.kill()
|
|
process.join()
|
|
elapsed = time.monotonic() - started
|
|
return list(results), timed_out, elapsed
|
|
|
|
|
|
def classify(results, timed_out, elapsed, single_player=True, randomized=False):
|
|
if timed_out:
|
|
return "failed", f"timed out after {elapsed:.0f}s"
|
|
if not results:
|
|
return "failed", "generation process exited without a result"
|
|
result = results[0]
|
|
if isinstance(result, BaseException):
|
|
if rooted_in(result, FileNotFoundError):
|
|
return "needs_input", str(result)
|
|
if randomized and rooted_in_named(result, "OptionError"):
|
|
# The world rejected the option combination we rolled - not a
|
|
# crash and not a fill failure, but the world validating its
|
|
# own input. Archipelago rolls every option independently, so
|
|
# a world with interdependent options (Blender's min/max
|
|
# similarity percent, for example) can always be handed a
|
|
# combination it considers invalid. A player writing
|
|
# "random" for both hits exactly this, so it is worth
|
|
# reporting - but as its own thing, not as a defect.
|
|
#
|
|
# Only when the options were randomized: a world that rejects
|
|
# its OWN DEFAULTS is genuinely broken and stays "failed".
|
|
return "invalid_options", f"rejected the rolled options: {result}"
|
|
from Fill import FillError
|
|
if rooted_in(result, FillError):
|
|
# A fill failure is a failure in both modes. This used to be
|
|
# excused in single-player on the theory that some games'
|
|
# item/location balance "only works out in a real multiworld"
|
|
# - the multi-game test exists precisely to check that theory,
|
|
# and it does not hold (Dead Cells fails both ways), so the
|
|
# excuse is gone: a seed that cannot be filled is a seed that
|
|
# cannot be generated, whoever else is in it.
|
|
where = "a multi-game seed" if not single_player else "a single-player seed"
|
|
return "failed", f"fill failed in {where}: {result}"
|
|
if rooted_in_named(result, "InvalidYamlException"):
|
|
# Some worlds (e.g. CTJoT) require a yaml pre-generated by an
|
|
# external tool and reject our generic single-player yaml
|
|
# outright - not a defect, just incompatible with this
|
|
# testing methodology.
|
|
return "passed", f"requires an externally-generated yaml (not a defect): {result}"
|
|
return "failed", f"{type(result).__name__}: {result}"
|
|
return "passed", str(result)
|
|
|
|
|
|
def resolve_core_game(name, world_types):
|
|
"""Match a sheet name to a world Archipelago actually registers.
|
|
|
|
The sheet and the worlds disagree about wording often enough that an
|
|
exact comparison finds only 72 of the 81 core rows. The rest differ
|
|
by an accent ("Pokemon Emerald"), a hyphen ("Choo-Choo Charles"),
|
|
capitalisation ("EarthBound"), or a series prefix the sheet adds and
|
|
the world does not ("The Legend of Zelda: Ocarina of Time").
|
|
|
|
Tried in order: the name as written, then folded to letters and
|
|
digits, then either side of a colon folded the same way. Either side,
|
|
because the sheet puts the distinguishing part before the colon as
|
|
often as after it - "Super Mario Land 2: The Golden Coins" is
|
|
registered as "Super Mario Land 2", while "The Legend of Zelda:
|
|
Ocarina of Time" is registered as "Ocarina of Time".
|
|
"""
|
|
if name in world_types:
|
|
return name
|
|
by_slug = {}
|
|
for registered in world_types:
|
|
by_slug.setdefault(slug(registered), registered)
|
|
candidates = [name]
|
|
if ":" in name:
|
|
head, tail = name.split(":", 1)
|
|
candidates += [tail, head]
|
|
for candidate in candidates:
|
|
registered = by_slug.get(slug(candidate))
|
|
if registered:
|
|
return registered
|
|
return None
|
|
|
|
|
|
# Deliberate copy of Name.slug in core/model/name.py. This file is mounted
|
|
# alone into the container, with no access to this checkout's
|
|
# packages, so it cannot import it - but the two must stay identical
|
|
# or a world folds to one key here and a different one on the host.
|
|
def slug(text):
|
|
return re.sub(r"[^a-z0-9]", "", unicodedata.normalize("NFKD", (text or "").lower()))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
# Exactly one of these. --apworld is the normal case: a downloaded
|
|
# file, identified by which world claims it as its zip_path. --game
|
|
# names a world that is already part of the Archipelago build, which
|
|
# has no file to point at and so cannot be identified that way.
|
|
parser.add_argument("--apworld")
|
|
parser.add_argument("--game", help="Test a world that ships with Archipelago, by name, "
|
|
"instead of an apworld file. Mutually exclusive with --apworld.")
|
|
parser.add_argument("--output-dir", required=True)
|
|
parser.add_argument("--timeout", type=int, default=300)
|
|
parser.add_argument("--companion-pool", default="",
|
|
help="Comma-separated core games to draw companions from. Empty "
|
|
"means every core world, which risks drawing one that cannot "
|
|
"generate here (see container/verify_companions.py).")
|
|
parser.add_argument("--companion-min", type=int, default=2,
|
|
help="Fewest core games to draw as companions for a multi-game seed.")
|
|
parser.add_argument("--companion-max", type=int, default=5,
|
|
help="Most core games to draw as companions for a multi-game seed.")
|
|
parser.add_argument("--random-repeats", type=int, default=3,
|
|
help="How many times the randomized-option modes run. Each attempt "
|
|
"rolls a different option combination.")
|
|
parser.add_argument("--repeats", type=int, default=10,
|
|
help="How many times to run the single-game and multi-game tests. "
|
|
"Stability is a pass rate, not a single verdict.")
|
|
parser.add_argument("--spoiler", type=int, default=3,
|
|
help="Spoiler level passed to Generate. 3 computes the full "
|
|
"playthrough, exercising accessibility logic.")
|
|
args = parser.parse_args()
|
|
if bool(args.apworld) == bool(args.game):
|
|
parser.error("pass exactly one of --apworld or --game")
|
|
|
|
# Determine the game name from the loaded World class itself rather
|
|
# than the apworld's archipelago.json manifest: older apworlds don't
|
|
# ship a manifest at all, and the core (<0.7.0 here) tolerates that
|
|
# and loads them anyway - a manifest-only lookup would misreport a
|
|
# perfectly loadable world as failed. World.zip_path is set only for
|
|
# worlds loaded from a .apworld zip (worlds/AutoWorld.py), and equals
|
|
# the apworld's own path, so it identifies our one mounted world
|
|
# exactly regardless of manifest presence.
|
|
# worlds/__init__.py logs each failed world's traceback and carries
|
|
# on; without a handler here that detail is lost and all the pipeline
|
|
# can say is "found 0 worlds".
|
|
load_log = io.StringIO()
|
|
log_handler = logging.StreamHandler(load_log)
|
|
log_handler.setLevel(logging.ERROR)
|
|
logging.getLogger().addHandler(log_handler)
|
|
try:
|
|
import worlds
|
|
from worlds.AutoWorld import AutoWorldRegister
|
|
finally:
|
|
logging.getLogger().removeHandler(log_handler)
|
|
|
|
ensure_tutorials_default()
|
|
|
|
if args.game:
|
|
# A core world is already loaded - there is no file to identify
|
|
# it by, and none of the load-failure reporting below applies. It
|
|
# either exists in this build or it does not, and "does not" is a
|
|
# naming problem on the sheet rather than a defect in a world.
|
|
resolved = resolve_core_game(args.game, AutoWorldRegister.world_types)
|
|
if resolved:
|
|
args.game = resolved
|
|
else:
|
|
# "unknown", not "unverified". Nothing about this world is
|
|
# broken - this Archipelago build simply has no world under
|
|
# that name, which is a naming mismatch on the sheet (or a
|
|
# world that has since been removed from core), and calling
|
|
# it Broken would blame a world for the sheet's wording.
|
|
print(json.dumps({
|
|
"game": None,
|
|
"outcome": "unknown",
|
|
"detail": (f"this Archipelago build registers no world named {args.game!r}, "
|
|
f"so there was nothing to test. The sheet lists this row as Core, "
|
|
f"so either its name differs from the world's own or the world is "
|
|
f"no longer part of Archipelago."),
|
|
"elapsed_seconds": 0,
|
|
}))
|
|
return 1
|
|
custom_games = [args.game]
|
|
else:
|
|
custom_games = [
|
|
name for name, world_type in AutoWorldRegister.world_types.items()
|
|
if str(getattr(world_type, "zip_path", "") or "") == args.apworld
|
|
]
|
|
if len(custom_games) != 1:
|
|
# Its own outcome, NOT "passed". This covers several distinct
|
|
# causes (a duplicate game name against a now-built-in world, an
|
|
# import-time crash, a file that isn't a valid apworld at all,
|
|
# a genuine collision between two unrelated apworlds) - none of
|
|
# them necessarily an apworld defect, but none of them a world
|
|
# this pipeline has verified either. Calling it "passed" promoted
|
|
# files that never loaded and crash-looped real WebHost startup
|
|
# Factorio Platformer did exactly that: not a valid zip, zero
|
|
# worlds loaded, recorded as a pass, promoted, and crash-looping
|
|
# real WebHost startup - it needed a hand-written blacklist entry
|
|
# to undo. promote.py takes "passed" alone, so this verdict now
|
|
# keeps such a file out of worlds_live/ on its own.
|
|
summary, traceback_text = describe_load_failure(load_log.getvalue(), args.apworld)
|
|
if len(custom_games) > 1:
|
|
detail = (f"this file registers {len(custom_games)} worlds, not one: "
|
|
f"{', '.join(custom_games)}. Only a single-world apworld can be "
|
|
f"promoted, since the pipeline cannot tell which one this row means.")
|
|
elif summary:
|
|
detail = f"the world failed to import: {summary}"
|
|
if traceback_text:
|
|
detail += f"\n\n{traceback_text.strip()}"
|
|
else:
|
|
# No captured traceback: the file is not a loadable apworld at
|
|
# all (not a zip, wrong layout), so nothing was even attempted.
|
|
detail = (f"no world loaded from this file and no import was attempted - it is "
|
|
f"probably not a valid .apworld (failed_world_loads="
|
|
f"{worlds.failed_world_loads})")
|
|
print(json.dumps({
|
|
"game": None,
|
|
"outcome": "unverified",
|
|
"detail": detail,
|
|
"elapsed_seconds": 0,
|
|
}))
|
|
return 1
|
|
|
|
game = custom_games[0]
|
|
|
|
Path(args.output_dir).mkdir(parents=True, exist_ok=True)
|
|
|
|
def run_mode(games, options_per_game=None, seed=None, single_player=None, randomized=False):
|
|
if single_player is None:
|
|
single_player = len(games) == 1
|
|
mode_results, timed_out, elapsed = run_with_timeout(
|
|
games, args.output_dir, args.timeout,
|
|
options_per_game=options_per_game, seed=seed, spoiler=args.spoiler)
|
|
outcome, detail = classify(mode_results, timed_out, elapsed,
|
|
single_player=single_player, randomized=randomized)
|
|
return {"outcome": outcome, "detail": detail, "elapsed_seconds": round(elapsed, 1),
|
|
"timed_out": timed_out}
|
|
|
|
# A timeout anywhere abandons the rest of this apworld. One world that
|
|
# hangs would otherwise cost timeout x repeats x modes - by far the
|
|
# largest thing in a run - and a world that cannot finish one seed has
|
|
# nothing useful to say about its stability across ten.
|
|
state = {"timed_out": False}
|
|
|
|
def skipped(reason):
|
|
return {"outcome": "unknown", "detail": reason, "elapsed_seconds": None}
|
|
|
|
def run_repeated(players, repeats, single_player=None, options=None, randomized=False):
|
|
"""The same test several times, reported as a pass rate.
|
|
|
|
Stability is not something a single generation can tell you: S, D
|
|
and M each get a different Archipelago seed, so the fill differs
|
|
every time, and a world that places items successfully four times
|
|
in five is genuinely unstable. Running once records whichever
|
|
result we happened to get.
|
|
|
|
Each repeat uses an explicit seed so a failure among ten passes is
|
|
reproducible - without it the interesting case is the one that
|
|
cannot be re-examined.
|
|
"""
|
|
attempts = []
|
|
for _ in range(max(repeats, 1)):
|
|
seed = random.randint(0, 2 ** 31 - 1)
|
|
# players is a callable so the multi-game test can draw a
|
|
# different set of companions every attempt - ten runs against
|
|
# one fixed pair only ever tests that pair.
|
|
games = players() if callable(players) else players
|
|
# A callable, so the randomized modes can roll a different
|
|
# point in the option space for every attempt - repeating one
|
|
# roll would only re-measure the same combination.
|
|
per_game = options() if callable(options) else options
|
|
result = run_mode(games, options_per_game=per_game, seed=seed,
|
|
single_player=single_player, randomized=randomized)
|
|
result["seed"] = seed
|
|
if len(games) > 1:
|
|
result["companions"] = games[1:]
|
|
attempts.append(result)
|
|
if result["timed_out"]:
|
|
state["timed_out"] = True
|
|
break
|
|
|
|
passed = [a for a in attempts if a["outcome"] == "passed"]
|
|
failed = [a for a in attempts if a["outcome"] != "passed"]
|
|
if not failed:
|
|
outcome, detail = "passed", f"generated {len(passed)}/{len(attempts)} times"
|
|
elif not passed:
|
|
# Every attempt agreeing on a non-failure reason keeps that
|
|
# reason. A world that needs a base ROM reports needs_input
|
|
# once and "failed all 10 attempts" ten times over, and
|
|
# flattening those to "failed" blames the world for a file
|
|
# this pipeline does not have - which matters most for core
|
|
# worlds, where roughly seven of them need a ROM and would
|
|
# otherwise all read as Broken.
|
|
outcomes = {attempt["outcome"] for attempt in attempts}
|
|
outcome = outcomes.pop() if len(outcomes) == 1 else "failed"
|
|
detail = f"failed all {len(attempts)} attempts. {failed[0]['detail']}"
|
|
if outcome != "failed":
|
|
detail = (f"{outcome} on all {len(attempts)} attempts. "
|
|
f"{failed[0]['detail']}")
|
|
else:
|
|
# The whole point of repeating: a world that works sometimes
|
|
# is worse for a player than one that never does, because
|
|
# nobody knows to avoid it.
|
|
outcome = "flaky"
|
|
detail = (f"generated {len(passed)}/{len(attempts)} times. "
|
|
f"First failure (seed {failed[0]['seed']}): {failed[0]['detail']}")
|
|
return {
|
|
"outcome": outcome,
|
|
"detail": detail,
|
|
"elapsed_seconds": round(sum(a["elapsed_seconds"] or 0 for a in attempts), 1),
|
|
"attempts": len(attempts),
|
|
# What was asked for, as opposed to what ran - a timeout cuts
|
|
# a mode short, and the retest cache compares against this so
|
|
# such a record is not re-run forever.
|
|
"requested_attempts": max(repeats, 1),
|
|
"passed_attempts": len(passed),
|
|
"failed_seeds": [a["seed"] for a in failed],
|
|
# Which companions each failing attempt drew, so a failure
|
|
# caused by one particular combination can be reproduced.
|
|
"failed_companions": [a.get("companions") for a in failed if a.get("companions")],
|
|
}
|
|
|
|
tests = {}
|
|
|
|
# 1. the world on its own, every option at its default.
|
|
tests["single"] = run_repeated([game], args.repeats)
|
|
tests["single"]["label"] = "Single game"
|
|
|
|
# 2. the same world twice. A world that keeps state on the class
|
|
# rather than the instance works alone and corrupts itself here -
|
|
# a bug class no other mode can see.
|
|
tests["duplicate"] = (skipped("skipped - an earlier test timed out") if state["timed_out"]
|
|
else run_mode([game, game], single_player=False))
|
|
tests["duplicate"]["label"] = "Two players, same game"
|
|
|
|
# 3. the world alongside other games. Each attempt draws a fresh
|
|
# random selection of core worlds rather than reusing one fixed
|
|
# pair: ten runs against the same two companions only ever tell you
|
|
# about those two, while a different combination each time samples
|
|
# what a real multiworld looks like.
|
|
configured_pool = [name.strip() for name in args.companion_pool.split(",") if name.strip()]
|
|
if configured_pool:
|
|
# Verified by the host once per run: every one of these generated
|
|
# a seed on its own in this exact environment, so a multi-game
|
|
# failure is attributable to the world under test rather than to
|
|
# a companion that was never going to work.
|
|
pool = [name for name in configured_pool if name != game]
|
|
else:
|
|
pool = core_worlds(exclude=game)
|
|
low = max(args.companion_min, 1)
|
|
high = max(args.companion_max, low)
|
|
|
|
def draw_companions():
|
|
count = min(random.randint(low, high), len(pool))
|
|
return random.sample(pool, count)
|
|
|
|
if state["timed_out"]:
|
|
tests["multi"] = skipped("skipped - an earlier test timed out")
|
|
elif pool:
|
|
tests["multi"] = run_repeated(
|
|
lambda: [game] + draw_companions(), args.repeats, single_player=False)
|
|
tests["multi"]["companion_pool"] = len(pool)
|
|
tests["multi"]["companion_range"] = [low, high]
|
|
tests["multi"]["detail"] += f" (against {low}-{high} core games drawn from {len(pool)})"
|
|
else:
|
|
tests["multi"] = skipped("no core worlds available to pair with")
|
|
tests["multi"]["label"] = "Multi game"
|
|
|
|
# 4 and 5. The same two shapes as above - the world alone, and the
|
|
# world alongside the companions - but with its options randomized
|
|
# instead of left at their defaults. The default option set is a
|
|
# single point in a world's option space and most option-dependent
|
|
# crashes are nowhere near it.
|
|
#
|
|
# One roll each, and every roll that does not produce a seed is a
|
|
# failure, including one the world rejects with OptionError. That
|
|
# last point is the whole reason these modes exist: the yaml a
|
|
# player writes is accepted as valid - nothing in the template or
|
|
# the upload path objects to "random" on two interdependent
|
|
# options - so the conflict only surfaces when generation is
|
|
# actually attempted, possibly after everyone has submitted, and it
|
|
# is intermittent. Re-rolling until the world accepts something
|
|
# would hide exactly the failure a real player hits.
|
|
#
|
|
# Each randomized run is gated on its own default-options
|
|
# counterpart: SR only when the world generates alone, MR only when
|
|
# it generates alongside the companions. A randomized run whose
|
|
# plain equivalent already fails would fail for the same underlying
|
|
# reason and cost a generation to learn nothing.
|
|
rolls = None
|
|
if state["timed_out"]:
|
|
aborted = "skipped - an earlier test timed out"
|
|
tests["single_random"] = skipped(aborted)
|
|
tests["multi_random"] = skipped(aborted)
|
|
elif tests["single"]["outcome"] == "passed" or tests["multi"]["outcome"] == "passed":
|
|
try:
|
|
# One roll per attempt, drawn up front so SR and MR run the
|
|
# SAME combinations as each other. Pairing them is what makes
|
|
# "SR passed but MR failed" attributable to the extra players
|
|
# rather than to two different option sets.
|
|
rolls = [random_options(game) for _ in range(max(args.random_repeats, 1))]
|
|
except BaseException as error:
|
|
rolls = None
|
|
unreadable = f"could not read this world's options: {type(error).__name__}: {error}"
|
|
tests["single_random"] = skipped(unreadable)
|
|
tests["multi_random"] = skipped(unreadable)
|
|
|
|
if rolls:
|
|
def roller():
|
|
# Walks the pre-drawn rolls in order, so attempt N of SR and
|
|
# attempt N of MR use the same option values.
|
|
supply = iter(rolls)
|
|
def next_roll():
|
|
return {game: next(supply)}
|
|
return next_roll
|
|
|
|
# "flaky" counts as generating: the world produced a seed at
|
|
# least once, so a randomized run can still tell us something.
|
|
if tests["single"]["outcome"] in ("passed", "flaky"):
|
|
result = run_repeated([game], len(rolls), single_player=True,
|
|
options=roller(), randomized=True)
|
|
result["randomized_options"] = len(rolls[0])
|
|
tests["single_random"] = result
|
|
else:
|
|
tests["single_random"] = skipped(
|
|
"skipped - the single-game test did not pass")
|
|
|
|
if tests["multi"]["outcome"] in ("passed", "flaky"):
|
|
result = run_repeated(lambda: [game] + draw_companions(), len(rolls),
|
|
single_player=False, options=roller(), randomized=True)
|
|
result["randomized_options"] = len(rolls[0])
|
|
tests["multi_random"] = result
|
|
else:
|
|
tests["multi_random"] = skipped(
|
|
"skipped - the multi-game test did not pass")
|
|
elif "single_random" not in tests:
|
|
neither = "skipped - neither the single-game nor the multi-game test passed"
|
|
tests["single_random"] = skipped(neither)
|
|
tests["multi_random"] = skipped(neither)
|
|
|
|
tests["single_random"]["label"] = "Random options, alone"
|
|
tests["multi_random"]["label"] = "Random options, multi game"
|
|
|
|
# The top-level fields stay exactly what they were - the single-player
|
|
# verdict - because that is what promotion and every existing consumer
|
|
# reads. The per-mode results live alongside it.
|
|
report = {
|
|
"game": game,
|
|
"outcome": tests["single"]["outcome"],
|
|
"detail": tests["single"]["detail"],
|
|
"elapsed_seconds": tests["single"]["elapsed_seconds"],
|
|
"multi_outcome": tests["multi"]["outcome"],
|
|
"multi_detail": tests["multi"]["detail"],
|
|
"multi_elapsed_seconds": tests["multi"]["elapsed_seconds"],
|
|
"multi_companions": tests["multi"].get("companions", []),
|
|
"tests": tests,
|
|
}
|
|
|
|
print(json.dumps(report))
|
|
# Exit code still reflects the single-player result alone: it is what
|
|
# gates promotion, and the other modes are reported rather than
|
|
# blocking.
|
|
return {"passed": 0, "needs_input": 2}.get(tests["single"]["outcome"], 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|