Skilly/APWorldTester
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

initial commit

Browse Source
This commit is contained in:
Skilly
2026-09-06 15:12:38 +02:00
parent 4caf096780
commit 0d77a07291
185 changed files with 15465 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
src/archipelago_tester/pipeline/download/__init__.py Normal file
View File

75
src/archipelago_tester/pipeline/download/asset_matcher.py Normal file
View File

@@ -0,0 +1,75 @@
"""Choosing which asset in a release belongs to this game."""
import re
from archipelago_tester.core.model.name import Name
class AssetMatcher:
"""Matches release assets and release titles to one sheet row."""
def __init__(self, config, game_name):
self.config = config
self.game = Name(game_name)
def named(self, assets, exact):
"""The assets whose filename names this game."""
matched = []
for asset in assets:
name = Name(asset["name"].rsplit(".", 1)[0]).slug
if (name == self.game.slug) if exact else (name in self.game.slug):
matched.append(asset)
return matched
def select(self, assets):
"""The assets to download for this game."""
if len(assets) <= 1:
return assets
for matches in (self.named(assets, True), self.named(assets, False)):
if len(matches) == 1:
return matches
return assets
def title_slug(self, release):
"""A release's own title, reduced to the game it is for."""
title = release.get("name") or release.get("tag_name") or ""
title = re.sub(r"\bv?\d+(?:\.\d+)*\b", " ", title)
title = re.sub(r"\bapworld\b", " ", title, flags=re.IGNORECASE)
return Name(title).slug
def uninformative(self):
"""Titles that name no game at all."""
return set(self.config.value(
"downloads", "uninformative_release_slugs", ()))
def claimed_by(self, title, sheet_names):
"""The other sheet row this title names, or None.
Exact equality only. This is not "the title looks a bit off" -
it is positive evidence that a DIFFERENT row is named, which is
the only thing strong enough to skip a release the link points
at. "Mega Man X2" is exactly a row and is skipped when resolving
"Mega Man X"; "Mega Man X1" is nobody's row and is accepted.
"""
if not sheet_names or not self.game.text:
return None
if not title or title in self.uninformative():
return None
if title == self.game.slug:
return None
for other in sheet_names:
slug = Name(other).slug
if slug != self.game.slug and slug == title:
return other
return None
def release_claimed(self, release, sheet_names):
return self.claimed_by(self.title_slug(release), sheet_names)
def identity_claimed(self, found_games, sheet_names):
"""Whether a downloaded file identifies as another row's game."""
for found in found_games or []:
other = self.claimed_by(Name(found).slug, sheet_names)
if other:
return other
return None

23
src/archipelago_tester/pipeline/download/auth_error.py Normal file
View File

@@ -0,0 +1,23 @@
"""A token failure, which is the whole run's problem."""
import requests
class GitHubAuthError(RuntimeError):
"""A bad or expired token affects every remaining request."""
@classmethod
def matching(cls, game, error):
"""This error as an auth failure, or None if it is not one.
Raised loudly so the whole pool stops, rather than quietly
failing game after game against a token that will not work.
"""
if not isinstance(error, requests.HTTPError):
return None
response = error.response
status = response.status_code if response is not None else None
if status not in (401, 403):
return None
return cls(f"GitHub rejected the request for {game.name} with "
f"{status} - check .env / token expiry.")

61
src/archipelago_tester/pipeline/download/context.py Normal file
View File

@@ -0,0 +1,61 @@
"""What every download shares."""
import os
from archipelago_tester.core.model.name import Name
from archipelago_tester.pipeline.download.asset_matcher import AssetMatcher
class DownloadContext:
"""Who is downloading what, and where it goes.
One object rather than the same six arguments threaded through
every handler and helper.
"""
def __init__(self, config, paths, session, game, identifier=None,
sheet_names=None, root_directory=None):
self.config = config
self.paths = paths
self.session = session
self.game = game
self.identifier = identifier
self.sheet_names = sheet_names
self.root = root_directory or paths.downloads
self.matcher = AssetMatcher(config, game.name)
@property
def target_directory(self):
return os.path.join(self.root, Name(self.game.name).directory)
def target_path(self, filename):
return os.path.join(self.target_directory, filename)
def setting(self, key, default=None):
return self.config.value("downloads", key, default)
@property
def plain_headers(self):
"""Headers for a host that is not GitHub.
The session carries a GitHub token, which must never be sent to
an unrelated site.
"""
return {"Authorization": None, "Accept": "*/*"}
@property
def json_headers(self):
"""Headers for another forge's JSON API, without the token."""
return {"Authorization": None, "Accept": "application/json"}
def wrong_game(self, path):
"""Why this file is another game's, or None to keep it."""
if self.identifier is None:
return None
identity = self.identifier.identify(path)
if not identity.get("ran"):
return None
found = identity.get("games") or []
if any(Name(self.game.name).matches(name) for name in found):
return None
return found or identity.get("detail")

156
src/archipelago_tester/pipeline/download/download_run.py Normal file
View File

@@ -0,0 +1,156 @@
"""Fetching every game's apworld, concurrently."""
import os
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests
from archipelago_tester.core.state.keys import StateKeys
from archipelago_tester.core.state.store import StateStore
from archipelago_tester.pipeline.download.auth_error import GitHubAuthError
from archipelago_tester.pipeline.download.context import DownloadContext
from archipelago_tester.pipeline.download.downloader import Downloader
from archipelago_tester.pipeline.download.superseded_assets import (
SupersededAssets,
)
class DownloadRun:
"""One pass over the sheet's rows.
`sheet_names` should be the FULL sheet: a file can identify as a
core row, which never reaches here.
"""
def __init__(self, config, paths, session, root_directory=None,
identifier=None, jobs=None, sheet_names=None,
previous_releases=None):
self.config = config
self.paths = paths
self.session = session
self.root = root_directory or paths.downloads
self.identifier = identifier
self.jobs = jobs or int(config.value("downloads", "jobs", 8))
self.sheet_names = sheet_names
self.previous_releases = previous_releases or {}
@staticmethod
def split(games):
"""The games worth a network call, and those answered now.
A row with no link has nothing to look up, so it never takes a
slot in the thread pool.
"""
downloadable, skipped = [], []
for game in games:
if not game.release and not game.links:
skipped.append((game.name, "no release link in the sheet"))
else:
downloadable.append(game)
return downloadable, skipped
def context_for(self, game):
return DownloadContext(
config=self.config,
paths=self.paths,
session=self.session,
game=game,
identifier=self.identifier,
sheet_names=self.sheet_names,
root_directory=self.root,
)
def one(self, game):
"""Download one game, which never raises for that game alone.
A network failure is that game's own result, not the run's -
except an auth failure, which is everyone's.
"""
try:
downloaded, reason, info = Downloader(
context=self.context_for(game),
previous=self.previous_releases.get(game.name),
).run()
except requests.RequestException as error:
auth = GitHubAuthError.matching(game, error)
if auth is not None:
raise auth from error
return game, None, str(error), None
return game, downloaded, reason, info
@staticmethod
def source_url(game):
"""Where this apworld came from, for the status page.
game.release is only ever a real releases link, so for anything
resolved through the plain links the first one stands in.
"""
return game.release or (game.links[0] if game.links else None)
def record(self, finished, releases, skipped, completed, total,
on_asset, on_game):
"""Record one finished game and tell the caller about it.
The release is recorded even on failure - it was still found,
so its version is worth showing. on_game is called for failures
too, unlike on_asset: a progress bar has to count every game
that finished, not only the ones that produced a file.
"""
game, downloaded, reason, info = finished
releases[game.name] = {"url": self.source_url(game), **(info or {})}
if downloaded is None:
skipped.append((game.name, reason))
if on_game:
on_game(completed, total, game.name, False)
return 0
paths = [path for path, _ in downloaded]
for path in SupersededAssets(os.path.dirname(paths[0]),
paths).prune():
print(f"{game.name}: removed superseded {os.path.basename(path)}"
f" - the release no longer offers it")
for path, status in downloaded:
if on_asset:
on_asset(game.name, path, status)
if on_game:
on_game(completed, total, game.name, True)
return len(downloaded)
@staticmethod
def result_of(future, executor):
"""One finished download, stopping the pool on auth failure."""
try:
return future.result()
except GitHubAuthError:
executor.shutdown(cancel_futures=True)
raise
def all(self, games, on_asset=None, on_game=None):
"""Every game's apworld: how many files, what was skipped, and
which release each row resolved to.
Recording holds a lock, so on_asset always runs serialized and
a caller's own progress counter needs none of its own.
"""
self.sheet_names = self.sheet_names or [g.name for g in games]
downloadable, skipped = self.split(games)
releases, lock = {}, threading.Lock()
total = completed = 0
with ThreadPoolExecutor(max_workers=self.jobs) as executor:
futures = [executor.submit(self.one, g) for g in downloadable]
for future in as_completed(futures):
finished = self.result_of(future, executor)
with lock:
completed += 1
total += self.record(
finished, releases, skipped, completed,
len(downloadable), on_asset, on_game)
return total, skipped, releases
def record_releases(self, releases, state_path=None):
"""Write each game's resolved release into the state file."""
store = StateStore(state_path or self.paths.state)
state = store.load()
state.setdefault(StateKeys.GAME_RELEASES, {}).update(releases)
store.save(state)
return state

75
src/archipelago_tester/pipeline/download/downloader.py Normal file
View File

@@ -0,0 +1,75 @@
"""Fetching one sheet row's apworld."""
from archipelago_tester.pipeline.download.handlers.gitea_release import (
GiteaRelease,
)
from archipelago_tester.pipeline.download.handlers.github_file import (
GitHubFile,
)
from archipelago_tester.pipeline.download.handlers.github_release import (
GitHubRelease,
)
from archipelago_tester.pipeline.download.handlers.github_repo import (
GitHubRepo,
)
from archipelago_tester.pipeline.download.handlers.gitlab_file import (
GitLabFile,
)
from archipelago_tester.pipeline.download.handlers.gitlab_release import (
GitLabRelease,
)
from archipelago_tester.pipeline.download.handlers.plain_file import PlainFile
from archipelago_tester.pipeline.download.unusable_links import UnusableLinks
from archipelago_tester.pipeline.download.url_shapes import UrlShapes
class Downloader:
"""Whatever shape this game's link is, get the apworld.
A releases link is resolved directly; anything else is tried
against every handler in turn, most specific first, so an exact
file beats browsing a whole repo and the plain-URL handler can
never shadow a forge-specific one.
"""
def __init__(self, context, previous=None):
self.context = context
self.shapes = UrlShapes(context.config)
self.previous = previous
def handlers(self):
return (
GitHubFile(self.shapes),
GitHubRepo(self.shapes),
GitLabFile(self.shapes),
GitLabRelease(self.shapes),
GiteaRelease(self.shapes, previous=self.previous),
PlainFile(self.shapes),
)
def from_release_link(self):
handler = GitHubRelease(self.shapes, previous=self.previous)
parsed = handler.parse(self.context.game.release)
if parsed is None:
return None, "not a recognized releases URL", None
return handler.download(parsed, self.context)
def from_link(self, link):
"""This link's result, or None if no handler recognised it."""
for handler in self.handlers():
parsed = handler.parse(link)
if parsed is not None:
return handler.download(parsed, self.context)
return None
def run(self):
"""The files downloaded, why none were, and the release info."""
game = self.context.game
if game.release:
return self.from_release_link()
for link in game.links:
result = self.from_link(link)
if result is not None:
return result
return None, UnusableLinks(self.context.config, game.links).detail(), \
None

0
src/archipelago_tester/pipeline/download/handlers/__init__.py Normal file
View File

45
src/archipelago_tester/pipeline/download/handlers/base.py Normal file
View File

@@ -0,0 +1,45 @@
"""What every link handler shares."""
import os
from archipelago_tester.pipeline.download.local_file import LocalFile
class LinkHandler:
"""One recognised shape of sheet link, and how to fetch from it.
parse() returning None means "not my shape", which is different from
"mine, and it failed": the caller only moves on to the next link in
the first case.
"""
def __init__(self, shapes):
self.shapes = shapes
def parse(self, link):
raise NotImplementedError
def download(self, parsed, context):
raise NotImplementedError
def file(self, context, filename):
return LocalFile(context.session, context.target_path(filename))
def collect(self, candidates, context, fetch, verify):
"""Download each candidate, dropping any that is another game."""
downloaded, kept, rejected = [], [], []
for candidate in candidates:
path, status = fetch(candidate)
if status in LocalFile.NEEDS_DOWNLOAD and verify:
mismatch = context.wrong_game(path)
if mismatch is not None:
os.remove(path)
rejected.append((candidate["name"], mismatch))
continue
downloaded.append((path, status))
kept.append(candidate)
return downloaded, kept, rejected
def rejected_detail(self, rejected, subject):
details = "; ".join(f"{name} -> {found}" for name, found in rejected)
return f"downloaded {subject} didn't match this game: {details}"

44
src/archipelago_tester/pipeline/download/handlers/gitea_release.py Normal file
View File

@@ -0,0 +1,44 @@
"""A releases page on a self-hosted Gitea or Forgejo instance."""
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
from archipelago_tester.pipeline.download.release_downloader import (
ReleaseDownloader,
)
class GiteaRelease(LinkHandler):
"""Codeberg and friends.
The release JSON is close enough to GitHub's that ReleaseDownloader
handles it unchanged, given headers that omit the GitHub token.
"""
def __init__(self, shapes, previous=None):
super().__init__(shapes)
self.previous = previous
def parse(self, link):
return self.shapes.gitea_releases(link)
def latest(self, context, host, owner, repository):
response = context.session.get(
f"https://{host}/api/v1/repos/{owner}/{repository}"
"/releases/latest",
headers=context.json_headers,
)
if response.status_code == 404:
return None
response.raise_for_status()
return response.json()
def download(self, parsed, context):
host, owner, repository = parsed
release = self.latest(context, host, owner, repository)
if release is None:
return None, "no release found on this host", None
return ReleaseDownloader(
context=context,
release=release,
headers=context.plain_headers,
previous_published_at=(self.previous or {}).get("published_at"),
).run()

58
src/archipelago_tester/pipeline/download/handlers/github_file.py Normal file
View File

@@ -0,0 +1,58 @@
"""A link straight at an apworld committed in a GitHub repo."""
import os
import urllib.parse
import requests
from archipelago_tester.pipeline.build.archipelago import (
ArchipelagoBuild,
)
from archipelago_tester.pipeline.download.local_file import LocalFile
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
class GitHubFile(LinkHandler):
"""A committed file, named exactly by the sheet.
No identity check: the link names one file, so there is no second
candidate a name comparison could choose between.
"""
def parse(self, link):
return self.shapes.github_file(link)
def download(self, parsed, context):
"""Fetch the file as it currently stands on that branch."""
owner, repository, branch, path = parsed
filename = os.path.basename(urllib.parse.unquote(path))
local = self.file(context, filename)
url = ("https://raw.githubusercontent.com/"
f"{owner}/{repository}/{branch}/{path}")
status = local.status(local.remote_size(url))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url)
published = self.published_at(context, owner, repository, path, branch)
return [(local.path, status)], None, {"published_at": published}
def published_at(self, context, owner, repository, path, branch):
"""When this file was last committed, or None.
Best effort: a blob link has no release to read a date from, and
a download should not fail over display information.
"""
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
"/commits",
params={"path": path, "sha": branch, "per_page": 1},
)
response.raise_for_status()
commits = response.json()
except requests.RequestException:
return None
if not commits:
return None
commit = commits[0].get("commit", {})
return ((commit.get("committer") or {}).get("date")
or (commit.get("author") or {}).get("date"))

81
src/archipelago_tester/pipeline/download/handlers/github_release.py Normal file
View File

@@ -0,0 +1,81 @@
"""A GitHub releases page, the sheet's usual link."""
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
from archipelago_tester.pipeline.download.release_downloader import (
ReleaseDownloader,
)
from archipelago_tester.pipeline.download.release_identity import (
ReleaseIdentity,
)
from archipelago_tester.pipeline.download.release_picker import ReleasePicker
from archipelago_tester.pipeline.download.walk_back import WalkBack
class GitHubRelease(LinkHandler):
"""Resolve which release the link means, then fetch its apworld."""
def __init__(self, shapes, previous=None):
super().__init__(shapes)
self.previous = previous
def parse(self, link):
return self.shapes.releases(link)
def resolve(self, parsed, context):
"""The release this link resolves to, or None.
A hand-pinned tag prefix wins, then a link naming one exact
tag, then the search term. Runs every pass: it is the one call
that can report a new release, which is the only thing able to
change a previously failed answer.
"""
owner, repository, term, tag = parsed
picker = ReleasePicker(context, owner, repository, term)
overrides = context.config.get("release_overrides") or {}
prefix = overrides.get(context.game.name)
if prefix:
return picker.by_tag_prefix(prefix)
if tag:
return picker.by_tag(tag)
return picker.pick()
def fetch(self, parsed, context, release):
"""This release's apworld, walking back if it ships none.
Only that one cause is walked back from. "Ships another game's
apworld" must not be, since an older release of a multi-world
repo is precisely the wrong answer there.
"""
owner, repository = parsed[0], parsed[1]
downloader = ReleaseDownloader(
context=context,
release=release,
# What this row's date already says, so a re-upload of
# unchanged bytes keeps it instead of moving it forward.
previous_published_at=(self.previous or {}).get("published_at"),
)
downloaded, reason, info = downloader.run()
if (downloaded is not None or context.identifier is None
or downloader.cause != ReleaseDownloader.NO_APWORLD):
return downloaded, reason, info
fallback, reason = WalkBack(
context=context,
owner=owner,
repository=repository,
release=release,
).run(reason)
return fallback if fallback is not None else (None, reason, info)
def download(self, parsed, context):
release = self.resolve(parsed, context)
if release is None:
return None, "no release matched the search term in the link", None
identity = ReleaseIdentity(release)
repeat = identity.repeated(self.previous)
if repeat is not None:
return None, repeat, identity.failure(
{"published_at": None}, repeat)
downloaded, reason, info = self.fetch(parsed, context, release)
if downloaded is None:
return None, reason, identity.failure(info, reason)
return downloaded, reason, info

115
src/archipelago_tester/pipeline/download/handlers/github_repo.py Normal file
View File

@@ -0,0 +1,115 @@
"""A bare repository link, for a world never cut into a release."""
import os
import requests
from archipelago_tester.pipeline.build.archipelago import (
ArchipelagoBuild,
)
from archipelago_tester.pipeline.download.local_file import LocalFile
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
class GitHubRepo(LinkHandler):
"""Browses the repo's own file tree for an apworld."""
def parse(self, link):
return self.shapes.github_repo(link)
def resolve_branch(self, context, owner, repository, branch):
"""The real default branch, so a version reads as a name.
"HEAD" works as a ref, but resolving it means the status page
shows a branch rather than the literal string.
"""
if branch != "HEAD":
return branch
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}")
response.raise_for_status()
return response.json().get("default_branch") or "HEAD"
except requests.RequestException:
return "HEAD"
def tree_entries(self, context, owner, repository, branch):
"""Every apworld in the tree, with its size.
One call: the trees API takes recursive=1 itself, and returns
each blob's size for free.
"""
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
f"/git/trees/{branch}",
params={"recursive": "1"},
)
response.raise_for_status()
except requests.RequestException:
return []
return [
(entry["path"], entry.get("size"))
for entry in response.json().get("tree", [])
if entry.get("type") == "blob"
and entry["path"].lower().endswith(ApworldFile.SUFFIX)
]
def candidates(self, entries):
return [
{"name": os.path.basename(path), "path": path, "size": size}
for path, size in entries
]
def download(self, parsed, context):
owner, repository, branch = parsed
branch = self.resolve_branch(context, owner, repository, branch)
entries = self.tree_entries(context, owner, repository, branch)
if not entries:
return None, ("repo link has no release and no .apworld file "
"found in it"), None
chosen = context.matcher.select(self.candidates(entries))
downloaded, kept, rejected = self.collect(
candidates=chosen,
context=context,
fetch=lambda candidate: self.fetch(
candidate, context, owner, repository, branch),
verify=len(chosen) > 1,
)
if not downloaded:
detail = self.rejected_detail(rejected, "file(s) from repo")
return None, detail, None
published = self.published_at(
context, owner, repository, kept[0]["path"], branch)
return downloaded, None, {"published_at": published}
def fetch(self, candidate, context, owner, repository, branch):
local = self.file(context, candidate["name"])
status = local.status(candidate.get("size"))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(
f"https://raw.githubusercontent.com/{owner}/{repository}"
f"/{branch}/{candidate['path']}")
return local.path, status
def published_at(self, context, owner, repository, path, branch):
"""When the first kept file was last committed.
Only the first: one API call per candidate otherwise.
"""
try:
response = context.session.get(
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
"/commits",
params={"path": path, "sha": branch, "per_page": 1},
)
response.raise_for_status()
commits = response.json()
except requests.RequestException:
return None
if not commits:
return None
commit = commits[0].get("commit", {})
return ((commit.get("committer") or {}).get("date")
or (commit.get("author") or {}).get("date"))

44
src/archipelago_tester/pipeline/download/handlers/gitlab_file.py Normal file
View File

@@ -0,0 +1,44 @@
"""A link straight at an apworld committed in a GitLab repo."""
import os
import urllib.parse
import requests
from archipelago_tester.pipeline.download.local_file import LocalFile
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
class GitLabFile(LinkHandler):
"""A committed file on GitLab, named exactly by the sheet."""
def parse(self, link):
return self.shapes.gitlab_file(link)
def download(self, parsed, context):
project_path, branch, path = parsed
filename = os.path.basename(urllib.parse.unquote(path))
local = self.file(context, filename)
url = f"https://gitlab.com/{project_path}/-/raw/{branch}/{path}"
headers = context.json_headers
status = local.status(local.remote_size(url, headers=headers))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url, headers=headers)
published = self.published_at(context, project_path, path, branch)
return [(local.path, status)], None, {"published_at": published}
def published_at(self, context, project_path, path, branch):
"""When this file was last committed, or None."""
encoded = urllib.parse.quote(project_path, safe="")
try:
response = context.session.get(
f"https://gitlab.com/api/v4/projects/{encoded}"
"/repository/commits",
params={"path": path, "ref_name": branch, "per_page": 1},
headers=context.json_headers,
)
response.raise_for_status()
commits = response.json()
except requests.RequestException:
return None
return commits[0].get("committed_date") if commits else None

74
src/archipelago_tester/pipeline/download/handlers/gitlab_release.py Normal file
View File

@@ -0,0 +1,74 @@
"""A GitLab releases page."""
import urllib.parse
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
from archipelago_tester.pipeline.download.local_file import LocalFile
class GitLabRelease(LinkHandler):
"""The newest release of a GitLab project.
GitLab's assets are shaped nothing like GitHub's - a release-level
list of {name, url, direct_asset_url} with no per-asset timestamp -
so they are reshaped here rather than reusing the release path.
"""
def parse(self, link):
return self.shapes.gitlab_releases(link)
def latest(self, context, project_path):
"""GitLab's own newest-release shortcut, or None."""
encoded = urllib.parse.quote(project_path, safe="")
response = context.session.get(
f"https://gitlab.com/api/v4/projects/{encoded}"
"/releases/permalink/latest",
headers=context.json_headers,
)
if response.status_code == 404:
return None
response.raise_for_status()
return response.json()
def assets(self, release, context):
"""The release's apworld links, in GitHub's asset shape."""
links = (release.get("assets") or {}).get("links", [])
return [
{
"name": link["name"],
"browser_download_url": (link.get("direct_asset_url")
or link["url"]),
}
for link in links
if (link.get("name") or "").lower().endswith(ApworldFile.SUFFIX)
]
def fetch(self, asset, context):
local = self.file(context, asset["name"])
url = asset["browser_download_url"]
headers = context.plain_headers
status = local.status(local.remote_size(url, headers=headers))
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url, headers=headers)
return local.path, status
def download(self, parsed, context):
release = self.latest(context, parsed)
if release is None:
return None, "no GitLab release found for this project", None
info = {"published_at": (release.get("released_at")
or release.get("created_at"))}
assets = self.assets(release, context)
if not assets:
return None, "latest GitLab release has no .apworld asset", info
chosen = context.matcher.select(assets)
downloaded, _, rejected = self.collect(
candidates=chosen,
context=context,
fetch=lambda asset: self.fetch(asset, context),
verify=len(chosen) > 1,
)
if not downloaded:
return None, self.rejected_detail(rejected, "asset(s)"), info
return downloaded, None, info

48
src/archipelago_tester/pipeline/download/handlers/plain_file.py Normal file
View File

@@ -0,0 +1,48 @@
"""A plain URL ending in .apworld, on any host."""
import email.utils
import requests
from archipelago_tester.pipeline.download.local_file import LocalFile
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
class PlainFile(LinkHandler):
"""Any other host. Tried last, so a forge handler always wins."""
def parse(self, link):
return self.shapes.plain_file(link)
def download(self, parsed, context):
url, filename = parsed
local = self.file(context, filename)
size, published = self.remote_facts(context, url)
status = local.status(size)
if status in LocalFile.NEEDS_DOWNLOAD:
local.download(url, headers=context.plain_headers)
return [(local.path, status)], None, {"published_at": published}
def remote_facts(self, context, url):
"""The size and date one HEAD request can offer.
A plain URL has no version to compare, so size is the whole
staleness check. Neither is required.
"""
try:
response = context.session.head(
url, headers=context.plain_headers, allow_redirects=True)
response.raise_for_status()
except requests.RequestException:
return None, None
length = response.headers.get("Content-Length")
size = int(length) if length and length.isdigit() else None
return size, self.modified_at(response.headers.get("Last-Modified"))
def modified_at(self, modified):
if not modified:
return None
try:
return email.utils.parsedate_to_datetime(modified).isoformat()
except (TypeError, ValueError):
return None

83
src/archipelago_tester/pipeline/download/local_file.py Normal file
View File

@@ -0,0 +1,83 @@
"""Deciding whether a local copy is still the current one."""
import datetime
import os
import requests
from archipelago_tester.core.model.instant import Instant
class LocalFile:
"""One file on disk, against what the remote currently offers.
"It exists" is not "it is current": an author can replace an
asset's content in place, same filename, same version. Two signals
catch that - the asset's own timestamp, and its size - and both are
needed. A replacement that happens to be the same length is
invisible to size alone, and a remote that reports no length at all
makes size no signal whatsoever.
"""
#: The statuses that mean the file has to be fetched. The other
#: two - "verified" and "assumed" - mean the copy on disk stands.
NEEDS_DOWNLOAD = ("missing", "stale")
def __init__(self, session, path):
self.session = session
self.path = path
def outdated(self, remote_updated_at):
"""Whether the remote was touched after this copy arrived.
mtime is when download() wrote this file, so a remote timestamp
later than it means the copy on disk predates what is being
offered now. Without this, a release that replaced its asset
with one of the same length was never fetched, and the recorded
test result went on describing content that no longer exists -
while the page showed the new release's date beside it.
"""
stamp = Instant.of(remote_updated_at)
if stamp is None:
return False
arrived = datetime.datetime.fromtimestamp(
os.path.getmtime(self.path), datetime.timezone.utc)
return arrived < stamp
def status(self, remote_size, remote_updated_at=None):
"""Whether this file is missing, stale, verified or assumed.
The timestamp is checked before the size, and before a missing
size falls through to "assumed": it is the stronger signal, and
the two cases it catches are exactly the ones size cannot.
"""
if not os.path.exists(self.path):
return "missing"
if self.outdated(remote_updated_at):
return "stale"
if remote_size is None:
return "assumed"
if os.path.getsize(self.path) == remote_size:
return "verified"
return "stale"
def remote_size(self, url, headers=None):
"""The size a HEAD request reports, or None."""
try:
response = self.session.head(url, headers=headers,
allow_redirects=True)
response.raise_for_status()
except requests.RequestException:
return None
length = response.headers.get("Content-Length")
return int(length) if length and length.isdigit() else None
def download(self, url, headers=None):
"""Stream a URL into this path, creating its directory."""
response = self.session.get(url, stream=True, headers=headers)
response.raise_for_status()
os.makedirs(os.path.dirname(self.path), exist_ok=True)
with open(self.path, "wb") as handle:
for chunk in response.iter_content(chunk_size=65536):
handle.write(chunk)
return self.path

179
src/archipelago_tester/pipeline/download/release_downloader.py Normal file
View File

@@ -0,0 +1,179 @@
"""Fetching the apworld a release carries."""
import os
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.core.model.fingerprint import Fingerprint
from archipelago_tester.pipeline.download.local_file import LocalFile
from archipelago_tester.pipeline.download.zip_search import ZipSearch
class ReleaseDownloader:
"""A GitHub-shaped release, and the apworld it ships.
Shared with Gitea and Forgejo, whose release JSON is close enough
that the same logic applies unchanged.
"""
#: Why run() came back empty. The caller decides what to do about
#: it from this, never from the message beside it: a release that
#: ships no apworld can be walked back from, and one that ships
#: another game's must not be.
NO_APWORLD = "no_apworld"
WRONG_GAME = "wrong_game"
NO_APWORLD_DETAIL = "latest release has no .apworld asset"
def __init__(self, context, release, headers=None,
require_identity=False, previous_published_at=None):
self.context = context
self.release = release
self.headers = headers
self.require_identity = require_identity
self.previous_published_at = previous_published_at
self.cause = None
def named_assets(self):
return [
asset for asset in self.release.get("assets", [])
if asset["name"].lower().endswith(ApworldFile.SUFFIX)
]
def missing_detail(self):
detail = self.NO_APWORLD_DETAIL
if any(asset["name"].lower().endswith(ZipSearch.SUFFIX)
for asset in self.release.get("assets", [])):
detail += " (checked inside its .zip asset(s) too, none found)"
return detail
def assets(self):
"""Every apworld this release offers, zips included."""
found = self.named_assets()
if found:
return found
return ZipSearch(self.context, self.release, self.headers).assets()
def verifies(self, assets):
"""Whether a file has to prove which game it is.
With one candidate there is nothing to choose between, and the
sheet's link points at this release - so that file is the row's
apworld whatever its World class calls itself. A walk-back
release overrides that, since the link no longer points at it.
"""
return len(assets) > 1 or self.require_identity
@staticmethod
def touched_at(asset):
"""When this asset was last written, as the remote reports it.
The same value published_at() reports for the row, read here so
the two cannot disagree: a file whose recorded release date is
newer than the copy on disk is a file that was never fetched.
"""
return asset.get("updated_at") or asset.get("created_at")
def fetch(self, asset):
"""The asset's path, how it got there, and whether it changed.
An asset whose timestamp moved is fetched again - the copy on
disk can no longer be trusted to be current - and only then can
the two be compared. Identical bytes mean the author re-uploaded
the same file, so nothing about this row actually changed. A
file lifted out of a zip is taken as changed: ZipSearch does its
own fetching, and there is no earlier copy here to compare to.
"""
if "extracted_path" in asset:
return asset["extracted_path"], asset["status"], True
local = LocalFile(self.context.session,
self.context.target_path(asset["name"]))
url = asset["browser_download_url"]
status = local.status(
local.remote_size(url, headers=self.headers),
remote_updated_at=self.touched_at(asset),
)
if status not in LocalFile.NEEDS_DOWNLOAD:
return local.path, status, False
before = (Fingerprint.of_file(local.path)
if os.path.exists(local.path) else None)
local.download(url, headers=self.headers)
return local.path, status, Fingerprint.of_file(local.path) != before
def rejection(self, path, assets):
"""Why this file is not ours, or None to keep it.
A walk-back file that registered no world at all is kept:
nothing was proven either way, and generation testing reports
the import failure far more usefully than a silent drop.
"""
identity = self.context.identifier.identify(path)
if not identity.get("ran"):
return None
found = identity.get("games") or []
if self.require_identity and not found and len(assets) == 1:
return None
other = self.context.matcher.identity_claimed(
found, self.context.sheet_names)
matches = bool(found) and self.context.matcher.game.matches(
found[0])
if other:
return f'{found} - belongs to the "{other}" row'
if self.verifies(assets) and not matches:
return found or identity.get("detail")
return None
def checks(self, status, assets):
return (status in LocalFile.NEEDS_DOWNLOAD
and self.context.identifier is not None
and (self.verifies(assets) or self.context.sheet_names))
def published_at(self, kept):
"""When the content this row now holds was published.
Each asset carries its own timestamp; a release's own stays
fixed at creation even when its files are replaced later, so
the assets' is the one to read. But that timestamp moves for a
re-upload of identical bytes too, and then it dates the upload
rather than the world: the page would show a release newer than
the test beside it while the tested file never changed, and the
Compatibility column would call a world current on the strength
of an upload that added nothing. So only an asset whose content
actually changed sets this date, and otherwise the one already
recorded stands. With nothing recorded there is nothing to
hold, and the asset's own timestamp is the best available.
"""
stamps = [(self.touched_at(asset), changed)
for asset, changed in kept]
stamps = [pair for pair in stamps if pair[0]]
replaced = [stamp for stamp, changed in stamps if changed]
if replaced:
return max(replaced)
if self.previous_published_at:
return self.previous_published_at
return max((stamp for stamp, _ in stamps), default=None)
def run(self):
"""The files downloaded, why none were, and the release info."""
assets = self.assets()
if not assets:
self.cause = self.NO_APWORLD
return None, self.missing_detail(), {"published_at": None}
assets = self.context.matcher.select(assets)
downloaded, kept, rejected = [], [], []
for asset in assets:
path, status, changed = self.fetch(asset)
reason = (self.rejection(path, assets)
if self.checks(status, assets) else None)
if reason is not None:
os.remove(path)
rejected.append((asset["name"], reason))
continue
downloaded.append((path, status))
kept.append((asset, changed))
if not downloaded:
self.cause = self.WRONG_GAME
details = "; ".join(f"{name} -> {found}"
for name, found in rejected)
return None, (f"downloaded asset(s) didn't match this game: "
f"{details}"), {"published_at": None}
return downloaded, None, {"published_at": self.published_at(kept)}

55
src/archipelago_tester/pipeline/download/release_identity.py Normal file
View File

@@ -0,0 +1,55 @@
"""Remembering which release a failure was decided on."""
from archipelago_tester.core.state.keys import StateKeys
class ReleaseIdentity:
"""What makes a resolved release the same one as last run.
Its tag and publication date, plus every asset's name and size. The
sizes matter because an author can replace an asset in place
without cutting a new tag.
"""
def __init__(self, release):
self.release = release
@property
def value(self):
assets = sorted(
(asset.get("name"), asset.get("size"))
for asset in self.release.get("assets") or []
)
return {
"tag": self.release.get("tag_name"),
"published_at": self.release.get("published_at"),
"assets": [{"name": name, "size": size} for name, size in assets],
}
def repeated(self, previous):
"""The recorded reason, if this exact release failed before.
A release that ships no apworld, or only another row's, fails
the same way every run - and finding that out costs a download
of every zip asset plus a container run to identify what came
back. None of that can reach a different answer while the
release is unchanged. Keyed on the release rather than the
link, because the link is usually /releases/latest and a newly
published release is exactly what would fix a world in this
state. Only ever holds a verdict about content: a network error
escapes as an exception and a container that could not start
keeps the file, so neither is cached here.
"""
if not previous:
return None
if previous.get(StateKeys.FAILED_RELEASE) != self.value:
return None
return previous.get(StateKeys.FAILED_REASON) or None
def failure(self, info, reason):
"""A failure recorded against the release it was decided on."""
return {
**(info or {}),
StateKeys.FAILED_RELEASE: self.value,
StateKeys.FAILED_REASON: reason,
}

151
src/archipelago_tester/pipeline/download/release_picker.py Normal file
View File

@@ -0,0 +1,151 @@
"""Which release a sheet link means."""
import re
import urllib.parse
from archipelago_tester.pipeline.build.archipelago import (
ArchipelagoBuild,
)
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.core.model.name import Name
class ReleasePicker:
"""One repository's releases, narrowed to this game's.
A repo dedicated to one world can answer with its latest release. A
repo hosting several has to be matched on the link's search term and
the release's own title, and must never answer with a release that
another sheet row already claims.
"""
def __init__(self, context, owner, repository, term=None):
self.context = context
self.owner = owner
self.repository = repository
self.term = term
self.matcher = context.matcher
@property
def repo_url(self):
return (f"{ArchipelagoBuild.API_ROOT}/repos/"
f"{self.owner}/{self.repository}")
def all(self, max_pages=10):
"""Every release, newest first, across every page.
GitHub pages at 30 by default and several source repos hold
more than that, which would silently hide real releases from
every lookup below.
"""
releases = []
for page in range(1, max_pages + 1):
response = self.context.session.get(
f"{self.repo_url}/releases",
params={"per_page": 100, "page": page},
)
response.raise_for_status()
batch = response.json()
releases.extend(batch)
if len(batch) < 100:
break
return releases
def by_tag(self, tag):
"""The release for exactly this tag, or None."""
encoded = urllib.parse.quote(tag, safe="")
response = self.context.session.get(
f"{self.repo_url}/releases/tags/{encoded}")
if response.status_code == 404:
return None
response.raise_for_status()
return response.json()
def by_tag_prefix(self, prefix):
"""The newest release whose tag starts with this prefix."""
for release in self.all():
if (release.get("tag_name") or "").startswith(prefix):
return release
return None
def claimed(self, release):
return self.matcher.release_claimed(release, self.context.sheet_names)
def newest(self):
"""GitHub's own newest full release.
Right whenever it resolves, since it skips in-progress
prereleases. The list's first entry is the fallback for a repo
where every release is flagged prerelease, which /latest
excludes rather than relaxing.
"""
response = self.context.session.get(f"{self.repo_url}/releases/latest")
if response.status_code != 404:
response.raise_for_status()
return response.json()
releases = self.all()
return releases[0] if releases else None
def titled_for_this_row(self):
for release in self.all():
if self.matcher.title_slug(release) == self.matcher.game.slug:
return release
return None
def without_a_term(self):
"""The release to use when the link named none.
"Newest in the repo" is a fine answer for a repo holding one
world and a coin flip for one holding twelve, so a newest that
is titled for somebody else's row gives way to the one titled
for this row.
"""
latest = self.newest()
if latest is None or not self.claimed(latest):
return latest
return self.titled_for_this_row() or latest
def by_title(self, releases):
"""The first release whose title holds every word of the term.
The sheet's term is built for GitHub's own search box, which
matches per word: requiring the whole phrase rejects real terms
like "Jurassic Park Randomizer (SNES) Jurassic Park", where the
link generator repeats the game name. That looseness is what
lets a term land on a sibling world, so a release titled for
another row is skipped and the search continues.
"""
words = re.findall(r"[a-z0-9]+", self.term.lower())
if not words:
return None
for release in releases:
title = (f"{release.get('name') or ''} "
f"{release.get('tag_name') or ''}").lower()
if all(word in title for word in words) \
and not self.claimed(release):
return release
return None
def by_asset_name(self, releases):
"""The release carrying an asset named for this game.
A fork mirroring many worlds often tags every release by build
date alone, so the game's name appears only in which apworlds
the release bundles.
"""
term = Name(self.term).slug
for release in releases:
for asset in release.get("assets", []):
name = asset["name"]
if not name.lower().endswith(ApworldFile.SUFFIX):
continue
if Name(name.rsplit(".", 1)[0]).slug == term:
return release
return None
def pick(self):
"""The release this link means, or None."""
if self.term is None:
return self.without_a_term()
releases = self.all()
return self.by_title(releases) or self.by_asset_name(releases)

58
src/archipelago_tester/pipeline/download/superseded_assets.py Normal file
View File

@@ -0,0 +1,58 @@
"""Clearing apworlds a game's release no longer offers."""
import os
class SupersededAssets:
"""The files left behind when an author renames their asset.
A download writes each asset under its own filename, so a release
that renamed one leaves the previous name sitting beside it. Both
are then real files in the game's folder, and everything downstream
keys on the path rather than the game - two work items, two test
records, two rows on the status page, and a promotion that refuses
both because they claim one game name between them.
Only ever run for a game whose download succeeded: with nothing
fetched there is no current set to compare against, and pruning on
a failed network call would delete the copy that still works.
"""
#: What this class considers. A game's folder holds nothing else,
#: but naming it here keeps a stray file from being deleted on the
#: strength of living in the wrong directory.
SUFFIX = ".apworld"
def __init__(self, directory, kept):
self.directory = directory
self.kept = {os.path.abspath(path) for path in kept}
def stale(self):
"""Every apworld in the folder that this download did not write."""
if not self.kept or not os.path.isdir(self.directory):
return []
found = []
for entry in sorted(os.listdir(self.directory)):
if not entry.lower().endswith(self.SUFFIX):
continue
path = os.path.join(self.directory, entry)
if os.path.isfile(path) and os.path.abspath(path) not in self.kept:
found.append(path)
return found
def prune(self):
"""Delete them, returning what went.
A file that cannot be removed is not worth failing a download
over - it means the next run sees a duplicate, which is what
was already happening.
"""
removed = []
for path in self.stale():
try:
os.remove(path)
except OSError as error:
print(f"could not remove superseded {path}: {error}")
continue
removed.append(path)
return removed

54
src/archipelago_tester/pipeline/download/unusable_links.py Normal file
View File

@@ -0,0 +1,54 @@
"""Why none of a game's links could be downloaded from."""
import urllib.parse
class UnusableLinks:
"""Categorise links no handler recognised.
Reaching here means every recognised shape was tried against every
link, so the status page can say something concrete instead of a
generic "failed".
"""
def __init__(self, config, links):
self.config = config
self.links = links
@property
def unreachable(self):
"""Hosts known in advance never to be downloadable.
A Discord message link is not a public file however it is
fetched, unlike an unknown host that merely has no handler yet.
"""
return self.config.value("downloads", "unreachable_link_hosts") or {}
def domains(self):
"""Each link's host, or a reason it has none."""
found = []
for link in self.links:
parsed = urllib.parse.urlparse(link)
if parsed.scheme not in ("http", "https"):
protocol = parsed.scheme or "(none)"
return None, f"unsupported link protocol {protocol!r}: {link}"
if not parsed.netloc:
return None, f"not a valid URL: {link}"
netloc = parsed.netloc.lower()
found.append(netloc[4:] if netloc.startswith("www.") else netloc)
return found, None
def detail(self):
domains, problem = self.domains()
if problem is not None:
return problem
for domain in domains:
if domain in self.unreachable:
return self.unreachable[domain]
if "github.com" in domains:
return ("GitHub link found, but not a recognized "
"releases/repo/file URL")
if "gitlab.com" in domains:
return "GitLab link found, but not a recognized releases/blob URL"
hosts = ", ".join(sorted(set(domains)))
return f"no supported download source (custom host: {hosts})"

168
src/archipelago_tester/pipeline/download/update_history.py Normal file
View File

@@ -0,0 +1,168 @@
"""Every distinct apworld file this pipeline has seen, per game.
One file per game, written only when that game gains content it has not
seen before - so a run touches the handful of games that actually
changed rather than rewriting one shared blob. Deliberately NOT under
downloads/: that directory is a cache and gets cleared to force a clean
re-download, which is harmless for everything else in it. History is the
one thing that cannot be rebuilt afterwards - which version existed on
which date has no other source once it is gone.
"""
import os
from datetime import datetime, timezone
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.core.model.fingerprint import Fingerprint
from archipelago_tester.core.model.name import Name
from archipelago_tester.core.state.keys import StateKeys
from archipelago_tester.core.state.store import StateStore
class UpdateHistory:
"""What each game's apworld was, and when.
Identity is the file's sha256, so a re-download of unchanged content
adds nothing and recording is safe to repeat within a run. The
game's sheet name is stored inside the file as well as encoded in
its name, because the filename is a folded directory name and a
sheet rename would otherwise leave a file nothing could identify.
"""
def __init__(self, paths, directory=None):
self.directory = directory or paths.history
def path_for(self, game_name):
folded = Name(game_name).directory
return os.path.join(self.directory, f"{folded}.json")
def load(self, game_name):
return StateStore.read(
self.path_for(game_name),
{"game": game_name, "watching_since": None,
"baseline_at": None, "files": []})
def save(self, game_name, history):
os.makedirs(self.directory, exist_ok=True)
StateStore(self.path_for(game_name)).save(history)
#: Everything recorded before this field existed was gathered from
#: whatever the hosts happened to say, at whatever moment the file
#: was first downloaded, and none of it measures how often a world
#: changes under observation. The first pass to touch a game draws
#: a line: that instant is the game's one starting point, and only
#: changes seen after it count. See UpdateCadence, which reads it.
@staticmethod
def watching_since(history, stamp):
"""When this pipeline began watching this game.
The earliest content it has already recorded, so a file written
before this field existed still gets an honest answer rather
than a window that starts the day the field was added. Only a
game with no history at all starts watching now.
"""
seen = [item.get("first_seen_at") for item in history["files"]
if item.get("first_seen_at")]
return min(seen) if seen else stamp
def entry_for(self, apworld_path, sha256, released_at, source_url, now):
"""One file's record.
`released_at` is the upload date the download stage saw;
first_seen_at is when this pipeline observed it, which is the
only honest timestamp for a file whose host reports no date.
"""
manifest = ApworldFile(apworld_path).manifest or {}
return {
"version": manifest.get("world_version"),
"released_at": released_at,
"first_seen_at": (now or datetime.now(timezone.utc)).isoformat(),
"sha256": sha256,
"filename": os.path.basename(apworld_path),
"source_url": source_url,
}
def record_file(self, game_name, apworld_path, sha256, released_at,
source_url=None, now=None):
"""Add this file to the game's history if its content is new.
A game whose content has not changed still records that it was
looked at. How often a world updates is only meaningful against
how long it has been watched, and without that a game seen once
and a game seen all year are indistinguishable.
"""
history = self.load(game_name)
now = now or datetime.now(timezone.utc)
opened = not history.get("watching_since")
if opened:
history["watching_since"] = self.watching_since(
history, now.isoformat())
if not history.get("baseline_at"):
opened = True
history["baseline_at"] = now.isoformat()
if any(item.get("sha256") == sha256 for item in history["files"]):
if opened:
self.save(game_name, history)
return None
entry = self.entry_for(apworld_path, sha256, released_at,
source_url, now)
history["game"] = game_name
history["files"].append(entry)
# Oldest first. released_at is missing often enough that it
# cannot be the only key - first_seen_at always exists and
# breaks those ties in the order the pipeline saw them.
history["files"].sort(
key=lambda item: (item.get("released_at") or "",
item.get("first_seen_at") or ""))
self.save(game_name, history)
return entry
def files_in(self, game_directory):
return [
name for name in sorted(os.listdir(game_directory))
if name.lower().endswith(ApworldFile.SUFFIX)
and os.path.isfile(os.path.join(game_directory, name))
]
def record_all(self, state, root_directory, now=None):
"""Record every apworld on disk that is not yet in history.
Hashes the files directly rather than reading apworld_tests:
that dict only gains a hash once a world has been TESTED, so a
file downloaded this run would not appear until the next one -
and this runs right after downloading, which is the point at
which a new version is known.
"""
releases = state.get(StateKeys.GAME_RELEASES) or {}
# game_releases is keyed by the sheet name while the download
# folder is that name folded - map one to the other rather than
# assuming they match as written.
by_folder = {Name(name).directory: info
for name, info in releases.items()}
added = []
if not os.path.isdir(root_directory):
return added
for folder in sorted(os.listdir(root_directory)):
game_directory = os.path.join(root_directory, folder)
if not os.path.isdir(game_directory):
continue
added += self.record_folder(
folder, game_directory, by_folder.get(folder) or {}, now)
return added
def record_folder(self, folder, game_directory, release, now):
"""Every new apworld in one game's download folder."""
added = []
for filename in self.files_in(game_directory):
path = os.path.join(game_directory, filename)
entry = self.record_file(
game_name=folder,
apworld_path=path,
sha256=Fingerprint.of_file(path),
released_at=release.get("published_at"),
source_url=release.get("url"),
now=now,
)
if entry:
added.append((folder, entry))
return added

98
src/archipelago_tester/pipeline/download/url_shapes.py Normal file
View File

@@ -0,0 +1,98 @@
"""Recognising the shapes a sheet link can take."""
import os
import re
import urllib.parse
from archipelago_tester.core.model.apworld import ApworldFile
class UrlShapes:
"""The link shapes this pipeline knows how to fetch from."""
def __init__(self, config):
self.config = config
def releases(self, url):
"""Owner, repo, search term and tag from a releases link.
A link naming one exact tag is read as such: without it, a repo
hosting several worlds would fall back to "latest", which drifts
to whichever world was tagged most recently.
"""
match = re.match(
r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+)/releases",
url or "", re.IGNORECASE)
if match is None:
return None
parsed = urllib.parse.urlparse(url)
terms = urllib.parse.parse_qs(parsed.query).get("q", [])
tag_match = re.match(
r"^/[^/]+/[^/]+/releases/tag/([^/?#]+)", parsed.path,
re.IGNORECASE)
return (
match.group(1),
match.group(2),
terms[0].strip('"') if terms else None,
urllib.parse.unquote(tag_match.group(1)) if tag_match else None,
)
def github_file(self, url):
"""Owner, repo, branch and path from a committed-file link."""
match = re.match(
r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+)"
r"/(?:blob|raw)/([^/]+)/(.+\.apworld)$",
url or "", re.IGNORECASE)
return match.groups() if match else None
def github_repo(self, url):
"""Owner, repo and branch from a bare repository link.
Deliberately strict, so it does not swallow issues or wiki
links: those should keep falling through to the generic "not a
releases URL" message rather than triggering a tree search that
could never succeed.
"""
match = re.match(
r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+?)"
r"(?:/tree/([^/?#]+))?/?$",
url or "", re.IGNORECASE)
if match is None:
return None
owner, repository, branch = match.groups()
return owner, repository, branch or "HEAD"
def gitlab_file(self, url):
"""Project path, branch and file path from a GitLab blob link."""
match = re.match(
r"^https?://(?:www\.)?gitlab\.com/([^/]+(?:/[^/]+)*?)"
r"/-/blob/([^/]+)/(.+\.apworld)$",
url or "", re.IGNORECASE)
return match.groups() if match else None
def gitlab_releases(self, url):
"""The project path from a GitLab releases link."""
match = re.match(
r"^https?://(?:www\.)?gitlab\.com/([^/]+(?:/[^/]+)*?)"
r"/-/releases(?:[/?#]|$)",
url or "", re.IGNORECASE)
return match.group(1) if match else None
def gitea_releases(self, url):
"""Host, owner and repo from a Gitea-style releases link."""
match = re.match(
r"^https?://(?!(?:www\.)?(?:github|gitlab)\.com)"
r"([^/]+)/([^/]+)/([^/]+)/releases(?:[/?#]|$)",
url or "", re.IGNORECASE)
return match.groups() if match else None
def plain_file(self, url):
"""A URL ending in .apworld on any host, and its filename."""
parsed = urllib.parse.urlparse(url or "")
if parsed.scheme not in ("http", "https") or not parsed.netloc:
return None
path = urllib.parse.unquote(parsed.path)
if not path.lower().endswith(ApworldFile.SUFFIX):
return None
filename = os.path.basename(path)
return (url, filename) if filename != ApworldFile.SUFFIX else None

77
src/archipelago_tester/pipeline/download/walk_back.py Normal file
View File

@@ -0,0 +1,77 @@
"""Falling back to an older release that still ships an apworld."""
import requests
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.pipeline.download.release_downloader import (
ReleaseDownloader,
)
from archipelago_tester.pipeline.download.release_picker import ReleasePicker
class WalkBack:
"""One narrow rescue: the newest release forgot the apworld.
An author cuts a release and re-attaches only a mod zip, leaving
the world perfectly downloadable one release back. Every other
failure is left alone on purpose - in particular "did not match
this game", where an older release holding a different game's
apworld is precisely the wrong answer.
"""
def __init__(self, context, owner, repository, release):
self.context = context
self.owner = owner
self.repository = repository
self.release = release
def candidate(self, release):
"""Whether this is an older release that ships an apworld.
Matches on asset filename alone: this runs over a repo's whole
history, and opening every zip in it would cost far more than
the problem is worth.
"""
cutoff = self.release.get("published_at")
published_at = release.get("published_at")
return bool(
not release.get("draft")
and release.get("tag_name") != self.release.get("tag_name")
and published_at
and (not cutoff or published_at < cutoff)
and any(asset["name"].lower().endswith(ApworldFile.SUFFIX)
for asset in release.get("assets", []))
)
def older(self):
"""The newest earlier release with an apworld, by date.
Ordering comes from published_at rather than the order GitHub
returns, which follows the underlying tag and is not the same
thing.
"""
picker = ReleasePicker(self.context, self.owner, self.repository)
try:
releases = picker.all()
except requests.RequestException:
return None
candidates = [r for r in releases if self.candidate(r)]
if not candidates:
return None
return max(candidates, key=lambda release: release["published_at"])
def run(self, reason):
"""The fallback's result, or None and a combined reason."""
older = self.older()
if older is None:
return None, reason
fallback = ReleaseDownloader(
context=self.context,
release=older,
require_identity=True,
).run()
if fallback[0] is not None:
fallback[2]["fallback_release_tag"] = older.get("tag_name")
return fallback, None
return None, (f"{reason}; older release {older.get('tag_name')!r} "
f"was tried too: {fallback[1]}")

96
src/archipelago_tester/pipeline/download/zip_search.py Normal file
View File

@@ -0,0 +1,96 @@
"""Looking for an apworld inside a release's zip assets."""
import os
import tempfile
import zipfile
import requests
from archipelago_tester.core.model.apworld import ApworldFile
from archipelago_tester.pipeline.download.local_file import LocalFile
class ZipSearch:
"""For a release that publishes a zip instead of an apworld.
Some repos attach a source snapshot with the built apworld nested
inside. Every zip is re-downloaded to scratch space each time, since
the member's own filename - what a normal download caches on - is
not known until the zip is opened.
"""
#: The extension a release's archive asset is recognised by.
SUFFIX = ".zip"
def __init__(self, context, release, headers=None):
self.context = context
self.release = release
self.headers = headers
def zip_assets(self):
return [
asset for asset in self.release.get("assets", [])
if asset["name"].lower().endswith(ZipSearch.SUFFIX)
]
def members(self, zip_path):
"""Every apworld inside the zip, with its size."""
try:
with zipfile.ZipFile(zip_path) as archive:
return [
(info.filename, info.file_size)
for info in archive.infolist()
if not info.filename.endswith("/")
and info.filename.lower().endswith(ApworldFile.SUFFIX)
]
except zipfile.BadZipFile:
return []
def extract(self, zip_path, member):
"""Unpack one member into the game's folder."""
target = self.context.target_path(os.path.basename(member))
os.makedirs(os.path.dirname(target), exist_ok=True)
with zipfile.ZipFile(zip_path) as archive:
with archive.open(member) as source:
with open(target, "wb") as handle:
handle.write(source.read())
return target
def unpack(self, zip_path, members, zip_asset):
"""The members as assets, already on disk."""
results = []
for member, size in members:
target = self.context.target_path(os.path.basename(member))
status = LocalFile(self.context.session, target).status(size)
if status in LocalFile.NEEDS_DOWNLOAD:
self.extract(zip_path, member)
results.append({
"name": os.path.basename(member),
"updated_at": zip_asset.get("updated_at"),
"created_at": zip_asset.get("created_at"),
"extracted_path": target,
"status": status,
})
return results
def search_one(self, zip_asset):
with tempfile.TemporaryDirectory() as scratch:
zip_path = os.path.join(scratch, zip_asset["name"])
local = LocalFile(self.context.session, zip_path)
try:
local.download(zip_asset["browser_download_url"],
headers=self.headers)
except requests.RequestException:
return []
members = self.members(zip_path)
if not members:
return []
return self.unpack(zip_path, members, zip_asset)
def assets(self):
"""The first zip that holds an apworld wins."""
for zip_asset in self.zip_assets():
found = self.search_one(zip_asset)
if found:
return found
return []