initial commit
This commit is contained in:
Skilly
45
src/archipelago_tester/pipeline/download/handlers/base.py
Normal file
45
src/archipelago_tester/pipeline/download/handlers/base.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""What every link handler shares."""
|
||||
|
||||
import os
|
||||
|
||||
from archipelago_tester.pipeline.download.local_file import LocalFile
|
||||
|
||||
|
||||
class LinkHandler:
|
||||
"""One recognised shape of sheet link, and how to fetch from it.
|
||||
|
||||
parse() returning None means "not my shape", which is different from
|
||||
"mine, and it failed": the caller only moves on to the next link in
|
||||
the first case.
|
||||
"""
|
||||
|
||||
def __init__(self, shapes):
|
||||
self.shapes = shapes
|
||||
|
||||
def parse(self, link):
|
||||
raise NotImplementedError
|
||||
|
||||
def download(self, parsed, context):
|
||||
raise NotImplementedError
|
||||
|
||||
def file(self, context, filename):
|
||||
return LocalFile(context.session, context.target_path(filename))
|
||||
|
||||
def collect(self, candidates, context, fetch, verify):
|
||||
"""Download each candidate, dropping any that is another game."""
|
||||
downloaded, kept, rejected = [], [], []
|
||||
for candidate in candidates:
|
||||
path, status = fetch(candidate)
|
||||
if status in LocalFile.NEEDS_DOWNLOAD and verify:
|
||||
mismatch = context.wrong_game(path)
|
||||
if mismatch is not None:
|
||||
os.remove(path)
|
||||
rejected.append((candidate["name"], mismatch))
|
||||
continue
|
||||
downloaded.append((path, status))
|
||||
kept.append(candidate)
|
||||
return downloaded, kept, rejected
|
||||
|
||||
def rejected_detail(self, rejected, subject):
|
||||
details = "; ".join(f"{name} -> {found}" for name, found in rejected)
|
||||
return f"downloaded {subject} didn't match this game: {details}"
|
||||
@@ -0,0 +1,44 @@
|
||||
"""A releases page on a self-hosted Gitea or Forgejo instance."""
|
||||
|
||||
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
|
||||
from archipelago_tester.pipeline.download.release_downloader import (
|
||||
ReleaseDownloader,
|
||||
)
|
||||
|
||||
|
||||
class GiteaRelease(LinkHandler):
|
||||
"""Codeberg and friends.
|
||||
|
||||
The release JSON is close enough to GitHub's that ReleaseDownloader
|
||||
handles it unchanged, given headers that omit the GitHub token.
|
||||
"""
|
||||
|
||||
def __init__(self, shapes, previous=None):
|
||||
super().__init__(shapes)
|
||||
self.previous = previous
|
||||
|
||||
def parse(self, link):
|
||||
return self.shapes.gitea_releases(link)
|
||||
|
||||
def latest(self, context, host, owner, repository):
|
||||
response = context.session.get(
|
||||
f"https://{host}/api/v1/repos/{owner}/{repository}"
|
||||
"/releases/latest",
|
||||
headers=context.json_headers,
|
||||
)
|
||||
if response.status_code == 404:
|
||||
return None
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def download(self, parsed, context):
|
||||
host, owner, repository = parsed
|
||||
release = self.latest(context, host, owner, repository)
|
||||
if release is None:
|
||||
return None, "no release found on this host", None
|
||||
return ReleaseDownloader(
|
||||
context=context,
|
||||
release=release,
|
||||
headers=context.plain_headers,
|
||||
previous_published_at=(self.previous or {}).get("published_at"),
|
||||
).run()
|
||||
@@ -0,0 +1,58 @@
|
||||
"""A link straight at an apworld committed in a GitHub repo."""
|
||||
|
||||
import os
|
||||
import urllib.parse
|
||||
|
||||
import requests
|
||||
|
||||
from archipelago_tester.pipeline.build.archipelago import (
|
||||
ArchipelagoBuild,
|
||||
)
|
||||
from archipelago_tester.pipeline.download.local_file import LocalFile
|
||||
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
|
||||
|
||||
|
||||
class GitHubFile(LinkHandler):
|
||||
"""A committed file, named exactly by the sheet.
|
||||
|
||||
No identity check: the link names one file, so there is no second
|
||||
candidate a name comparison could choose between.
|
||||
"""
|
||||
|
||||
def parse(self, link):
|
||||
return self.shapes.github_file(link)
|
||||
|
||||
def download(self, parsed, context):
|
||||
"""Fetch the file as it currently stands on that branch."""
|
||||
owner, repository, branch, path = parsed
|
||||
filename = os.path.basename(urllib.parse.unquote(path))
|
||||
local = self.file(context, filename)
|
||||
url = ("https://raw.githubusercontent.com/"
|
||||
f"{owner}/{repository}/{branch}/{path}")
|
||||
status = local.status(local.remote_size(url))
|
||||
if status in LocalFile.NEEDS_DOWNLOAD:
|
||||
local.download(url)
|
||||
published = self.published_at(context, owner, repository, path, branch)
|
||||
return [(local.path, status)], None, {"published_at": published}
|
||||
|
||||
def published_at(self, context, owner, repository, path, branch):
|
||||
"""When this file was last committed, or None.
|
||||
|
||||
Best effort: a blob link has no release to read a date from, and
|
||||
a download should not fail over display information.
|
||||
"""
|
||||
try:
|
||||
response = context.session.get(
|
||||
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
|
||||
"/commits",
|
||||
params={"path": path, "sha": branch, "per_page": 1},
|
||||
)
|
||||
response.raise_for_status()
|
||||
commits = response.json()
|
||||
except requests.RequestException:
|
||||
return None
|
||||
if not commits:
|
||||
return None
|
||||
commit = commits[0].get("commit", {})
|
||||
return ((commit.get("committer") or {}).get("date")
|
||||
or (commit.get("author") or {}).get("date"))
|
||||
@@ -0,0 +1,81 @@
|
||||
"""A GitHub releases page, the sheet's usual link."""
|
||||
|
||||
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
|
||||
from archipelago_tester.pipeline.download.release_downloader import (
|
||||
ReleaseDownloader,
|
||||
)
|
||||
from archipelago_tester.pipeline.download.release_identity import (
|
||||
ReleaseIdentity,
|
||||
)
|
||||
from archipelago_tester.pipeline.download.release_picker import ReleasePicker
|
||||
from archipelago_tester.pipeline.download.walk_back import WalkBack
|
||||
|
||||
|
||||
class GitHubRelease(LinkHandler):
|
||||
"""Resolve which release the link means, then fetch its apworld."""
|
||||
|
||||
def __init__(self, shapes, previous=None):
|
||||
super().__init__(shapes)
|
||||
self.previous = previous
|
||||
|
||||
def parse(self, link):
|
||||
return self.shapes.releases(link)
|
||||
|
||||
def resolve(self, parsed, context):
|
||||
"""The release this link resolves to, or None.
|
||||
|
||||
A hand-pinned tag prefix wins, then a link naming one exact
|
||||
tag, then the search term. Runs every pass: it is the one call
|
||||
that can report a new release, which is the only thing able to
|
||||
change a previously failed answer.
|
||||
"""
|
||||
owner, repository, term, tag = parsed
|
||||
picker = ReleasePicker(context, owner, repository, term)
|
||||
overrides = context.config.get("release_overrides") or {}
|
||||
prefix = overrides.get(context.game.name)
|
||||
if prefix:
|
||||
return picker.by_tag_prefix(prefix)
|
||||
if tag:
|
||||
return picker.by_tag(tag)
|
||||
return picker.pick()
|
||||
|
||||
def fetch(self, parsed, context, release):
|
||||
"""This release's apworld, walking back if it ships none.
|
||||
|
||||
Only that one cause is walked back from. "Ships another game's
|
||||
apworld" must not be, since an older release of a multi-world
|
||||
repo is precisely the wrong answer there.
|
||||
"""
|
||||
owner, repository = parsed[0], parsed[1]
|
||||
downloader = ReleaseDownloader(
|
||||
context=context,
|
||||
release=release,
|
||||
# What this row's date already says, so a re-upload of
|
||||
# unchanged bytes keeps it instead of moving it forward.
|
||||
previous_published_at=(self.previous or {}).get("published_at"),
|
||||
)
|
||||
downloaded, reason, info = downloader.run()
|
||||
if (downloaded is not None or context.identifier is None
|
||||
or downloader.cause != ReleaseDownloader.NO_APWORLD):
|
||||
return downloaded, reason, info
|
||||
fallback, reason = WalkBack(
|
||||
context=context,
|
||||
owner=owner,
|
||||
repository=repository,
|
||||
release=release,
|
||||
).run(reason)
|
||||
return fallback if fallback is not None else (None, reason, info)
|
||||
|
||||
def download(self, parsed, context):
|
||||
release = self.resolve(parsed, context)
|
||||
if release is None:
|
||||
return None, "no release matched the search term in the link", None
|
||||
identity = ReleaseIdentity(release)
|
||||
repeat = identity.repeated(self.previous)
|
||||
if repeat is not None:
|
||||
return None, repeat, identity.failure(
|
||||
{"published_at": None}, repeat)
|
||||
downloaded, reason, info = self.fetch(parsed, context, release)
|
||||
if downloaded is None:
|
||||
return None, reason, identity.failure(info, reason)
|
||||
return downloaded, reason, info
|
||||
115
src/archipelago_tester/pipeline/download/handlers/github_repo.py
Normal file
115
src/archipelago_tester/pipeline/download/handlers/github_repo.py
Normal file
@@ -0,0 +1,115 @@
|
||||
"""A bare repository link, for a world never cut into a release."""
|
||||
|
||||
import os
|
||||
|
||||
import requests
|
||||
|
||||
from archipelago_tester.pipeline.build.archipelago import (
|
||||
ArchipelagoBuild,
|
||||
)
|
||||
from archipelago_tester.pipeline.download.local_file import LocalFile
|
||||
from archipelago_tester.core.model.apworld import ApworldFile
|
||||
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
|
||||
|
||||
|
||||
class GitHubRepo(LinkHandler):
|
||||
"""Browses the repo's own file tree for an apworld."""
|
||||
|
||||
def parse(self, link):
|
||||
return self.shapes.github_repo(link)
|
||||
|
||||
def resolve_branch(self, context, owner, repository, branch):
|
||||
"""The real default branch, so a version reads as a name.
|
||||
|
||||
"HEAD" works as a ref, but resolving it means the status page
|
||||
shows a branch rather than the literal string.
|
||||
"""
|
||||
if branch != "HEAD":
|
||||
return branch
|
||||
try:
|
||||
response = context.session.get(
|
||||
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}")
|
||||
response.raise_for_status()
|
||||
return response.json().get("default_branch") or "HEAD"
|
||||
except requests.RequestException:
|
||||
return "HEAD"
|
||||
|
||||
def tree_entries(self, context, owner, repository, branch):
|
||||
"""Every apworld in the tree, with its size.
|
||||
|
||||
One call: the trees API takes recursive=1 itself, and returns
|
||||
each blob's size for free.
|
||||
"""
|
||||
try:
|
||||
response = context.session.get(
|
||||
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
|
||||
f"/git/trees/{branch}",
|
||||
params={"recursive": "1"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
except requests.RequestException:
|
||||
return []
|
||||
return [
|
||||
(entry["path"], entry.get("size"))
|
||||
for entry in response.json().get("tree", [])
|
||||
if entry.get("type") == "blob"
|
||||
and entry["path"].lower().endswith(ApworldFile.SUFFIX)
|
||||
]
|
||||
|
||||
def candidates(self, entries):
|
||||
return [
|
||||
{"name": os.path.basename(path), "path": path, "size": size}
|
||||
for path, size in entries
|
||||
]
|
||||
|
||||
def download(self, parsed, context):
|
||||
owner, repository, branch = parsed
|
||||
branch = self.resolve_branch(context, owner, repository, branch)
|
||||
entries = self.tree_entries(context, owner, repository, branch)
|
||||
if not entries:
|
||||
return None, ("repo link has no release and no .apworld file "
|
||||
"found in it"), None
|
||||
chosen = context.matcher.select(self.candidates(entries))
|
||||
downloaded, kept, rejected = self.collect(
|
||||
candidates=chosen,
|
||||
context=context,
|
||||
fetch=lambda candidate: self.fetch(
|
||||
candidate, context, owner, repository, branch),
|
||||
verify=len(chosen) > 1,
|
||||
)
|
||||
if not downloaded:
|
||||
detail = self.rejected_detail(rejected, "file(s) from repo")
|
||||
return None, detail, None
|
||||
published = self.published_at(
|
||||
context, owner, repository, kept[0]["path"], branch)
|
||||
return downloaded, None, {"published_at": published}
|
||||
|
||||
def fetch(self, candidate, context, owner, repository, branch):
|
||||
local = self.file(context, candidate["name"])
|
||||
status = local.status(candidate.get("size"))
|
||||
if status in LocalFile.NEEDS_DOWNLOAD:
|
||||
local.download(
|
||||
f"https://raw.githubusercontent.com/{owner}/{repository}"
|
||||
f"/{branch}/{candidate['path']}")
|
||||
return local.path, status
|
||||
|
||||
def published_at(self, context, owner, repository, path, branch):
|
||||
"""When the first kept file was last committed.
|
||||
|
||||
Only the first: one API call per candidate otherwise.
|
||||
"""
|
||||
try:
|
||||
response = context.session.get(
|
||||
f"{ArchipelagoBuild.API_ROOT}/repos/{owner}/{repository}"
|
||||
"/commits",
|
||||
params={"path": path, "sha": branch, "per_page": 1},
|
||||
)
|
||||
response.raise_for_status()
|
||||
commits = response.json()
|
||||
except requests.RequestException:
|
||||
return None
|
||||
if not commits:
|
||||
return None
|
||||
commit = commits[0].get("commit", {})
|
||||
return ((commit.get("committer") or {}).get("date")
|
||||
or (commit.get("author") or {}).get("date"))
|
||||
@@ -0,0 +1,44 @@
|
||||
"""A link straight at an apworld committed in a GitLab repo."""
|
||||
|
||||
import os
|
||||
import urllib.parse
|
||||
|
||||
import requests
|
||||
|
||||
from archipelago_tester.pipeline.download.local_file import LocalFile
|
||||
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
|
||||
|
||||
|
||||
class GitLabFile(LinkHandler):
|
||||
"""A committed file on GitLab, named exactly by the sheet."""
|
||||
|
||||
def parse(self, link):
|
||||
return self.shapes.gitlab_file(link)
|
||||
|
||||
def download(self, parsed, context):
|
||||
project_path, branch, path = parsed
|
||||
filename = os.path.basename(urllib.parse.unquote(path))
|
||||
local = self.file(context, filename)
|
||||
url = f"https://gitlab.com/{project_path}/-/raw/{branch}/{path}"
|
||||
headers = context.json_headers
|
||||
status = local.status(local.remote_size(url, headers=headers))
|
||||
if status in LocalFile.NEEDS_DOWNLOAD:
|
||||
local.download(url, headers=headers)
|
||||
published = self.published_at(context, project_path, path, branch)
|
||||
return [(local.path, status)], None, {"published_at": published}
|
||||
|
||||
def published_at(self, context, project_path, path, branch):
|
||||
"""When this file was last committed, or None."""
|
||||
encoded = urllib.parse.quote(project_path, safe="")
|
||||
try:
|
||||
response = context.session.get(
|
||||
f"https://gitlab.com/api/v4/projects/{encoded}"
|
||||
"/repository/commits",
|
||||
params={"path": path, "ref_name": branch, "per_page": 1},
|
||||
headers=context.json_headers,
|
||||
)
|
||||
response.raise_for_status()
|
||||
commits = response.json()
|
||||
except requests.RequestException:
|
||||
return None
|
||||
return commits[0].get("committed_date") if commits else None
|
||||
@@ -0,0 +1,74 @@
|
||||
"""A GitLab releases page."""
|
||||
|
||||
import urllib.parse
|
||||
|
||||
from archipelago_tester.core.model.apworld import ApworldFile
|
||||
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
|
||||
from archipelago_tester.pipeline.download.local_file import LocalFile
|
||||
|
||||
|
||||
class GitLabRelease(LinkHandler):
|
||||
"""The newest release of a GitLab project.
|
||||
|
||||
GitLab's assets are shaped nothing like GitHub's - a release-level
|
||||
list of {name, url, direct_asset_url} with no per-asset timestamp -
|
||||
so they are reshaped here rather than reusing the release path.
|
||||
"""
|
||||
|
||||
def parse(self, link):
|
||||
return self.shapes.gitlab_releases(link)
|
||||
|
||||
def latest(self, context, project_path):
|
||||
"""GitLab's own newest-release shortcut, or None."""
|
||||
encoded = urllib.parse.quote(project_path, safe="")
|
||||
response = context.session.get(
|
||||
f"https://gitlab.com/api/v4/projects/{encoded}"
|
||||
"/releases/permalink/latest",
|
||||
headers=context.json_headers,
|
||||
)
|
||||
if response.status_code == 404:
|
||||
return None
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def assets(self, release, context):
|
||||
"""The release's apworld links, in GitHub's asset shape."""
|
||||
links = (release.get("assets") or {}).get("links", [])
|
||||
return [
|
||||
{
|
||||
"name": link["name"],
|
||||
"browser_download_url": (link.get("direct_asset_url")
|
||||
or link["url"]),
|
||||
}
|
||||
for link in links
|
||||
if (link.get("name") or "").lower().endswith(ApworldFile.SUFFIX)
|
||||
]
|
||||
|
||||
def fetch(self, asset, context):
|
||||
local = self.file(context, asset["name"])
|
||||
url = asset["browser_download_url"]
|
||||
headers = context.plain_headers
|
||||
status = local.status(local.remote_size(url, headers=headers))
|
||||
if status in LocalFile.NEEDS_DOWNLOAD:
|
||||
local.download(url, headers=headers)
|
||||
return local.path, status
|
||||
|
||||
def download(self, parsed, context):
|
||||
release = self.latest(context, parsed)
|
||||
if release is None:
|
||||
return None, "no GitLab release found for this project", None
|
||||
info = {"published_at": (release.get("released_at")
|
||||
or release.get("created_at"))}
|
||||
assets = self.assets(release, context)
|
||||
if not assets:
|
||||
return None, "latest GitLab release has no .apworld asset", info
|
||||
chosen = context.matcher.select(assets)
|
||||
downloaded, _, rejected = self.collect(
|
||||
candidates=chosen,
|
||||
context=context,
|
||||
fetch=lambda asset: self.fetch(asset, context),
|
||||
verify=len(chosen) > 1,
|
||||
)
|
||||
if not downloaded:
|
||||
return None, self.rejected_detail(rejected, "asset(s)"), info
|
||||
return downloaded, None, info
|
||||
@@ -0,0 +1,48 @@
|
||||
"""A plain URL ending in .apworld, on any host."""
|
||||
|
||||
import email.utils
|
||||
|
||||
import requests
|
||||
|
||||
from archipelago_tester.pipeline.download.local_file import LocalFile
|
||||
from archipelago_tester.pipeline.download.handlers.base import LinkHandler
|
||||
|
||||
|
||||
class PlainFile(LinkHandler):
|
||||
"""Any other host. Tried last, so a forge handler always wins."""
|
||||
|
||||
def parse(self, link):
|
||||
return self.shapes.plain_file(link)
|
||||
|
||||
def download(self, parsed, context):
|
||||
url, filename = parsed
|
||||
local = self.file(context, filename)
|
||||
size, published = self.remote_facts(context, url)
|
||||
status = local.status(size)
|
||||
if status in LocalFile.NEEDS_DOWNLOAD:
|
||||
local.download(url, headers=context.plain_headers)
|
||||
return [(local.path, status)], None, {"published_at": published}
|
||||
|
||||
def remote_facts(self, context, url):
|
||||
"""The size and date one HEAD request can offer.
|
||||
|
||||
A plain URL has no version to compare, so size is the whole
|
||||
staleness check. Neither is required.
|
||||
"""
|
||||
try:
|
||||
response = context.session.head(
|
||||
url, headers=context.plain_headers, allow_redirects=True)
|
||||
response.raise_for_status()
|
||||
except requests.RequestException:
|
||||
return None, None
|
||||
length = response.headers.get("Content-Length")
|
||||
size = int(length) if length and length.isdigit() else None
|
||||
return size, self.modified_at(response.headers.get("Last-Modified"))
|
||||
|
||||
def modified_at(self, modified):
|
||||
if not modified:
|
||||
return None
|
||||
try:
|
||||
return email.utils.parsedate_to_datetime(modified).isoformat()
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
Reference in New Issue
Block a user