169 lines
7.2 KiB
Python
169 lines
7.2 KiB
Python
"""Every distinct apworld file this pipeline has seen, per game.
|
|
|
|
One file per game, written only when that game gains content it has not
|
|
seen before - so a run touches the handful of games that actually
|
|
changed rather than rewriting one shared blob. Deliberately NOT under
|
|
downloads/: that directory is a cache and gets cleared to force a clean
|
|
re-download, which is harmless for everything else in it. History is the
|
|
one thing that cannot be rebuilt afterwards - which version existed on
|
|
which date has no other source once it is gone.
|
|
"""
|
|
|
|
import os
|
|
from datetime import datetime, timezone
|
|
|
|
from archipelago_tester.core.model.apworld import ApworldFile
|
|
from archipelago_tester.core.model.fingerprint import Fingerprint
|
|
from archipelago_tester.core.model.name import Name
|
|
from archipelago_tester.core.state.keys import StateKeys
|
|
from archipelago_tester.core.state.store import StateStore
|
|
|
|
|
|
class UpdateHistory:
|
|
"""What each game's apworld was, and when.
|
|
|
|
Identity is the file's sha256, so a re-download of unchanged content
|
|
adds nothing and recording is safe to repeat within a run. The
|
|
game's sheet name is stored inside the file as well as encoded in
|
|
its name, because the filename is a folded directory name and a
|
|
sheet rename would otherwise leave a file nothing could identify.
|
|
"""
|
|
|
|
def __init__(self, paths, directory=None):
|
|
self.directory = directory or paths.history
|
|
|
|
def path_for(self, game_name):
|
|
folded = Name(game_name).directory
|
|
return os.path.join(self.directory, f"{folded}.json")
|
|
|
|
def load(self, game_name):
|
|
return StateStore.read(
|
|
self.path_for(game_name),
|
|
{"game": game_name, "watching_since": None,
|
|
"baseline_at": None, "files": []})
|
|
|
|
def save(self, game_name, history):
|
|
os.makedirs(self.directory, exist_ok=True)
|
|
StateStore(self.path_for(game_name)).save(history)
|
|
|
|
#: Everything recorded before this field existed was gathered from
|
|
#: whatever the hosts happened to say, at whatever moment the file
|
|
#: was first downloaded, and none of it measures how often a world
|
|
#: changes under observation. The first pass to touch a game draws
|
|
#: a line: that instant is the game's one starting point, and only
|
|
#: changes seen after it count. See UpdateCadence, which reads it.
|
|
@staticmethod
|
|
def watching_since(history, stamp):
|
|
"""When this pipeline began watching this game.
|
|
|
|
The earliest content it has already recorded, so a file written
|
|
before this field existed still gets an honest answer rather
|
|
than a window that starts the day the field was added. Only a
|
|
game with no history at all starts watching now.
|
|
"""
|
|
seen = [item.get("first_seen_at") for item in history["files"]
|
|
if item.get("first_seen_at")]
|
|
return min(seen) if seen else stamp
|
|
|
|
def entry_for(self, apworld_path, sha256, released_at, source_url, now):
|
|
"""One file's record.
|
|
|
|
`released_at` is the upload date the download stage saw;
|
|
first_seen_at is when this pipeline observed it, which is the
|
|
only honest timestamp for a file whose host reports no date.
|
|
"""
|
|
manifest = ApworldFile(apworld_path).manifest or {}
|
|
return {
|
|
"version": manifest.get("world_version"),
|
|
"released_at": released_at,
|
|
"first_seen_at": (now or datetime.now(timezone.utc)).isoformat(),
|
|
"sha256": sha256,
|
|
"filename": os.path.basename(apworld_path),
|
|
"source_url": source_url,
|
|
}
|
|
|
|
def record_file(self, game_name, apworld_path, sha256, released_at,
|
|
source_url=None, now=None):
|
|
"""Add this file to the game's history if its content is new.
|
|
|
|
A game whose content has not changed still records that it was
|
|
looked at. How often a world updates is only meaningful against
|
|
how long it has been watched, and without that a game seen once
|
|
and a game seen all year are indistinguishable.
|
|
"""
|
|
history = self.load(game_name)
|
|
now = now or datetime.now(timezone.utc)
|
|
opened = not history.get("watching_since")
|
|
if opened:
|
|
history["watching_since"] = self.watching_since(
|
|
history, now.isoformat())
|
|
if not history.get("baseline_at"):
|
|
opened = True
|
|
history["baseline_at"] = now.isoformat()
|
|
if any(item.get("sha256") == sha256 for item in history["files"]):
|
|
if opened:
|
|
self.save(game_name, history)
|
|
return None
|
|
entry = self.entry_for(apworld_path, sha256, released_at,
|
|
source_url, now)
|
|
history["game"] = game_name
|
|
history["files"].append(entry)
|
|
# Oldest first. released_at is missing often enough that it
|
|
# cannot be the only key - first_seen_at always exists and
|
|
# breaks those ties in the order the pipeline saw them.
|
|
history["files"].sort(
|
|
key=lambda item: (item.get("released_at") or "",
|
|
item.get("first_seen_at") or ""))
|
|
self.save(game_name, history)
|
|
return entry
|
|
|
|
def files_in(self, game_directory):
|
|
return [
|
|
name for name in sorted(os.listdir(game_directory))
|
|
if name.lower().endswith(ApworldFile.SUFFIX)
|
|
and os.path.isfile(os.path.join(game_directory, name))
|
|
]
|
|
|
|
def record_all(self, state, root_directory, now=None):
|
|
"""Record every apworld on disk that is not yet in history.
|
|
|
|
Hashes the files directly rather than reading apworld_tests:
|
|
that dict only gains a hash once a world has been TESTED, so a
|
|
file downloaded this run would not appear until the next one -
|
|
and this runs right after downloading, which is the point at
|
|
which a new version is known.
|
|
"""
|
|
releases = state.get(StateKeys.GAME_RELEASES) or {}
|
|
# game_releases is keyed by the sheet name while the download
|
|
# folder is that name folded - map one to the other rather than
|
|
# assuming they match as written.
|
|
by_folder = {Name(name).directory: info
|
|
for name, info in releases.items()}
|
|
added = []
|
|
if not os.path.isdir(root_directory):
|
|
return added
|
|
for folder in sorted(os.listdir(root_directory)):
|
|
game_directory = os.path.join(root_directory, folder)
|
|
if not os.path.isdir(game_directory):
|
|
continue
|
|
added += self.record_folder(
|
|
folder, game_directory, by_folder.get(folder) or {}, now)
|
|
return added
|
|
|
|
def record_folder(self, folder, game_directory, release, now):
|
|
"""Every new apworld in one game's download folder."""
|
|
added = []
|
|
for filename in self.files_in(game_directory):
|
|
path = os.path.join(game_directory, filename)
|
|
entry = self.record_file(
|
|
game_name=folder,
|
|
apworld_path=path,
|
|
sha256=Fingerprint.of_file(path),
|
|
released_at=release.get("published_at"),
|
|
source_url=release.get("url"),
|
|
now=now,
|
|
)
|
|
if entry:
|
|
added.append((folder, entry))
|
|
return added
|