"""Every distinct apworld file this pipeline has seen, per game. One file per game, written only when that game gains content it has not seen before - so a run touches the handful of games that actually changed rather than rewriting one shared blob. Deliberately NOT under downloads/: that directory is a cache and gets cleared to force a clean re-download, which is harmless for everything else in it. History is the one thing that cannot be rebuilt afterwards - which version existed on which date has no other source once it is gone. """ import os from datetime import datetime, timezone from archipelago_tester.core.model.apworld import ApworldFile from archipelago_tester.core.model.fingerprint import Fingerprint from archipelago_tester.core.model.name import Name from archipelago_tester.core.state.keys import StateKeys from archipelago_tester.core.state.store import StateStore class UpdateHistory: """What each game's apworld was, and when. Identity is the file's sha256, so a re-download of unchanged content adds nothing and recording is safe to repeat within a run. The game's sheet name is stored inside the file as well as encoded in its name, because the filename is a folded directory name and a sheet rename would otherwise leave a file nothing could identify. """ def __init__(self, paths, directory=None): self.directory = directory or paths.history def path_for(self, game_name): folded = Name(game_name).directory return os.path.join(self.directory, f"{folded}.json") def load(self, game_name): return StateStore.read( self.path_for(game_name), {"game": game_name, "watching_since": None, "baseline_at": None, "files": []}) def save(self, game_name, history): os.makedirs(self.directory, exist_ok=True) StateStore(self.path_for(game_name)).save(history) #: Everything recorded before this field existed was gathered from #: whatever the hosts happened to say, at whatever moment the file #: was first downloaded, and none of it measures how often a world #: changes under observation. The first pass to touch a game draws #: a line: that instant is the game's one starting point, and only #: changes seen after it count. See UpdateCadence, which reads it. @staticmethod def watching_since(history, stamp): """When this pipeline began watching this game. The earliest content it has already recorded, so a file written before this field existed still gets an honest answer rather than a window that starts the day the field was added. Only a game with no history at all starts watching now. """ seen = [item.get("first_seen_at") for item in history["files"] if item.get("first_seen_at")] return min(seen) if seen else stamp def entry_for(self, apworld_path, sha256, released_at, source_url, now): """One file's record. `released_at` is the upload date the download stage saw; first_seen_at is when this pipeline observed it, which is the only honest timestamp for a file whose host reports no date. """ manifest = ApworldFile(apworld_path).manifest or {} return { "version": manifest.get("world_version"), "released_at": released_at, "first_seen_at": (now or datetime.now(timezone.utc)).isoformat(), "sha256": sha256, "filename": os.path.basename(apworld_path), "source_url": source_url, } def record_file(self, game_name, apworld_path, sha256, released_at, source_url=None, now=None): """Add this file to the game's history if its content is new. A game whose content has not changed still records that it was looked at. How often a world updates is only meaningful against how long it has been watched, and without that a game seen once and a game seen all year are indistinguishable. """ history = self.load(game_name) now = now or datetime.now(timezone.utc) opened = not history.get("watching_since") if opened: history["watching_since"] = self.watching_since( history, now.isoformat()) if not history.get("baseline_at"): opened = True history["baseline_at"] = now.isoformat() if any(item.get("sha256") == sha256 for item in history["files"]): if opened: self.save(game_name, history) return None entry = self.entry_for(apworld_path, sha256, released_at, source_url, now) history["game"] = game_name history["files"].append(entry) # Oldest first. released_at is missing often enough that it # cannot be the only key - first_seen_at always exists and # breaks those ties in the order the pipeline saw them. history["files"].sort( key=lambda item: (item.get("released_at") or "", item.get("first_seen_at") or "")) self.save(game_name, history) return entry def files_in(self, game_directory): return [ name for name in sorted(os.listdir(game_directory)) if name.lower().endswith(ApworldFile.SUFFIX) and os.path.isfile(os.path.join(game_directory, name)) ] def record_all(self, state, root_directory, now=None): """Record every apworld on disk that is not yet in history. Hashes the files directly rather than reading apworld_tests: that dict only gains a hash once a world has been TESTED, so a file downloaded this run would not appear until the next one - and this runs right after downloading, which is the point at which a new version is known. """ releases = state.get(StateKeys.GAME_RELEASES) or {} # game_releases is keyed by the sheet name while the download # folder is that name folded - map one to the other rather than # assuming they match as written. by_folder = {Name(name).directory: info for name, info in releases.items()} added = [] if not os.path.isdir(root_directory): return added for folder in sorted(os.listdir(root_directory)): game_directory = os.path.join(root_directory, folder) if not os.path.isdir(game_directory): continue added += self.record_folder( folder, game_directory, by_folder.get(folder) or {}, now) return added def record_folder(self, folder, game_directory, release, now): """Every new apworld in one game's download folder.""" added = [] for filename in self.files_in(game_directory): path = os.path.join(game_directory, filename) entry = self.record_file( game_name=folder, apworld_path=path, sha256=Fingerprint.of_file(path), released_at=release.get("published_at"), source_url=release.get("url"), now=now, ) if entry: added.append((folder, entry)) return added