initial commit

src/archipelago_tester/pipeline/download/update_history.py

@@ -0,0 +1,168 @@
+"""Every distinct apworld file this pipeline has seen, per game.
+
+One file per game, written only when that game gains content it has not
+seen before - so a run touches the handful of games that actually
+changed rather than rewriting one shared blob. Deliberately NOT under
+downloads/: that directory is a cache and gets cleared to force a clean
+re-download, which is harmless for everything else in it. History is the
+one thing that cannot be rebuilt afterwards - which version existed on
+which date has no other source once it is gone.
+"""
+
+import os
+from datetime import datetime, timezone
+
+from archipelago_tester.core.model.apworld import ApworldFile
+from archipelago_tester.core.model.fingerprint import Fingerprint
+from archipelago_tester.core.model.name import Name
+from archipelago_tester.core.state.keys import StateKeys
+from archipelago_tester.core.state.store import StateStore
+
+
+class UpdateHistory:
+    """What each game's apworld was, and when.
+
+    Identity is the file's sha256, so a re-download of unchanged content
+    adds nothing and recording is safe to repeat within a run. The
+    game's sheet name is stored inside the file as well as encoded in
+    its name, because the filename is a folded directory name and a
+    sheet rename would otherwise leave a file nothing could identify.
+    """
+
+    def __init__(self, paths, directory=None):
+        self.directory = directory or paths.history
+
+    def path_for(self, game_name):
+        folded = Name(game_name).directory
+        return os.path.join(self.directory, f"{folded}.json")
+
+    def load(self, game_name):
+        return StateStore.read(
+            self.path_for(game_name),
+            {"game": game_name, "watching_since": None,
+             "baseline_at": None, "files": []})
+
+    def save(self, game_name, history):
+        os.makedirs(self.directory, exist_ok=True)
+        StateStore(self.path_for(game_name)).save(history)
+
+    #: Everything recorded before this field existed was gathered from
+    #: whatever the hosts happened to say, at whatever moment the file
+    #: was first downloaded, and none of it measures how often a world
+    #: changes under observation. The first pass to touch a game draws
+    #: a line: that instant is the game's one starting point, and only
+    #: changes seen after it count. See UpdateCadence, which reads it.
+    @staticmethod
+    def watching_since(history, stamp):
+        """When this pipeline began watching this game.
+
+        The earliest content it has already recorded, so a file written
+        before this field existed still gets an honest answer rather
+        than a window that starts the day the field was added. Only a
+        game with no history at all starts watching now.
+        """
+        seen = [item.get("first_seen_at") for item in history["files"]
+                if item.get("first_seen_at")]
+        return min(seen) if seen else stamp
+
+    def entry_for(self, apworld_path, sha256, released_at, source_url, now):
+        """One file's record.
+
+        `released_at` is the upload date the download stage saw;
+        first_seen_at is when this pipeline observed it, which is the
+        only honest timestamp for a file whose host reports no date.
+        """
+        manifest = ApworldFile(apworld_path).manifest or {}
+        return {
+            "version": manifest.get("world_version"),
+            "released_at": released_at,
+            "first_seen_at": (now or datetime.now(timezone.utc)).isoformat(),
+            "sha256": sha256,
+            "filename": os.path.basename(apworld_path),
+            "source_url": source_url,
+        }
+
+    def record_file(self, game_name, apworld_path, sha256, released_at,
+                    source_url=None, now=None):
+        """Add this file to the game's history if its content is new.
+
+        A game whose content has not changed still records that it was
+        looked at. How often a world updates is only meaningful against
+        how long it has been watched, and without that a game seen once
+        and a game seen all year are indistinguishable.
+        """
+        history = self.load(game_name)
+        now = now or datetime.now(timezone.utc)
+        opened = not history.get("watching_since")
+        if opened:
+            history["watching_since"] = self.watching_since(
+                history, now.isoformat())
+        if not history.get("baseline_at"):
+            opened = True
+            history["baseline_at"] = now.isoformat()
+        if any(item.get("sha256") == sha256 for item in history["files"]):
+            if opened:
+                self.save(game_name, history)
+            return None
+        entry = self.entry_for(apworld_path, sha256, released_at,
+                               source_url, now)
+        history["game"] = game_name
+        history["files"].append(entry)
+        # Oldest first. released_at is missing often enough that it
+        # cannot be the only key - first_seen_at always exists and
+        # breaks those ties in the order the pipeline saw them.
+        history["files"].sort(
+            key=lambda item: (item.get("released_at") or "",
+                              item.get("first_seen_at") or ""))
+        self.save(game_name, history)
+        return entry
+
+    def files_in(self, game_directory):
+        return [
+            name for name in sorted(os.listdir(game_directory))
+            if name.lower().endswith(ApworldFile.SUFFIX)
+            and os.path.isfile(os.path.join(game_directory, name))
+        ]
+
+    def record_all(self, state, root_directory, now=None):
+        """Record every apworld on disk that is not yet in history.
+
+        Hashes the files directly rather than reading apworld_tests:
+        that dict only gains a hash once a world has been TESTED, so a
+        file downloaded this run would not appear until the next one -
+        and this runs right after downloading, which is the point at
+        which a new version is known.
+        """
+        releases = state.get(StateKeys.GAME_RELEASES) or {}
+        # game_releases is keyed by the sheet name while the download
+        # folder is that name folded - map one to the other rather than
+        # assuming they match as written.
+        by_folder = {Name(name).directory: info
+                     for name, info in releases.items()}
+        added = []
+        if not os.path.isdir(root_directory):
+            return added
+        for folder in sorted(os.listdir(root_directory)):
+            game_directory = os.path.join(root_directory, folder)
+            if not os.path.isdir(game_directory):
+                continue
+            added += self.record_folder(
+                folder, game_directory, by_folder.get(folder) or {}, now)
+        return added
+
+    def record_folder(self, folder, game_directory, release, now):
+        """Every new apworld in one game's download folder."""
+        added = []
+        for filename in self.files_in(game_directory):
+            path = os.path.join(game_directory, filename)
+            entry = self.record_file(
+                game_name=folder,
+                apworld_path=path,
+                sha256=Fingerprint.of_file(path),
+                released_at=release.get("published_at"),
+                source_url=release.get("url"),
+                now=now,
+            )
+            if entry:
+                added.append((folder, entry))
+        return added