diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e98b960..39679e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: CI on: push: + branches-ignore: [gh-pages] pull_request: permissions: diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 3b962aa..70a9574 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -6,16 +6,14 @@ on: workflow_dispatch: permissions: - contents: read - pages: write - id-token: write + contents: write concurrency: - group: pages + group: pages-production cancel-in-progress: true jobs: - build: + deploy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -24,16 +22,12 @@ jobs: enable-cache: true - name: Build tvtime-core wheel into web/dist run: ./scripts/build-web.sh - - uses: actions/upload-pages-artifact@v3 + - name: Deploy to gh-pages root + uses: JamesIves/github-pages-deploy-action@v4 with: - path: web - - deploy: - needs: build - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - id: deployment - uses: actions/deploy-pages@v4 + folder: web + branch: gh-pages + clean: true + # Keep open PR previews when production redeploys. + clean-exclude: | + pr-preview/ diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml new file mode 100644 index 0000000..56063eb --- /dev/null +++ b/.github/workflows/preview.yml @@ -0,0 +1,32 @@ +name: PR preview + +on: + pull_request: + types: [opened, reopened, synchronize, closed] + +permissions: + contents: write + pull-requests: write + +concurrency: + group: preview-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + preview: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build the site + if: github.event.action != 'closed' + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + - if: github.event.action != 'closed' + run: ./scripts/build-web.sh + - name: Deploy / update / remove the PR preview + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: web + preview-branch: gh-pages + umbrella-dir: pr-preview diff --git a/README.md b/README.md index 3d32a44..0c8a731 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # TV Time Data Recovery +[![CI](https://github.com/burgesQ/tvtime/actions/workflows/ci.yml/badge.svg)](https://github.com/burgesQ/tvtime/actions/workflows/ci.yml) +[![Deploy Pages](https://github.com/burgesQ/tvtime/actions/workflows/pages.yml/badge.svg)](https://github.com/burgesQ/tvtime/actions/workflows/pages.yml) +[![Web app](https://img.shields.io/badge/web%20app-burgesq.github.io%2Ftvtime-2f6f4f)](https://burgesq.github.io/tvtime/) +[![License: WTFPL](https://img.shields.io/badge/License-WTFPL-brightgreen.svg)](LICENSE) + Recover your library and watch history from the TV Time Android app's local cache (`DioCache.db`) and export it to CSV. @@ -8,16 +13,25 @@ API responses it has seen. When the online service is unavailable, that cache is often the only remaining copy of your data. This project parses it back into usable CSV files. +If that cache (`DioCache.db`) has been cleared and reads as empty, the tool also +recovers a **name-level library and your "up next" list** from the app's Flutter +preferences (`shared_prefs/FlutterSharedPreferences.xml` and +`HomeWidgetPreferences.xml`). You can hand it the whole `tvtime-private.tar` you +pulled from the phone and it will find and merge whatever it can. + > Not affiliated with or endorsed by TV Time. This recovers only data that is still > cached locally on your device; it is **not** an official export and completeness > depends entirely on what your device kept. ## Three ways to use it -- **Web app** (recommended, no install): upload your `DioCache.db` and download CSVs. - The file is processed **entirely in your browser** and is never uploaded anywhere. -- **CLI**: `tvtime-recover /path/to/DioCache.db ./tvtime-export` -- **Library** (`tvtime-core`): `recover(db_bytes)` for use in your own scripts. +- **Web app** (recommended, no install): upload your `DioCache.db`, the whole + `tvtime-private.tar`, or your `shared_prefs` files and download CSVs. Everything is + processed **entirely in your browser** and is never uploaded anywhere. +- **CLI**: `tvtime-recover --out ./tvtime-export` (or `--zip out.zip`), + where each input is a `DioCache.db`, a `tvtime-private` tar/zip, or a prefs `.xml`. +- **Library** (`tvtime-core`): `recover_files({name: bytes})` (or `recover(db_bytes)` + for a single DioCache) for use in your own scripts. ## Getting your DioCache.db diff --git a/docs/extraction-guide.md b/docs/extraction-guide.md index 00c494f..e4551e2 100644 --- a/docs/extraction-guide.md +++ b/docs/extraction-guide.md @@ -105,3 +105,22 @@ sqlite3 "/path/to/DioCache.db" "SELECT COUNT(*) FROM cache_dio;" You want a `cache_dio` table, an integrity result of `ok`, and a non-zero row count. Then feed `DioCache.db` to the [web app](../web/index.html) or the CLI. + +## If `cache_dio` is empty + +`DioCache.db` is only an HTTP cache, so it is frequently empty (a `SELECT COUNT(*) +FROM cache_dio;` of `0`). Do not give up — the app keeps a durable, name-level copy of +your library and your "up next" list in its Flutter preferences: + +``` +shared_prefs/FlutterSharedPreferences.xml # library names by category (braze_* keys) +shared_prefs/HomeWidgetPreferences.xml # the "up next" episodes (showsWatchList) +``` + +The tool reads these too. The easiest path is to hand it the **whole** +`tvtime-private.tar` (or `.zip`) you extracted — it will locate `DioCache.db` and the +`shared_prefs` files inside and merge whatever it finds. You can also select the two +XML files directly. + +> Note: this recovers show and movie **names** (plus your up-next episodes), not a full +> per-episode watch history with timestamps — that only ever lived in `DioCache.db`. diff --git a/packages/tvtime-cli/pyproject.toml b/packages/tvtime-cli/pyproject.toml index deadb4a..f990b23 100644 --- a/packages/tvtime-cli/pyproject.toml +++ b/packages/tvtime-cli/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tvtime-cli" -version = "0.1.0" +version = "0.2.0" description = "Command-line tool to recover TV Time data from a DioCache.db" readme = "README.md" requires-python = ">=3.10" diff --git a/packages/tvtime-cli/src/tvtime_cli/__init__.py b/packages/tvtime-cli/src/tvtime_cli/__init__.py index a5387f9..93f08ce 100644 --- a/packages/tvtime-cli/src/tvtime_cli/__init__.py +++ b/packages/tvtime-cli/src/tvtime_cli/__init__.py @@ -1,7 +1,8 @@ """Command-line front end for TV Time data recovery. -Reads a ``DioCache.db`` and writes the recovered records either as CSV files in -an output directory or as a single ZIP archive. +Reads one or more inputs — a ``DioCache.db``, a ``tvtime-private`` tar/zip, or +loose ``shared_prefs`` XML files — and writes the recovered records either as +CSV files in a directory or as a single ZIP. """ from __future__ import annotations @@ -10,7 +11,7 @@ import sys from pathlib import Path -from tvtime_core import recover +from tvtime_core import recover_files __all__ = ["main"] @@ -18,13 +19,18 @@ def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="tvtime-recover", - description="Recover TV Time data from an Android DioCache.db into CSV.", + description="Recover TV Time data from an Android DioCache.db or shared_prefs into CSV.", ) - parser.add_argument("database", type=Path, help="Path to the extracted DioCache.db") parser.add_argument( - "output", + "inputs", + type=Path, + nargs="+", + help="Inputs: a DioCache.db, a tvtime-private tar/zip, or shared_prefs .xml files", + ) + parser.add_argument( + "-o", + "--out", type=Path, - nargs="?", help="Directory to write the CSV exports into", ) parser.add_argument( @@ -51,12 +57,13 @@ def main(argv: list[str] | None = None) -> int: parser = _build_parser() args = parser.parse_args(argv) - if args.zip_path is None and args.output is None: + if args.zip_path is None and args.out is None: parser.print_usage(sys.stderr) - print("tvtime-recover: provide an output directory or --zip PATH", file=sys.stderr) + print("tvtime-recover: provide an output directory (--out) or --zip PATH", file=sys.stderr) return 2 - result = recover(args.database.read_bytes()) + files = {str(path): path.read_bytes() for path in args.inputs} + result = recover_files(files) summary = result.summary() if args.zip_path is not None: @@ -64,12 +71,12 @@ def main(argv: list[str] | None = None) -> int: args.zip_path.write_bytes(result.to_zip_bytes()) destination: Path = args.zip_path else: - _write_directory(result.to_csv_map(), result.raw_cache, args.output) - destination = args.output + _write_directory(result.to_csv_map(), result.raw_cache, args.out) + destination = args.out - print(f"Cache entries: {summary['cache_entries']}") - print(f"Library records: {summary['library']} {summary['library_by_type']}") + print(f"Inputs read: {len(files)}") + print(f"Library records: {summary['library']} {summary['library_by_source']}") print(f"Movie watch-history records: {summary['movie_history']}") - print(f"Cached episode records: {summary['episodes']}") + print(f"Cached/up-next episode records: {summary['episodes']}") print(f"Exports written to: {destination}") return 0 diff --git a/packages/tvtime-cli/tests/test_cli.py b/packages/tvtime-cli/tests/test_cli.py index b695f30..475489a 100644 --- a/packages/tvtime-cli/tests/test_cli.py +++ b/packages/tvtime-cli/tests/test_cli.py @@ -49,7 +49,7 @@ def test_cli_writes_export_directory(tmp_path: Path, capsys: pytest.CaptureFixtu _make_db(db_path) out_dir = tmp_path / "export" - exit_code = main([str(db_path), str(out_dir)]) + exit_code = main([str(db_path), "--out", str(out_dir)]) assert exit_code == 0 assert (out_dir / "tvtime_library.csv").exists() @@ -69,6 +69,19 @@ def test_cli_writes_single_zip(tmp_path: Path) -> None: assert "tvtime_library.csv" in archive.namelist() +def test_cli_accepts_multiple_inputs(tmp_path: Path) -> None: + db_a = tmp_path / "a.db" + db_b = tmp_path / "b.db" + _make_db(db_a) + _make_db(db_b) + out_dir = tmp_path / "export" + + exit_code = main([str(db_a), str(db_b), "--out", str(out_dir)]) + + assert exit_code == 0 + assert (out_dir / "tvtime_library.csv").exists() + + def test_cli_requires_an_output(tmp_path: Path) -> None: db_path = tmp_path / "DioCache.db" _make_db(db_path) diff --git a/packages/tvtime-core/pyproject.toml b/packages/tvtime-core/pyproject.toml index 264723b..4618f2a 100644 --- a/packages/tvtime-core/pyproject.toml +++ b/packages/tvtime-core/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tvtime-core" -version = "0.1.0" +version = "0.2.0" description = "Parse a TV Time Android DioCache.db into CSV/ZIP exports (stdlib only)" readme = "README.md" requires-python = ">=3.10" diff --git a/packages/tvtime-core/src/tvtime_core/__init__.py b/packages/tvtime-core/src/tvtime_core/__init__.py index f996646..8fef20b 100644 --- a/packages/tvtime-core/src/tvtime_core/__init__.py +++ b/packages/tvtime-core/src/tvtime_core/__init__.py @@ -4,5 +4,6 @@ from tvtime_core.models import RecoveryResult from tvtime_core.parser import recover +from tvtime_core.sources import recover_files -__all__ = ["RecoveryResult", "recover"] +__all__ = ["RecoveryResult", "recover", "recover_files"] diff --git a/packages/tvtime-core/src/tvtime_core/models.py b/packages/tvtime-core/src/tvtime_core/models.py index 46dbfce..77585cf 100644 --- a/packages/tvtime-core/src/tvtime_core/models.py +++ b/packages/tvtime-core/src/tvtime_core/models.py @@ -16,25 +16,39 @@ def _rows_to_csv(rows: list[dict[str, Any]]) -> str: - """Render rows as CSV text (BOM-free). Empty input yields an empty string.""" + """Render rows as CSV text (BOM-free). Empty input yields an empty string. + + Rows may be heterogeneous (they can come from different recovery sources); + the header is the union of keys in first-seen order and missing cells blank. + """ if not rows: return "" + fieldnames: list[str] = [] + for row in rows: + for key in row: + if key not in fieldnames: + fieldnames.append(key) buffer = io.StringIO() - writer = csv.DictWriter(buffer, fieldnames=list(rows[0])) + writer = csv.DictWriter(buffer, fieldnames=fieldnames, restval="") writer.writeheader() writer.writerows(rows) return buffer.getvalue() -def counts_by_type(rows: list[dict[str, Any]]) -> dict[str, int]: - """Count rows by their ``entity_type`` field.""" +def _counts_by(rows: list[dict[str, Any]], key: str, default: str) -> dict[str, int]: + """Count rows by a given field.""" counts: dict[str, int] = {} for row in rows: - key = str(row.get("entity_type", "unknown")) - counts[key] = counts.get(key, 0) + 1 + value = str(row.get(key, default)) + counts[value] = counts.get(value, 0) + 1 return counts +def counts_by_type(rows: list[dict[str, Any]]) -> dict[str, int]: + """Count rows by their ``entity_type`` field.""" + return _counts_by(rows, "entity_type", "unknown") + + @dataclass class RecoveryResult: """Structured output of :func:`tvtime_core.parser.recover`.""" @@ -71,6 +85,7 @@ def summary(self) -> dict[str, Any]: "cache_entries": len(self.cache_index), "library": len(self.library), "library_by_type": counts_by_type(self.library), + "library_by_source": _counts_by(self.library, "source", "dio_cache"), "movie_history": len(self.movie_history), "episodes": len(self.episodes), } diff --git a/packages/tvtime-core/src/tvtime_core/prefs.py b/packages/tvtime-core/src/tvtime_core/prefs.py new file mode 100644 index 0000000..d74f988 --- /dev/null +++ b/packages/tvtime-core/src/tvtime_core/prefs.py @@ -0,0 +1,134 @@ +"""Recover TV Time data from the app's Flutter shared-preferences files. + +When ``DioCache.db`` is empty, durable data still lives in ``shared_prefs``: + +- ``FlutterSharedPreferences.xml`` holds ``braze_*`` keys, each a + base64-encoded, Java-serialized ``ArrayList`` of show/movie names. +- ``HomeWidgetPreferences.xml`` holds ``showsWatchList``, a JSON list of the + "up next" episodes. + +All parsing is pure and standard-library only, so it runs under Pyodide. +""" + +from __future__ import annotations + +import base64 +import binascii +import json +from dataclasses import dataclass, field +from typing import Any +from xml.etree import ElementTree as ET + +# Braze pref key -> (bucket, category). +BRAZE_CATEGORIES: dict[str, tuple[str, str]] = { + "flutter.braze_shows_continue_watching_names": ("shows", "continue_watching"), + "flutter.braze_shows_not_watched_names": ("shows", "not_watched"), + "flutter.braze_shows_not_started_names": ("shows", "not_started"), + "flutter.braze_favorite_movies": ("movies", "favorite_movies"), + "flutter.braze_available_movies_to_track": ("movies", "movies_to_track"), +} + + +@dataclass +class FlutterPrefs: + """Show and movie names recovered from ``FlutterSharedPreferences.xml``.""" + + shows: dict[str, list[str]] = field(default_factory=dict) + movies: dict[str, list[str]] = field(default_factory=dict) + + +def decode_java_string_list(value: str) -> list[str]: + """Decode a Braze base64 / Java-serialized ``ArrayList`` value. + + Returns ``[]`` for anything that is not a recognizable list. Repeated + strings serialized as back-references are not resolved (the observed data + contains only unique, inline strings). + """ + if not value: + return [] + try: + raw = base64.b64decode(value) + except (binascii.Error, ValueError): + return [] + start = raw.find(b"\xac\xed\x00\x05") # Java STREAM_MAGIC + version + if start < 0: + return [] + data = raw[start:] + marker = data.find(b"\x77\x04") # TC_BLOCKDATA holding the element count + if marker < 0: + return [] + index = marker + 6 # skip the marker and its 4-byte count + names: list[str] = [] + length = len(data) + while index < length and data[index] == 0x74: # TC_STRING + if index + 3 > length: + break + size = int.from_bytes(data[index + 1 : index + 3], "big") + chunk = data[index + 3 : index + 3 + size] + if len(chunk) < size: + break + names.append(chunk.decode("utf-8", "replace")) + index += 3 + size + return names + + +def _string_prefs(xml_text: str) -> dict[str, str]: + """Return ``{name: text}`` for every ```` in the prefs XML.""" + try: + root = ET.fromstring(xml_text.encode("utf-8")) + except (ET.ParseError, ValueError): + return {} + result: dict[str, str] = {} + for element in root.iter("string"): + name = element.get("name") + if name is not None: + result[name] = (element.text or "").strip() + return result + + +def parse_flutter_prefs(xml_text: str) -> FlutterPrefs: + """Parse ``FlutterSharedPreferences.xml`` into categorized show/movie names. + + Deliberately ignores ``flutter.user`` (the personal profile). + """ + strings = _string_prefs(xml_text) + prefs = FlutterPrefs() + for key, (bucket, category) in BRAZE_CATEGORIES.items(): + if key not in strings: + continue + names = decode_java_string_list(strings[key]) + target = prefs.shows if bucket == "shows" else prefs.movies + target[category] = names + return prefs + + +def parse_home_widget(xml_text: str) -> list[dict[str, Any]]: + """Parse ``HomeWidgetPreferences.xml`` ``showsWatchList`` into up-next episodes.""" + raw = _string_prefs(xml_text).get("showsWatchList") + if not raw: + return [] + try: + items = json.loads(raw) + except json.JSONDecodeError: + return [] + if not isinstance(items, list): + return [] + up_next: list[dict[str, Any]] = [] + for item in items: + if not isinstance(item, dict): + continue + show_raw = item.get("show") + show = show_raw if isinstance(show_raw, dict) else {} + season_raw = item.get("season") + season = season_raw if isinstance(season_raw, dict) else {} + up_next.append( + { + "show": show.get("name"), + "season": season.get("number"), + "episode": item.get("number"), + "episode_name": item.get("name"), + "network": item.get("network"), + "category": item.get("to_watch_category"), + } + ) + return up_next diff --git a/packages/tvtime-core/src/tvtime_core/sources.py b/packages/tvtime-core/src/tvtime_core/sources.py new file mode 100644 index 0000000..5760701 --- /dev/null +++ b/packages/tvtime-core/src/tvtime_core/sources.py @@ -0,0 +1,170 @@ +"""Unified, multi-source recovery entry point. + +Accepts any mix of inputs a user might have — a ``DioCache.db``, the whole +``tvtime-private.tar``/``.zip`` extracted from the phone, or loose +``shared_prefs`` XML files — classifies each, runs the matching parser, and +merges everything into one :class:`RecoveryResult`. + +Pure and standard-library only, so it runs unchanged under Pyodide. +""" + +from __future__ import annotations + +import io +import sqlite3 +import tarfile +import zipfile +from collections.abc import Mapping +from typing import Any + +from tvtime_core.models import RecoveryResult +from tvtime_core.parser import recover +from tvtime_core.prefs import parse_flutter_prefs, parse_home_widget + +_SQLITE_MAGIC = b"SQLite format 3\x00" +_ZIP_MAGIC = b"PK\x03\x04" +_MAX_MEMBER_BYTES = 25 * 1024 * 1024 # skip oversized archive members + + +def _looks_like_tar(data: bytes) -> bool: + return len(data) > 262 and data[257:262] == b"ustar" + + +def _relevant_member(name: str) -> bool: + lowered = name.lower() + return lowered.endswith((".db", ".sqlite", ".sqlite3", ".xml")) + + +def _expand(name: str, data: bytes) -> dict[str, bytes]: + """Return the files to classify from one input, expanding tar/zip archives.""" + lowered = name.lower() + if lowered.endswith(".zip") or data[:4] == _ZIP_MAGIC: + return _expand_zip(data) + if lowered.endswith((".tar", ".tar.gz", ".tgz")) or _looks_like_tar(data): + return _expand_tar(data) + return {name: data} + + +def _expand_zip(data: bytes) -> dict[str, bytes]: + found: dict[str, bytes] = {} + try: + with zipfile.ZipFile(io.BytesIO(data)) as archive: + for info in archive.infolist(): + if info.is_dir() or not _relevant_member(info.filename): + continue + if info.file_size > _MAX_MEMBER_BYTES: + continue + found[info.filename] = archive.read(info) + except (zipfile.BadZipFile, OSError): + return {} + return found + + +def _expand_tar(data: bytes) -> dict[str, bytes]: + found: dict[str, bytes] = {} + try: + with tarfile.open(fileobj=io.BytesIO(data)) as archive: + for member in archive.getmembers(): + if not member.isfile() or not _relevant_member(member.name): + continue + if member.size > _MAX_MEMBER_BYTES: + continue + handle = archive.extractfile(member) + if handle is not None: + found[member.name] = handle.read() + except (tarfile.TarError, OSError): + return {} + return found + + +def _classify(data: bytes) -> str: + if data[: len(_SQLITE_MAGIC)] == _SQLITE_MAGIC: + return "sqlite" + text = data[:4096].decode("utf-8", "ignore") + if "showsWatchList" in text: + return "home_widget" + if "flutter.braze_" in text or "FlutterSharedPreferences" in text or "flutter." in text: + return "flutter_prefs" + return "unknown" + + +def _merge_category(row: dict[str, Any], category: str) -> None: + existing = str(row.get("progress_status", "")) + parts = [part for part in existing.split(" | ") if part] + if category not in parts: + parts.append(category) + row["progress_status"] = " | ".join(parts) + + +def _up_next_episode(entry: dict[str, Any]) -> dict[str, Any]: + return { + "show_name": entry.get("show"), + "season_number": entry.get("season"), + "episode_number": entry.get("episode"), + "episode_name": entry.get("episode_name"), + "network": entry.get("network"), + "to_watch_category": entry.get("category"), + "source": "home_widget", + } + + +def recover_files(files: Mapping[str, bytes]) -> RecoveryResult: + """Recover from any mix of DioCache, archive, and prefs inputs.""" + expanded: dict[str, bytes] = {} + for name, data in files.items(): + expanded.update(_expand(name, data)) + + library: dict[tuple[str, str], dict[str, Any]] = {} + episodes: list[dict[str, Any]] = [] + movie_history: list[dict[str, Any]] = [] + cache_index: list[dict[str, Any]] = [] + raw_cache: dict[str, str] = {} + + for data in expanded.values(): + kind = _classify(data) + if kind == "sqlite": + try: + dio = recover(data) + except sqlite3.Error: + # A SQLite file that is not a DioCache (no cache_dio table), + # e.g. the app's analytics or image-cache databases. + continue + for row in dio.library: + tagged = {**row, "source": "dio_cache"} + library[(str(tagged["entity_type"]), str(tagged["name"]).casefold())] = tagged + episodes.extend({**ep, "source": "dio_cache"} for ep in dio.episodes) + movie_history.extend(dio.movie_history) + cache_index.extend(dio.cache_index) + raw_cache.update(dio.raw_cache) + elif kind == "flutter_prefs": + prefs = parse_flutter_prefs(data.decode("utf-8", "replace")) + for bucket, entity in (("shows", "show"), ("movies", "movie")): + for category, names in getattr(prefs, bucket).items(): + for name in names: + key = (entity, name.casefold()) + if key in library: + _merge_category(library[key], category) + else: + library[key] = { + "name": name, + "entity_type": entity, + "progress_status": category, + "source": "flutter_prefs", + } + elif kind == "home_widget": + episodes.extend( + _up_next_episode(entry) + for entry in parse_home_widget(data.decode("utf-8", "replace")) + ) + + sorted_library = sorted( + library.values(), + key=lambda item: (str(item["entity_type"]), str(item["name"]).casefold()), + ) + return RecoveryResult( + library=sorted_library, + movie_history=movie_history, + episodes=episodes, + cache_index=cache_index, + raw_cache=raw_cache, + ) diff --git a/packages/tvtime-core/tests/_helpers.py b/packages/tvtime-core/tests/_helpers.py new file mode 100644 index 0000000..57833b5 --- /dev/null +++ b/packages/tvtime-core/tests/_helpers.py @@ -0,0 +1,61 @@ +"""Shared builders for synthetic prefs fixtures used by the test suite.""" + +from __future__ import annotations + +import base64 +import html +import json +import struct +from typing import Any + +from tvtime_core.prefs import BRAZE_CATEGORIES + +# Braze stores each list as base64(prefix + Java-serialized ArrayList). +BRAZE_PREFIX = b"This is the prefix for a list." +_CATEGORY_TO_KEY = {category: key for key, (_bucket, category) in BRAZE_CATEGORIES.items()} + + +def java_serialize_string_list(items: list[str]) -> bytes: + """Encode a list of strings exactly as Java serializes ArrayList.""" + out = bytearray() + out += b"\xac\xed\x00\x05" # STREAM_MAGIC + version + out += b"\x73\x72" # TC_OBJECT, TC_CLASSDESC + name = b"java.util.ArrayList" + out += struct.pack(">H", len(name)) + name + out += b"\x78\x81\xd2\x1d\x99\xc7\x61\x9d" # serialVersionUID + out += b"\x03" # SC_WRITE_METHOD | SC_SERIALIZABLE + out += struct.pack(">H", 1) # one field + out += b"\x49" # 'I' (int) + fname = b"size" + out += struct.pack(">H", len(fname)) + fname + out += b"\x78\x70" # TC_ENDBLOCKDATA, TC_NULL (no superclass) + out += struct.pack(">i", len(items)) # size field value + out += b"\x77\x04" + struct.pack(">i", len(items)) # TC_BLOCKDATA capacity + for item in items: + encoded = item.encode("utf-8") + out += b"\x74" + struct.pack(">H", len(encoded)) + encoded # TC_STRING + out += b"\x78" # TC_ENDBLOCKDATA + return bytes(out) + + +def braze_value(items: list[str]) -> str: + return base64.b64encode(BRAZE_PREFIX + java_serialize_string_list(items)).decode() + + +def make_flutter_prefs_xml( + shows: dict[str, list[str]] | None = None, + movies: dict[str, list[str]] | None = None, +) -> str: + """Build a FlutterSharedPreferences.xml from category -> names mappings.""" + entries = [] + for category, names in {**(shows or {}), **(movies or {})}.items(): + key = _CATEGORY_TO_KEY[category] + entries.append(f' {braze_value(names)}') + body = "\n".join(entries) + return f"\n\n{body}\n" + + +def make_home_widget_xml(watchlist: list[dict[str, Any]]) -> str: + """Build a HomeWidgetPreferences.xml with the given showsWatchList.""" + encoded = html.escape(json.dumps(watchlist)) + return f'{encoded}' diff --git a/packages/tvtime-core/tests/test_prefs.py b/packages/tvtime-core/tests/test_prefs.py new file mode 100644 index 0000000..d23a3e6 --- /dev/null +++ b/packages/tvtime-core/tests/test_prefs.py @@ -0,0 +1,77 @@ +"""Tests for the Flutter shared-preferences recovery parsers.""" + +from __future__ import annotations + +import base64 + +from _helpers import braze_value, make_flutter_prefs_xml, make_home_widget_xml + +from tvtime_core.prefs import ( + decode_java_string_list, + parse_flutter_prefs, + parse_home_widget, +) + + +def test_decode_round_trips_names() -> None: + names = ["The Bear", "House of the Dragon", "Validé"] + assert decode_java_string_list(braze_value(names)) == names + + +def test_decode_empty_list() -> None: + assert decode_java_string_list(braze_value([])) == [] + + +def test_decode_rejects_non_base64() -> None: + assert decode_java_string_list("clearly not base64 @@@") == [] + + +def test_decode_rejects_missing_java_header() -> None: + assert decode_java_string_list(base64.b64encode(b"no java stream here").decode()) == [] + + +def test_parse_flutter_prefs_categorizes_shows_and_movies() -> None: + xml = make_flutter_prefs_xml( + shows={"continue_watching": ["The Bear", "Soda"], "not_started": ["Gen V"]}, + movies={"movies_to_track": ["Dune: Part Two", "Oppenheimer"]}, + ) + prefs = parse_flutter_prefs(xml) + assert prefs.shows["continue_watching"] == ["The Bear", "Soda"] + assert prefs.shows["not_started"] == ["Gen V"] + assert prefs.movies["movies_to_track"] == ["Dune: Part Two", "Oppenheimer"] + + +def test_parse_flutter_prefs_never_exposes_profile() -> None: + # The user profile must not be parsed or surfaced by this route. + assert set(parse_flutter_prefs("").__dataclass_fields__) == {"shows", "movies"} + + +def test_parse_home_widget_up_next() -> None: + watchlist = [ + { + "number": 1, + "name": "Soda", + "network": "Hulu", + "to_watch_category": "continue_watching", + "season": {"number": 5}, + "show": {"name": "The Bear"}, + }, + { + "number": 1, + "name": "June", + "season": {"number": 2}, + "show": {"name": "The Handmaid's Tale"}, + "to_watch_category": "continue_watching", + }, + ] + up_next = parse_home_widget(make_home_widget_xml(watchlist)) + assert len(up_next) == 2 + assert up_next[0]["show"] == "The Bear" + assert up_next[0]["season"] == 5 + assert up_next[0]["episode"] == 1 + assert up_next[0]["episode_name"] == "Soda" + assert up_next[0]["network"] == "Hulu" + + +def test_parse_home_widget_missing_key_is_empty() -> None: + assert parse_home_widget("") == [] diff --git a/packages/tvtime-core/tests/test_sources.py b/packages/tvtime-core/tests/test_sources.py new file mode 100644 index 0000000..5f1fc61 --- /dev/null +++ b/packages/tvtime-core/tests/test_sources.py @@ -0,0 +1,132 @@ +"""Tests for the unified multi-source recovery entry point.""" + +from __future__ import annotations + +import io +import sqlite3 +import tarfile +import zipfile +from pathlib import Path + +from _helpers import make_flutter_prefs_xml, make_home_widget_xml + +from tvtime_core.sources import recover_files + + +def _tar(members: dict[str, bytes]) -> bytes: + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as tar: + for name, data in members.items(): + info = tarfile.TarInfo(name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buffer.getvalue() + + +def _zip(members: dict[str, bytes]) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for name, data in members.items(): + archive.writestr(name, data) + return buffer.getvalue() + + +def test_prefs_only_builds_library() -> None: + xml = make_flutter_prefs_xml( + shows={"continue_watching": ["The Bear"], "not_started": ["Gen V", "UFOs"]}, + movies={"movies_to_track": ["Dune: Part Two", "Oppenheimer"]}, + ) + result = recover_files({"FlutterSharedPreferences.xml": xml.encode()}) + + names = {row["name"] for row in result.library} + assert {"The Bear", "Gen V", "UFOs", "Dune: Part Two", "Oppenheimer"} <= names + bear = next(row for row in result.library if row["name"] == "The Bear") + assert bear["entity_type"] == "show" + assert bear["source"] == "flutter_prefs" + assert bear["progress_status"] == "continue_watching" + dune = next(row for row in result.library if row["name"] == "Dune: Part Two") + assert dune["entity_type"] == "movie" + + +def test_home_widget_becomes_up_next_episodes() -> None: + watchlist = [ + { + "number": 1, + "name": "Soda", + "season": {"number": 5}, + "show": {"name": "The Bear"}, + "network": "Hulu", + "to_watch_category": "continue_watching", + } + ] + result = recover_files({"HomeWidgetPreferences.xml": make_home_widget_xml(watchlist).encode()}) + + assert len(result.episodes) == 1 + episode = result.episodes[0] + assert episode["show_name"] == "The Bear" + assert episode["season_number"] == 5 + assert episode["episode_number"] == 1 + assert episode["source"] == "home_widget" + + +def test_merges_dio_and_prefs_with_dedup(diocache_bytes: bytes) -> None: + # DioCache holds "Breaking Bad" (show) and "Inception" (movie). + xml = make_flutter_prefs_xml(shows={"not_started": ["Breaking Bad", "Gen V"]}) + result = recover_files( + {"DioCache.db": diocache_bytes, "FlutterSharedPreferences.xml": xml.encode()} + ) + + breaking_bad = [row for row in result.library if row["name"] == "Breaking Bad"] + assert len(breaking_bad) == 1 # deduped across sources + assert breaking_bad[0]["source"] == "dio_cache" # richer DioCache row wins + assert "not_started" in breaking_bad[0]["progress_status"] # prefs category merged in + assert any( + row["name"] == "Gen V" and row["source"] == "flutter_prefs" for row in result.library + ) + + +def test_recovers_from_tar_archive(diocache_bytes: bytes) -> None: + xml = make_flutter_prefs_xml(shows={"continue_watching": ["The Bear"]}) + archive = _tar( + { + "app/db/DioCache.db": diocache_bytes, + "app/shared_prefs/FlutterSharedPreferences.xml": xml.encode(), + "app/files/poster.jpg": b"\xff\xd8\xff not an image but ignored", + } + ) + result = recover_files({"tvtime-private.tar": archive}) + + names = {row["name"] for row in result.library} + assert "Breaking Bad" in names # from DioCache inside the tar + assert "The Bear" in names # from the prefs XML inside the tar + assert "tvtime_library.csv" in result.to_csv_map() + + +def test_recovers_from_zip_archive() -> None: + xml = make_flutter_prefs_xml(movies={"movies_to_track": ["Tenet"]}) + archive = _zip({"shared_prefs/FlutterSharedPreferences.xml": xml.encode()}) + result = recover_files({"backup.zip": archive}) + assert any(row["name"] == "Tenet" for row in result.library) + + +def test_ignores_sqlite_without_cache_dio(tmp_path: Path) -> None: + # The tvtime-private tar carries other SQLite DBs (analytics, image cache) + # that have no cache_dio table; they must be skipped, not crash. + other = tmp_path / "other.db" + connection = sqlite3.connect(other) + connection.execute("CREATE TABLE unrelated (x TEXT)") + connection.commit() + connection.close() + + result = recover_files({"other.db": other.read_bytes()}) + assert result.library == [] + + +def test_summary_reports_sources(diocache_bytes: bytes) -> None: + xml = make_flutter_prefs_xml(shows={"not_started": ["Gen V"]}) + result = recover_files( + {"DioCache.db": diocache_bytes, "FlutterSharedPreferences.xml": xml.encode()} + ) + by_source = result.summary()["library_by_source"] + assert by_source["dio_cache"] >= 1 + assert by_source["flutter_prefs"] >= 1 diff --git a/scripts/build-web.sh b/scripts/build-web.sh index 7df5128..e75d17d 100755 --- a/scripts/build-web.sh +++ b/scripts/build-web.sh @@ -17,5 +17,9 @@ if [ -z "$WHEEL" ]; then exit 1 fi +# uv drops a ".gitignore" (containing "*") in the output dir; remove it so +# git-based Pages deploys (which respect .gitignore) still publish the wheel. +rm -f "$DIST/.gitignore" + printf '{\n "wheel": "%s"\n}\n' "$WHEEL" >"$DIST/manifest.json" echo "Built web/dist/$WHEEL" diff --git a/uv.lock b/uv.lock index 805fe54..acca712 100644 --- a/uv.lock +++ b/uv.lock @@ -375,7 +375,7 @@ wheels = [ [[package]] name = "tvtime-cli" -version = "0.1.0" +version = "0.2.0" source = { editable = "packages/tvtime-cli" } dependencies = [ { name = "tvtime-core" }, @@ -386,7 +386,7 @@ requires-dist = [{ name = "tvtime-core", editable = "packages/tvtime-core" }] [[package]] name = "tvtime-core" -version = "0.1.0" +version = "0.2.0" source = { editable = "packages/tvtime-core" } [[package]] diff --git a/web/app.js b/web/app.js index cde694c..b7eb76f 100644 --- a/web/app.js +++ b/web/app.js @@ -10,28 +10,29 @@ const PYTHON_DRIVER = ` import json from pathlib import Path -from tvtime_core import recover +from tvtime_core import recover_files _LAST_RESULT = None _PREVIEW_COLUMNS = [ "name", "entity_type", - "watched", + "source", "progress_status", "seen_episodes", "aired_episodes", ] -def recover_payload(path): +def recover_payload(dir_path): global _LAST_RESULT - _LAST_RESULT = recover(Path(path).read_bytes()) + files = {p.name: p.read_bytes() for p in sorted(Path(dir_path).iterdir()) if p.is_file()} + _LAST_RESULT = recover_files(files) # Write the ZIP to the virtual FS so JS can read it back as a Uint8Array, # avoiding any ambiguity in how bytes cross the Python/JS boundary. Path("/tmp/tvtime-export.zip").write_bytes(_LAST_RESULT.to_zip_bytes()) preview = [ {column: row.get(column, "") for column in _PREVIEW_COLUMNS} - for row in _LAST_RESULT.library[:200] + for row in _LAST_RESULT.library ] return json.dumps( { @@ -53,10 +54,15 @@ const elements = { downloadCsvs: document.getElementById("download-csvs"), previewTable: document.getElementById("preview-table"), previewNote: document.getElementById("preview-note"), + previewFilter: document.getElementById("preview-filter"), }; +// Current preview data and the active entity-type filter ("all" | "movie" | "show"). +const previewState = { columns: [], rows: [], filter: "all" }; + let pyodidePromise = null; let currentZipUrl = null; +let runCounter = 0; function setStatus(message, kind = "") { elements.status.textContent = message; @@ -100,19 +106,30 @@ function loadScript(src) { }); } -async function handleFile(file) { - if (!file) { +async function handleFiles(fileList) { + const files = Array.from(fileList || []); + if (!files.length) { return; } elements.results.hidden = true; try { const pyodide = await getPyodide(); - setStatus(`Reading ${file.name}…`, "busy"); - const bytes = new Uint8Array(await file.arrayBuffer()); - pyodide.FS.writeFile("/tmp/DioCache.db", bytes); + setStatus(files.length === 1 ? `Reading ${files[0].name}…` : `Reading ${files.length} files…`, "busy"); + + const dir = `/tmp/uploads_${++runCounter}`; + pyodide.FS.mkdir(dir); + await Promise.all( + files.map(async (file, index) => { + // Prefix with the index so files with the same basename never collide; + // the extension is preserved so archives are still detected by name. + const safe = file.name.split(/[\\/]/).pop().replace(/[^A-Za-z0-9._-]/g, "_"); + const bytes = new Uint8Array(await file.arrayBuffer()); + pyodide.FS.writeFile(`${dir}/${index}_${safe}`, bytes); + }), + ); setStatus("Recovering your data…", "busy"); - const payloadJson = pyodide.runPython('recover_payload("/tmp/DioCache.db")'); + const payloadJson = pyodide.runPython(`recover_payload(${JSON.stringify(dir)})`); const payload = JSON.parse(payloadJson); const zipBytes = pyodide.FS.readFile("/tmp/tvtime-export.zip"); @@ -147,15 +164,18 @@ function friendlyError(error) { function render(payload, zipBytes) { const summary = payload.summary; - if (!summary.cache_entries) { - setStatus("No cached entries were found in this database. Nothing to export.", "error"); + if (!summary.library && !summary.episodes && !summary.cache_entries) { + setStatus( + "Nothing recoverable was found. Try the whole tvtime-private archive, or your shared_prefs files.", + "error", + ); return; } renderSummary(summary); renderZipButton(zipBytes); renderCsvButtons(payload.csvs); - renderPreview(payload.preview_columns, payload.preview_rows, summary.library); + renderPreview(payload.preview_columns, payload.preview_rows); elements.results.hidden = false; setStatus(`Recovered ${summary.library} library entries.`, ""); @@ -206,23 +226,61 @@ function renderCsvButtons(csvs) { } } -function renderPreview(columns, rows, libraryTotal) { - const shown = rows.length; +function renderPreview(columns, rows) { + previewState.columns = columns; + previewState.rows = rows; + previewState.filter = "all"; + renderFilterControl(); + renderPreviewTable(); +} + +const FILTERS = [ + { key: "all", label: "All", match: () => true }, + { key: "movie", label: "Movies", match: (row) => row.entity_type === "movie" }, + { key: "show", label: "Series", match: (row) => row.entity_type === "show" }, +]; + +function countFor(filter) { + return previewState.rows.filter(filter.match).length; +} + +function renderFilterControl() { + elements.previewFilter.innerHTML = ""; + for (const filter of FILTERS) { + const button = document.createElement("button"); + button.type = "button"; + button.className = "filter-button" + (filter.key === previewState.filter ? " active" : ""); + button.textContent = `${filter.label} (${countFor(filter)})`; + button.setAttribute("aria-pressed", String(filter.key === previewState.filter)); + button.onclick = () => { + previewState.filter = filter.key; + renderFilterControl(); + renderPreviewTable(); + }; + elements.previewFilter.appendChild(button); + } +} + +function renderPreviewTable() { + const { columns, rows, filter } = previewState; + const active = FILTERS.find((entry) => entry.key === filter) || FILTERS[0]; + const visible = rows.filter(active.match); + elements.previewNote.textContent = - shown < libraryTotal - ? `Showing the first ${shown} of ${libraryTotal} entries. Download the CSV for the full list.` - : `Showing all ${shown} entries.`; + filter === "all" + ? `Showing all ${visible.length} entries.` + : `Showing ${visible.length} of ${rows.length} entries.`; const head = - "" + + "" + columns.map((column) => `${escapeHtml(column)}`).join("") + ""; const body = "" + - rows + visible .map( (row) => - "" + + '' + columns.map((column) => `${escapeHtml(formatCell(row[column]))}`).join("") + "", ) @@ -260,7 +318,7 @@ function triggerDownload(url, filename) { // Wire up the file input and drag-and-drop. elements.fileInput.addEventListener("change", (event) => { - handleFile(event.target.files[0]); + handleFiles(event.target.files); }); ["dragenter", "dragover"].forEach((name) => { @@ -278,9 +336,9 @@ elements.fileInput.addEventListener("change", (event) => { }); elements.drop.addEventListener("drop", (event) => { - const file = event.dataTransfer.files[0]; - if (file) { - elements.fileInput.files = event.dataTransfer.files; - handleFile(file); + const dropped = event.dataTransfer.files; + if (dropped && dropped.length) { + elements.fileInput.files = dropped; + handleFiles(dropped); } }); diff --git a/web/index.html b/web/index.html index 4852c4c..41a50d1 100644 --- a/web/index.html +++ b/web/index.html @@ -19,7 +19,8 @@

TV Time Data Recovery

- Turn your TV Time Android cache (DioCache.db) into CSV files. + Turn your TV Time Android data (DioCache.db or the app's + shared_prefs) into CSV files.

@@ -29,11 +30,19 @@

TV Time Data Recovery

uploaded to any server.

-
- +
+
@@ -50,7 +59,8 @@

TV Time Data Recovery

-

Library preview

+

Library

+

diff --git a/web/style.css b/web/style.css index c40630b..bc99cc4 100644 --- a/web/style.css +++ b/web/style.css @@ -224,12 +224,51 @@ code { margin-bottom: 0.25rem; } +.filter { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + margin: 0.5rem 0 0.75rem; +} + +.filter-button { + font: inherit; + font-size: 0.9rem; + padding: 0.35rem 0.85rem; + border-radius: 999px; + border: 1px solid var(--border); + background: var(--card); + color: var(--text); + cursor: pointer; +} + +.filter-button:hover { + border-color: var(--accent); +} + +.filter-button.active { + background: var(--accent); + color: var(--accent-text); + border-color: var(--accent); + font-weight: 600; +} + .preview-note { color: var(--muted); font-size: 0.9rem; margin-top: 0; } +.check-col { + width: 1.5rem; + text-align: center; + padding-right: 0.25rem; +} + +.check-col input { + cursor: pointer; +} + .table-scroll { overflow-x: auto; border: 1px solid var(--border);