diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/renumber_misnumbered_fragments.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/renumber_misnumbered_fragments.py new file mode 100644 index 00000000..132220a5 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/renumber_misnumbered_fragments.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +""" +renumber_misnumbered_fragments.py + +One-time, idempotent repair for ADFA-5171: some chunked Content rows number +their continuation fragments "-2", "-3", ... with no "-1" +at all. WebServer.kt's reassembly loop always probes "-1" first, so +for these rows it finds nothing and stops after the base CHUNK_SIZE-byte +row - silently truncating (ContentTypes.compression = 'none') or failing to +decompress (compression = 'brotli', since the truncated stream is missing +its tail). + +Every writer in this tool (insert_chunked_content, used by populate_db.py, +insert_optimized_media.py, and migrate_content_to_dictionary_brotli.py) has +always numbered fragments starting at "-1" - none of them produced this, so +it predates this pipeline: inherited data, not something today's code +writes. This script repairs existing databases that still carry it. + +Detects every base row whose content is exactly CHUNK_SIZE bytes, that +isn't itself a fragment of some other chain, and whose own fragment chain +(found by LIKE-querying "-%" and sorting on the numeric suffix, not +by constructed path) doesn't start at 1. A chain with a gap in its +suffixes (a real missing chunk, a different failure than this one) is left +alone and reported rather than guessed at. Renumbers matching chains to a +contiguous "-1", "-2", ... run, lowest original suffix first, so each +rename's target path is always the one just vacated by the previous rename +in the same chain (see renumber_chain). Content bytes are never touched - +only paths move - so this is safe regardless of a row's compression. + +A base row that's exactly CHUNK_SIZE with no continuation fragments at all +is left alone: that's a file that is genuinely exactly 1,048,576 bytes, not +a truncated chain, and WebServer.kt already serves it correctly. + +Idempotent: a chain renumbered by this script starts at -1 afterward, so a +second run finds nothing left to fix. + +Usage: + python3 renumber_misnumbered_fragments.py +""" +import re +import sqlite3 +import sys +from pathlib import Path + +from populate_db import CHUNK_SIZE, backup_database + +FRAGMENT_SUFFIX_RE = re.compile(r"^(.*)-(\d+)$") + + +def find_fragment_paths(conn) -> set: + """Every Content.path that is itself a "-" continuation + fragment of some other row in this table - lets the scan below skip a + fragment that would otherwise also look like a candidate base of its + own (fragments are never themselves further chunked).""" + all_paths = {row[0] for row in conn.execute("SELECT path FROM Content")} + fragments = set() + for path in all_paths: + m = FRAGMENT_SUFFIX_RE.match(path) + if m and m.group(1) in all_paths: + fragments.add(path) + return fragments + + +def chain_fragments(conn, base_path: str) -> list: + """Every "-" row present, as (n, path) sorted by n - found + by LIKE query and parsed suffix, not by constructed path, so it doesn't + matter what N the chain actually starts at or whether it has gaps.""" + rows = conn.execute("SELECT path FROM Content WHERE path LIKE ?", (f"{base_path}-%",)).fetchall() + fragments = [] + for (path,) in rows: + m = FRAGMENT_SUFFIX_RE.match(path) + if m and m.group(1) == base_path: + fragments.append((int(m.group(2)), path)) + fragments.sort(key=lambda item: item[0]) + return fragments + + +def is_contiguous_from_one(fragments: list) -> bool: + return [n for n, _path in fragments] == list(range(1, len(fragments) + 1)) + + +def renumber_chain(conn, base_path: str, fragments: list) -> None: + """Renumbers `fragments` (n, path), sorted ascending by n, to a + contiguous "-1", "-2", ... run. Processed lowest-n first: each target + "-" is either untouched already or was the original path + of the fragment just renamed in the previous iteration, so it's always + free by the time this claims it.""" + for i, (_n, path) in enumerate(fragments, start=1): + new_path = f"{base_path}-{i}" + if path != new_path: + conn.execute("UPDATE Content SET path = ? WHERE path = ?", (new_path, path)) + + +def find_chains(conn, fragment_paths: set) -> tuple: + """Returns (misnumbered, gapped): base paths whose content is exactly + CHUNK_SIZE bytes and aren't themselves a fragment of another chain, + split by whether their fragment chain (if any) is a contiguous run not + starting at 1 (misnumbered - safe to repair) or has an actual gap + (gapped - a real missing chunk, left alone and reported instead of + guessed at).""" + candidates = conn.execute("SELECT path FROM Content WHERE length(content) = ?", (CHUNK_SIZE,)).fetchall() + misnumbered = [] + gapped = [] + for (path,) in candidates: + if path in fragment_paths: + continue + fragments = chain_fragments(conn, path) + if not fragments or fragments[0][0] == 1: + continue + suffixes = [n for n, _path in fragments] + if suffixes == list(range(suffixes[0], suffixes[0] + len(suffixes))): + misnumbered.append((path, fragments)) + else: + gapped.append((path, fragments)) + return misnumbered, gapped + + +def repair(conn) -> dict: + fragment_paths = find_fragment_paths(conn) + misnumbered, gapped = find_chains(conn, fragment_paths) + for base_path, fragments in gapped: + suffixes = [n for n, _path in fragments] + print(f"warning: {base_path!r} has a gapped fragment chain (suffixes {suffixes}); left untouched", + file=sys.stderr) + for base_path, fragments in misnumbered: + renumber_chain(conn, base_path, fragments) + return { + "chains_renumbered": len(misnumbered), + "fragments_moved": sum(len(f) for _, f in misnumbered), + "chains_gapped": len(gapped), + } + + +def main() -> None: + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + db_path = Path(sys.argv[1]) + if not db_path.is_file(): + print(f"error: {db_path} does not exist", file=sys.stderr) + sys.exit(1) + + print(f"Backing up {db_path}...", file=sys.stderr) + backup_path = backup_database(db_path) + print(f"Backup written to {backup_path}", file=sys.stderr) + + conn = sqlite3.connect(db_path) + try: + conn.execute("BEGIN") + stats = repair(conn) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + print("Vacuuming database to reclaim freed space...", file=sys.stderr) + vacuum_conn = sqlite3.connect(db_path) + try: + vacuum_conn.execute("VACUUM") + finally: + vacuum_conn.close() + + print( + f"Renumbered {stats['chains_renumbered']} chain(s), moved {stats['fragments_moved']} fragment row(s). " + f"{stats['chains_gapped']} chain(s) had a real gap and were left untouched." + ) + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_renumber_misnumbered_fragments.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_renumber_misnumbered_fragments.py new file mode 100644 index 00000000..bc742c32 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_renumber_misnumbered_fragments.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Tests for renumber_misnumbered_fragments.py (ADFA-5171). + +Run directly: python3 test_renumber_misnumbered_fragments.py +""" +import sqlite3 +import tempfile +import unittest +from pathlib import Path + +from populate_db import CHUNK_SIZE +from renumber_misnumbered_fragments import repair + +SCHEMA_SQL = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + languageID INTEGER NOT NULL, + content BLOB NOT NULL, + contentTypeID INTEGER NOT NULL, + templateId INTEGER, + UNIQUE(path) +); +""" + + +def chunk_bytes(n: int, fill: bytes) -> bytes: + return (fill * (n // len(fill) + 1))[:n] + + +class RenumberMisnumberedFragmentsTest(unittest.TestCase): + def setUp(self): + fd, path = tempfile.mkstemp(suffix=".db") + Path(path).unlink(missing_ok=True) + self.db_path = Path(path) + self.conn = sqlite3.connect(self.db_path) + self.conn.executescript(SCHEMA_SQL) + self.conn.execute("INSERT INTO Languages (value) VALUES ('en-US')") + self.conn.execute("INSERT INTO ContentTypes (value, compression) VALUES ('image/gif', 'none')") + self.conn.commit() + + def tearDown(self): + self.conn.close() + self.db_path.unlink(missing_ok=True) + + def insert(self, path: str, content: bytes): + self.conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID) VALUES (?, 1, ?, 1)", + (path, content), + ) + + def all_paths(self) -> set: + return {row[0] for row in self.conn.execute("SELECT path FROM Content")} + + def content_at(self, path: str) -> bytes: + return self.conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone()[0] + + def test_renumbers_chain_starting_at_minus_2(self): + base = "a/devsite/media/size-range.gif" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", chunk_bytes(CHUNK_SIZE, b"B")) + self.insert(f"{base}-3", chunk_bytes(CHUNK_SIZE, b"C")) + self.insert(f"{base}-4", chunk_bytes(CHUNK_SIZE, b"D")) + self.insert(f"{base}-5", b"E" * 100) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 1) + self.assertEqual(stats["fragments_moved"], 4) + self.assertEqual(stats["chains_gapped"], 0) + self.assertEqual( + self.all_paths(), + {base, f"{base}-1", f"{base}-2", f"{base}-3", f"{base}-4"}, + ) + self.assertEqual(self.content_at(f"{base}-1"), chunk_bytes(CHUNK_SIZE, b"B")) + self.assertEqual(self.content_at(f"{base}-2"), chunk_bytes(CHUNK_SIZE, b"C")) + self.assertEqual(self.content_at(f"{base}-3"), chunk_bytes(CHUNK_SIZE, b"D")) + self.assertEqual(self.content_at(f"{base}-4"), b"E" * 100) + + def test_single_orphaned_continuation(self): + base = "j/html/api/index-all.html" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", b"tail" * 10) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 1) + self.assertEqual(stats["fragments_moved"], 1) + self.assertEqual(self.all_paths(), {base, f"{base}-1"}) + self.assertEqual(self.content_at(f"{base}-1"), b"tail" * 10) + + def test_correctly_numbered_chain_untouched(self): + base = "k/html/already-fine.html" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-1", chunk_bytes(CHUNK_SIZE, b"B")) + self.insert(f"{base}-2", b"tail") + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["fragments_moved"], 0) + self.assertEqual(self.all_paths(), {base, f"{base}-1", f"{base}-2"}) + + def test_idempotent_second_run(self): + base = "a/devsite/media/size-range.gif" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", b"tail") + self.conn.commit() + + repair(self.conn) + self.conn.commit() + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["fragments_moved"], 0) + + def test_exact_size_file_with_no_continuation_left_alone(self): + path = "k/html/exactly-one-mb.bin" + self.insert(path, chunk_bytes(CHUNK_SIZE, b"A")) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["chains_gapped"], 0) + self.assertEqual(self.all_paths(), {path}) + + def test_chain_with_real_gap_reported_and_left_untouched(self): + base = "k/html/actually-missing-a-chunk.html" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", chunk_bytes(CHUNK_SIZE, b"B")) + self.insert(f"{base}-4", b"tail") # -3 is genuinely missing + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["chains_gapped"], 1) + self.assertEqual(self.all_paths(), {base, f"{base}-2", f"{base}-4"}) + + +if __name__ == "__main__": + unittest.main()