ADFA-5171: Repair chunked Content rows misnumbered from -2

ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/renumber_misnumbered_fragments.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""
+renumber_misnumbered_fragments.py
+
+One-time, idempotent repair for ADFA-5171: some chunked Content rows number
+their continuation fragments "<path>-2", "<path>-3", ... with no "<path>-1"
+at all. WebServer.kt's reassembly loop always probes "<path>-1" first, so
+for these rows it finds nothing and stops after the base CHUNK_SIZE-byte
+row - silently truncating (ContentTypes.compression = 'none') or failing to
+decompress (compression = 'brotli', since the truncated stream is missing
+its tail).
+
+Every writer in this tool (insert_chunked_content, used by populate_db.py,
+insert_optimized_media.py, and migrate_content_to_dictionary_brotli.py) has
+always numbered fragments starting at "-1" - none of them produced this, so
+it predates this pipeline: inherited data, not something today's code
+writes. This script repairs existing databases that still carry it.
+
+Detects every base row whose content is exactly CHUNK_SIZE bytes, that
+isn't itself a fragment of some other chain, and whose own fragment chain
+(found by LIKE-querying "<path>-%" and sorting on the numeric suffix, not
+by constructed path) doesn't start at 1. A chain with a gap in its
+suffixes (a real missing chunk, a different failure than this one) is left
+alone and reported rather than guessed at. Renumbers matching chains to a
+contiguous "-1", "-2", ... run, lowest original suffix first, so each
+rename's target path is always the one just vacated by the previous rename
+in the same chain (see renumber_chain). Content bytes are never touched -
+only paths move - so this is safe regardless of a row's compression.
+
+A base row that's exactly CHUNK_SIZE with no continuation fragments at all
+is left alone: that's a file that is genuinely exactly 1,048,576 bytes, not
+a truncated chain, and WebServer.kt already serves it correctly.
+
+Idempotent: a chain renumbered by this script starts at -1 afterward, so a
+second run finds nothing left to fix.
+
+Usage:
+    python3 renumber_misnumbered_fragments.py <db_path>
+"""
+import re
+import sqlite3
+import sys
+from pathlib import Path
+
+from populate_db import CHUNK_SIZE, backup_database
+
+FRAGMENT_SUFFIX_RE = re.compile(r"^(.*)-(\d+)$")
+
+
+def find_fragment_paths(conn) -> set:
+    """Every Content.path that is itself a "<base>-<N>" continuation
+    fragment of some other row in this table - lets the scan below skip a
+    fragment that would otherwise also look like a candidate base of its
+    own (fragments are never themselves further chunked)."""
+    all_paths = {row[0] for row in conn.execute("SELECT path FROM Content")}
+    fragments = set()
+    for path in all_paths:
+        m = FRAGMENT_SUFFIX_RE.match(path)
+        if m and m.group(1) in all_paths:
+            fragments.add(path)
+    return fragments
+
+
+def chain_fragments(conn, base_path: str) -> list:
+    """Every "<base_path>-<N>" row present, as (n, path) sorted by n - found
+    by LIKE query and parsed suffix, not by constructed path, so it doesn't
+    matter what N the chain actually starts at or whether it has gaps."""
+    rows = conn.execute("SELECT path FROM Content WHERE path LIKE ?", (f"{base_path}-%",)).fetchall()
+    fragments = []
+    for (path,) in rows:
+        m = FRAGMENT_SUFFIX_RE.match(path)
+        if m and m.group(1) == base_path:
+            fragments.append((int(m.group(2)), path))
+    fragments.sort(key=lambda item: item[0])
+    return fragments
+
+
+def is_contiguous_from_one(fragments: list) -> bool:
+    return [n for n, _path in fragments] == list(range(1, len(fragments) + 1))
+
+
+def renumber_chain(conn, base_path: str, fragments: list) -> None:
+    """Renumbers `fragments` (n, path), sorted ascending by n, to a
+    contiguous "-1", "-2", ... run. Processed lowest-n first: each target
+    "<base_path>-<i>" is either untouched already or was the original path
+    of the fragment just renamed in the previous iteration, so it's always
+    free by the time this claims it."""
+    for i, (_n, path) in enumerate(fragments, start=1):
+        new_path = f"{base_path}-{i}"
+        if path != new_path:
+            conn.execute("UPDATE Content SET path = ? WHERE path = ?", (new_path, path))
+
+
+def find_chains(conn, fragment_paths: set) -> tuple:
+    """Returns (misnumbered, gapped): base paths whose content is exactly
+    CHUNK_SIZE bytes and aren't themselves a fragment of another chain,
+    split by whether their fragment chain (if any) is a contiguous run not
+    starting at 1 (misnumbered - safe to repair) or has an actual gap
+    (gapped - a real missing chunk, left alone and reported instead of
+    guessed at)."""
+    candidates = conn.execute("SELECT path FROM Content WHERE length(content) = ?", (CHUNK_SIZE,)).fetchall()
+    misnumbered = []
+    gapped = []
+    for (path,) in candidates:
+        if path in fragment_paths:
+            continue
+        fragments = chain_fragments(conn, path)
+        if not fragments or fragments[0][0] == 1:
+            continue
+        suffixes = [n for n, _path in fragments]
+        if suffixes == list(range(suffixes[0], suffixes[0] + len(suffixes))):
+            misnumbered.append((path, fragments))
+        else:
+            gapped.append((path, fragments))
+    return misnumbered, gapped
+
+
+def repair(conn) -> dict:
+    fragment_paths = find_fragment_paths(conn)
+    misnumbered, gapped = find_chains(conn, fragment_paths)
+    for base_path, fragments in gapped:
+        suffixes = [n for n, _path in fragments]
+        print(f"warning: {base_path!r} has a gapped fragment chain (suffixes {suffixes}); left untouched",
+              file=sys.stderr)
+    for base_path, fragments in misnumbered:
+        renumber_chain(conn, base_path, fragments)
+    return {
+        "chains_renumbered": len(misnumbered),
+        "fragments_moved": sum(len(f) for _, f in misnumbered),
+        "chains_gapped": len(gapped),
+    }
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} <db_path>", file=sys.stderr)
+        sys.exit(1)
+    db_path = Path(sys.argv[1])
+    if not db_path.is_file():
+        print(f"error: {db_path} does not exist", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"Backing up {db_path}...", file=sys.stderr)
+    backup_path = backup_database(db_path)
+    print(f"Backup written to {backup_path}", file=sys.stderr)
+
+    conn = sqlite3.connect(db_path)
+    try:
+        conn.execute("BEGIN")
+        stats = repair(conn)
+        conn.commit()
+    except Exception:
+        conn.rollback()
+        raise
+    finally:
+        conn.close()
+
+    print("Vacuuming database to reclaim freed space...", file=sys.stderr)
+    vacuum_conn = sqlite3.connect(db_path)
+    try:
+        vacuum_conn.execute("VACUUM")
+    finally:
+        vacuum_conn.close()
+
+    print(
+        f"Renumbered {stats['chains_renumbered']} chain(s), moved {stats['fragments_moved']} fragment row(s). "
+        f"{stats['chains_gapped']} chain(s) had a real gap and were left untouched."
+    )
+
+
+if __name__ == "__main__":
+    main()

ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_renumber_misnumbered_fragments.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Tests for renumber_misnumbered_fragments.py (ADFA-5171).
+
+Run directly: python3 test_renumber_misnumbered_fragments.py
+"""
+import sqlite3
+import tempfile
+import unittest
+from pathlib import Path
+
+from populate_db import CHUNK_SIZE
+from renumber_misnumbered_fragments import repair
+
+SCHEMA_SQL = """
+CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE);
+CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL);
+CREATE TABLE Content (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    path TEXT NOT NULL,
+    languageID INTEGER NOT NULL,
+    content BLOB NOT NULL,
+    contentTypeID INTEGER NOT NULL,
+    templateId INTEGER,
+    UNIQUE(path)
+);
+"""
+
+
+def chunk_bytes(n: int, fill: bytes) -> bytes:
+    return (fill * (n // len(fill) + 1))[:n]
+
+
+class RenumberMisnumberedFragmentsTest(unittest.TestCase):
+    def setUp(self):
+        fd, path = tempfile.mkstemp(suffix=".db")
+        Path(path).unlink(missing_ok=True)
+        self.db_path = Path(path)
+        self.conn = sqlite3.connect(self.db_path)
+        self.conn.executescript(SCHEMA_SQL)
+        self.conn.execute("INSERT INTO Languages (value) VALUES ('en-US')")
+        self.conn.execute("INSERT INTO ContentTypes (value, compression) VALUES ('image/gif', 'none')")
+        self.conn.commit()
+
+    def tearDown(self):
+        self.conn.close()
+        self.db_path.unlink(missing_ok=True)
+
+    def insert(self, path: str, content: bytes):
+        self.conn.execute(
+            "INSERT INTO Content (path, languageID, content, contentTypeID) VALUES (?, 1, ?, 1)",
+            (path, content),
+        )
+
+    def all_paths(self) -> set:
+        return {row[0] for row in self.conn.execute("SELECT path FROM Content")}
+
+    def content_at(self, path: str) -> bytes:
+        return self.conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone()[0]
+
+    def test_renumbers_chain_starting_at_minus_2(self):
+        base = "a/devsite/media/size-range.gif"
+        self.insert(base, chunk_bytes(CHUNK_SIZE, b"A"))
+        self.insert(f"{base}-2", chunk_bytes(CHUNK_SIZE, b"B"))
+        self.insert(f"{base}-3", chunk_bytes(CHUNK_SIZE, b"C"))
+        self.insert(f"{base}-4", chunk_bytes(CHUNK_SIZE, b"D"))
+        self.insert(f"{base}-5", b"E" * 100)
+        self.conn.commit()
+
+        stats = repair(self.conn)
+        self.conn.commit()
+
+        self.assertEqual(stats["chains_renumbered"], 1)
+        self.assertEqual(stats["fragments_moved"], 4)
+        self.assertEqual(stats["chains_gapped"], 0)
+        self.assertEqual(
+            self.all_paths(),
+            {base, f"{base}-1", f"{base}-2", f"{base}-3", f"{base}-4"},
+        )
+        self.assertEqual(self.content_at(f"{base}-1"), chunk_bytes(CHUNK_SIZE, b"B"))
+        self.assertEqual(self.content_at(f"{base}-2"), chunk_bytes(CHUNK_SIZE, b"C"))
+        self.assertEqual(self.content_at(f"{base}-3"), chunk_bytes(CHUNK_SIZE, b"D"))
+        self.assertEqual(self.content_at(f"{base}-4"), b"E" * 100)
+
+    def test_single_orphaned_continuation(self):
+        base = "j/html/api/index-all.html"
+        self.insert(base, chunk_bytes(CHUNK_SIZE, b"A"))
+        self.insert(f"{base}-2", b"tail" * 10)
+        self.conn.commit()
+
+        stats = repair(self.conn)
+        self.conn.commit()
+
+        self.assertEqual(stats["chains_renumbered"], 1)
+        self.assertEqual(stats["fragments_moved"], 1)
+        self.assertEqual(self.all_paths(), {base, f"{base}-1"})
+        self.assertEqual(self.content_at(f"{base}-1"), b"tail" * 10)
+
+    def test_correctly_numbered_chain_untouched(self):
+        base = "k/html/already-fine.html"
+        self.insert(base, chunk_bytes(CHUNK_SIZE, b"A"))
+        self.insert(f"{base}-1", chunk_bytes(CHUNK_SIZE, b"B"))
+        self.insert(f"{base}-2", b"tail")
+        self.conn.commit()
+
+        stats = repair(self.conn)
+        self.conn.commit()
+
+        self.assertEqual(stats["chains_renumbered"], 0)
+        self.assertEqual(stats["fragments_moved"], 0)
+        self.assertEqual(self.all_paths(), {base, f"{base}-1", f"{base}-2"})
+
+    def test_idempotent_second_run(self):
+        base = "a/devsite/media/size-range.gif"
+        self.insert(base, chunk_bytes(CHUNK_SIZE, b"A"))
+        self.insert(f"{base}-2", b"tail")
+        self.conn.commit()
+
+        repair(self.conn)
+        self.conn.commit()
+        stats = repair(self.conn)
+        self.conn.commit()
+
+        self.assertEqual(stats["chains_renumbered"], 0)
+        self.assertEqual(stats["fragments_moved"], 0)
+
+    def test_exact_size_file_with_no_continuation_left_alone(self):
+        path = "k/html/exactly-one-mb.bin"
+        self.insert(path, chunk_bytes(CHUNK_SIZE, b"A"))
+        self.conn.commit()
+
+        stats = repair(self.conn)
+        self.conn.commit()
+
+        self.assertEqual(stats["chains_renumbered"], 0)
+        self.assertEqual(stats["chains_gapped"], 0)
+        self.assertEqual(self.all_paths(), {path})
+
+    def test_chain_with_real_gap_reported_and_left_untouched(self):
+        base = "k/html/actually-missing-a-chunk.html"
+        self.insert(base, chunk_bytes(CHUNK_SIZE, b"A"))
+        self.insert(f"{base}-2", chunk_bytes(CHUNK_SIZE, b"B"))
+        self.insert(f"{base}-4", b"tail")  # -3 is genuinely missing
+        self.conn.commit()
+
+        stats = repair(self.conn)
+        self.conn.commit()
+
+        self.assertEqual(stats["chains_renumbered"], 0)
+        self.assertEqual(stats["chains_gapped"], 1)
+        self.assertEqual(self.all_paths(), {base, f"{base}-2", f"{base}-4"})
+
+
+if __name__ == "__main__":
+    unittest.main()