diff --git a/.editorconfig b/.editorconfig index d52cabe..2da1302 100644 --- a/.editorconfig +++ b/.editorconfig @@ -63,7 +63,7 @@ end_of_line = unset insert_final_newline = false trim_trailing_whitespace = false -# Caddy map files are tab-separated redirect tables generated by `checks/build-redirects.py`. +# Caddy map files are space-separated redirect tables generated by `capture/build-redirects.py`. # Trailing whitespace is significant to the parse, and the generator owns the formatting. [deploy/maps/*.map] trim_trailing_whitespace = false diff --git a/.gitattributes b/.gitattributes index e740a2f..190c347 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,17 +15,35 @@ # A CRLF shebang breaks execution, so scripts stay LF regardless of the default. *.sh text eol=lf -# The URL-parity and redirect-map generators are shebang-executable and run by path in CI. -# They are pinned individually rather than by a blanket `*.py` rule. -checks/build-redirects.py text eol=lf +# Shebang-executable files carrying no extension, which every rule above matches by +# extension and therefore misses. `ops/vps-backup-pull` is run by systemd on the backup +# host. A named line per file, as with the Python rules below, and `check-eol-pins.py` +# fails if a tracked shebang file ever lands without one. +ops/vps-backup-pull text eol=lf + +# Shebang-executable Python, where a CRLF on line one is a broken interpreter line rather +# than a cosmetic difference. `capture/` is pinned whole, because everything there is a +# script; under `checks/` the two executables are named, because that directory also holds +# lists and fixtures. Neither is a blanket `*.py` rule, so a future non-executable module +# elsewhere is not swept in by accident, and a new executable needs a line here. +capture/*.py text eol=lf checks/check-url-parity.py text eol=lf +checks/check-env-docs.py text eol=lf +checks/check-eol-pins.py text eol=lf -# These formats are parsed line by line by a daemon rather than by a shell. -# Caddy and OpenSSH both reject or silently mis-parse a CRLF file. -# The deploy shell is an extensionless shebang script that matches no rule above. +# Caddy config is parsed line by line by a daemon rather than by a shell, and a CRLF file +# is rejected or silently mis-parsed. Both files are named: the bundle's `Caddyfile`, and +# the bootstrap that is installed into the container's config directory and is the only +# Caddy file outside the release. +# +# The restricted `authorized_keys` and the forced-command deploy shell were pinned here +# too, and this repository has never carried either: they live on the server, described in +# OPERATIONS.md "Server Hardening". A pin binds nothing for a file that does not exist, +# and the comment claiming to cover "the extensionless shebang script" is what let the +# real one, `ops/vps-backup-pull`, sit unpinned above. `check-eol-pins.py` now fails on a +# pattern matching no tracked file, so neither can come back silently. deploy/Caddyfile text eol=lf -deploy/blog-deploy-shell text eol=lf -deploy/authorized_keys text eol=lf +deploy/bootstrap.Caddyfile text eol=lf # Caddy map files are tabular data read by `map` directives. # They stay LF for the same reason as the Caddyfile. diff --git a/.github/workflows/deploy-site-task.yml b/.github/workflows/deploy-site-task.yml index e1b8f5d..5964693 100644 --- a/.github/workflows/deploy-site-task.yml +++ b/.github/workflows/deploy-site-task.yml @@ -69,21 +69,47 @@ jobs: steps: # Full history, because a shallow clone silently changes page metadata if git info is on. + # The mtime restore below needs it too: a shallow clone has no commit to date a file from. - name: Checkout code step uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - # The pin lives in the action, so the deploy and validation cannot install different generators. - - name: Install Hugo step - uses: ./.github/actions/install-hugo - - # REQUIRE_BROTLI below makes a missing binary fatal, so this keeps the build from failing. - - name: Install brotli step + # One update for the job, because each one is a network round trip that can fail on its + # own. REQUIRE_BROTLI later makes a missing brotli fatal, so this keeps the build from + # failing, and git-restore-mtime is what the next step runs. + - name: Install build tools step run: | set -Eeuo pipefail sudo apt-get update - sudo apt-get install --yes --no-install-recommends brotli + sudo apt-get install --yes --no-install-recommends brotli git-restore-mtime + + # Git stores no mtimes, so a checkout stamps every file with the moment it was written. + # The deploy uploads with --link-dest against the previous release, and a file only links + # when size and mtime both match, so today nothing links and every release is a full copy. + # Restoring the last-commit time makes static/ match between releases: measured across two + # independent clones, all 1052 files land on identical mtimes, which is the same 1052 Hugo + # reports as static files and the same 1052 that link on a locally built release. + # + # static/ only. The generated pages are written fresh by every build and can never match, + # and walking the whole tree to prove that costs history reads for nothing. + # + # ORDERING: this is deliberately behind the live media check that #64 added. While every + # file arrives as a fresh inode, the upload re-asserts the mode contract on every deploy. + # Once a third of the tree arrives as hard links, a link carries the mode its inode chain + # began with, so a media file that acquires a bad one stays present, correctly named and + # unreadable, through every later release. The live check is what notices that, by + # requesting images and failing on the 403. + # `git restore-mtime`, the subcommand form, because the package installs into git's + # exec-path at /usr/lib/git-core rather than onto PATH, so the bare name does not resolve. + - name: Restore file mtimes step + run: | + set -Eeuo pipefail + git restore-mtime static + + # The pin lives in the action, so the deploy and validation cannot install different generators. + - name: Install Hugo step + uses: ./.github/actions/install-hugo # Derived once and used three times, as the directory name, the stamp, and EXPECT_RELEASE. # Deriving it twice yields ids seconds apart, and the gate then asserts a phantom version. diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index b0506c5..49b2fba 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -53,11 +53,13 @@ jobs: run: docker run --rm --pull=always -v "$PWD":/check --workdir /check mstruebing/editorconfig-checker:latest # The shell clean-compile is shellcheck at default severity plus `shfmt -d`, both reporting nothing. - # The formatter reads .editorconfig, which pins these scripts to tabs. + # The formatter reads .editorconfig, which pins the .sh files to tabs. + # ops/vps-backup-pull carries no extension, so it takes the [*] default of four spaces instead. + # Use `-d` rather than `-w` here and locally: the container writes as root and would take ownership of the tree. - name: Lint shell scripts step run: | set -Eeuo pipefail - scripts=(checks/check-live-urls.sh deploy/make-release.sh) + scripts=(checks/check-live-urls.sh deploy/make-release.sh ops/vps-backup-pull ops/install.sh capture/run-wp2hugo.sh) docker run --rm --pull=always -v "$PWD":/mnt --workdir /mnt \ koalaman/shellcheck:stable "${scripts[@]}" docker run --rm --pull=always -v "$PWD":/mnt --workdir /mnt \ @@ -71,6 +73,18 @@ jobs: done python3 -c 'import yaml,sys; yaml.safe_load(open("hugo.yaml"))' + # Every configuration value is described once, in ENVIRONMENT.md. + # A new value gets added wherever its author is working, and nothing else notices a + # missing row. Runs both directions: undocumented values, and rows describing nothing. + - name: Check environment docs step + run: python3 checks/check-env-docs.py + + # The .gitattributes line-ending pins are hand-maintained, and nothing read them back. + # This is what reads them back: it fails on a tracked shebang file with no LF pin, + # and on a pin naming no tracked file. + - name: Check line-ending pins step + run: python3 checks/check-eol-pins.py + # The pin lives in the action, so validation and the deploy cannot install different generators. - name: Install Hugo step uses: ./.github/actions/install-hugo diff --git a/.gitignore b/.gitignore index 8c5d34f..0678a5b 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,7 @@ hugo_stats.json *.user .claude -# Python byproducts from the check and redirect-map generators under `checks/`. +# Python byproducts from the gates under `checks/` and the provenance tools under `capture/`. __pycache__/ *.py[cod] .venv/ @@ -30,13 +30,16 @@ __pycache__/ # Host-specific values: deploy roots, base URLs, container names, and uids. # Each names one particular machine rather than the project. # The whole directory is ignored so a value added later lands ignored by default. -# `deploy/env.example` is the committed template and sits outside the directory. -# The last pattern is the backstop for one written outside the directory, matching the -# `..env` shape those files are named for rather than a single literal name. -# `deploy/env.example` does not end in `.env`, so it is unaffected. +# `example.env` is the committed template and sits at the repository root. +# The `*.env` pattern is the backstop for a real environment file written outside the +# directory, matching the `..env` shape those files are named for +# rather than a single literal name. It also matches the template, so the template is +# negated on the line after it, anchored so it only exempts the one at the root. Order +# matters: a negation placed before its pattern does nothing. secrets/ **/secrets *.env +!/example.env # The working copies of the host channel described in OPERATIONS.md. # They carry server internals, and the host's own backup is what makes them durable. diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md new file mode 100644 index 0000000..c9bfdd9 --- /dev/null +++ b/ENVIRONMENT.md @@ -0,0 +1,106 @@ +# ENVIRONMENT.md + +Every configuration value this repository reads or writes, described once. [`OPERATIONS.md`](./OPERATIONS.md) is the procedure and this is the reference the procedure points at, so a value is explained here and named elsewhere. + +**A description lives here and nowhere else.** The `.example` files state the format, the scripts state the defaults, and this file states what a value means. [`checks/check-env-docs.py`](./checks/check-env-docs.py) fails if a value is declared or consumed anywhere and has no row below, or has a row and is declared nowhere, so the two stay in step without depending on anyone remembering. + +**Values are grouped by where they live rather than by what reads them**, because where a value lives determines who can change it, what happens when it is wrong, and whether it reaches a public history. + +## The mechanism + +A local run reads one file. `secrets/..env` is sourced with `set -a`, selected by `ENV_FILE`, and defaults to `secrets/local.production.env`. The whole `secrets/` directory is gitignored, so a value naming a machine never reaches the published history, and [`example.env`](./example.env) is the tracked template that documents the shape. + +Two consequences of `set -a` are worth stating because both have surprised someone. Sourcing overwrites a variable the caller exported first, so exporting `DEPLOY_ROOT` by hand does not switch environments and only `ENV_FILE` does. And a named file that does not exist is a hard failure rather than a fall-through, because on a host serving two sites the ambient value is the other site's root. + +CI reads no file. The deploy workflow resolves the same values from the GitHub Environment, which is why the shapes have to match even though the sources do not. + +## Repository environment files + +Held in `secrets/..env`, one file per environment. Template: [`example.env`](./example.env). + +| Value | Names | Notes | +| --- | --- | --- | +| `DEPLOY_ROOT` | where a release is written, and what the container mounts read-only at `/srv/blog` | The first argument to `make-release.sh` wins over it. | +| `HUGO_BASEURL` | the site base URL | Baked into the canonical tag, the feed links, and every absolute permalink. Must be set for anything that is not production, or a mirror serves pages pointing at production and every gate still passes. | +| `CADDY_APPDATA` | the container's persistent state root, deliberately outside `DEPLOY_ROOT` | Holds `config/` with the bootstrap Caddyfile and `data/` with Caddy state. A release writes neither. Nothing reads this value, so it is recorded to keep a rebuild from depending on memory. | +| `CADDY_CONTAINER` | the container serving this environment | A release needs no restart, because Caddy reloads in process. Restarting is the remedy when the watcher dies, which it does silently after one failed load. | +| `EXPECT_SITE_ENV` | the environment that must answer, compared against the `X-Blog-Env` header the bundle stamps | A proxy rule aimed at the wrong container returns a healthy 200 under the right hostname, so the check refuses to start rather than proving nothing. | +| `PANGOLIN_ACCESS_TOKEN_ID` | the resource access token's id, for an environment behind the auth gate | Set both or neither. Leave both unset for a site that is public. | +| `PANGOLIN_ACCESS_TOKEN` | the token itself | Read by `check-live-urls.sh`. Staging keeps its gate on because it serves a byte-identical copy of the public site. | +| `CAPTURE_ROOT` | the provenance capture, holding the WordPress exports, the crawl of the old platform, and the inventories derived from it | Every script under [`capture/`](./capture/) reads beneath it, and all but one write there too. The exception is [`capture/build-redirects.py`](./capture/build-redirects.py), which writes the committed maps under `deploy/maps/` in this repository, and which also accepts the capture as a first argument that wins over this value. Environment-independent, so it belongs in the default file only. | +| `CAPTURE_SOURCE_URL` | the old platform's base URL, the site the crawl and the URL verification ran against | **Not `HUGO_BASEURL`.** The two hold the same string after the cutover and mean different things, so merging them points a verification run at the new site while every check still passes. Environment-independent. | +| `CAPTURE_SOURCE_API` | the old platform's REST API for that site, carrying its numeric site id | Read for the post and page bodies in **rendered** form, which is what expands shortcodes so a media reference is seen the way a reader's browser sees it. Environment-independent. | +| `CAPTURE_AUTHOR_SLUG` | the old platform's author slug, used to backfill the author archive and its pagination | Optional, and an account name rather than a site value, which is why it is a variable at all. Unset, [`capture/classify.py`](./capture/classify.py) skips the backfill and says so, rather than emitting a list that is silently short by the author URLs. Environment-independent. | +| `VPS_SSH_HOST` | the VPS administrative login | Not the deploy account. See "Two credentials" below. Environment-independent. | +| `VPS_TRAEFIK_LOG` | today's live access log on the VPS, still being appended to | Never pulled, because rotation is what makes a file eligible. An analysis covering today reads it over SSH. Nothing sources it. | +| `VPS_TRAEFIK_LOG_ARCHIVE` | the rotated access logs on the VPS, and the source of the off-host copy | Also read by the pull, below. | +| `VPS_COMMS_DIR` | the two agent channel files on the VPS | Nothing sources it, and the transfer commands in `OPERATIONS.md` are spelled out rather than using it. See "The one place indirection is wrong" below. | +| `BACKUP_ARCHIVE_ROOT` | the off-host encrypted archives and the plaintext `hostconfig` tree beside them | Written by the pull, read by a rebuild. | +| `LOG_ARCHIVE_ROOT` | the off-host copy of the rotated logs | Written by the pull, read by the log review. | + +Three more are named in the template but commented out, because CI resolves them from the GitHub Environment and a local run deploys to a path and needs none of them: `DEPLOY_SSH_HOST`, `DEPLOY_SSH_USER`, `DEPLOY_SSH_KNOWN_HOSTS`. They are listed there so the local file and the environment describe the same shape. + +## The backup host + +Held in `/etc/vps-backup-pull.env`, read by `vps-backup-pull` through the unit's `EnvironmentFile`. Template: [`example.env`](./example.env). [`ops/install.sh`](./ops/install.sh) generates it by copying from the repository environment file, which is why the four shared names are spelled identically in both. + +| Value | Names | Notes | +| --- | --- | --- | +| `VPS_SSH_HOST` | where to pull from | Required. No default. | +| `BACKUP_ARCHIVE_ROOT` | where the archives and host config land | Required. No default. | +| `LOG_ARCHIVE_ROOT` | where both log sets land | Required unless `--no-logs`. No default. Mode 700, because query strings are logged in full. | +| `VPS_ARCHIVE_DIR` | the encrypted archives on the VPS | Defaults to the documented layout. | +| `VPS_TRAEFIK_LOG_ARCHIVE` | the rotated edge access logs on the VPS | Defaults to the documented layout. | +| `VPS_BLOG_LOG_DIR` | one-off Caddy container dumps on the VPS, kept from before rotation existed | Defaults to the documented layout. | +| `SSH_OPTS` | the SSH options the transfer uses | `BatchMode` makes an unusable key fail immediately rather than hanging a timed run on a password prompt nobody sees. | + +**The three marked required carry no default on purpose.** An address and a destination belong to one host, and a wrong-but-valid destination is a backup nobody can find, so the pull names what is missing and refuses to run rather than falling back to something plausible. + +**`systemd` parses this file itself rather than passing it to a shell**, so there is no expansion and no command substitution, and a `$` or a backtick is a literal character. It does strip matching quotes, verified rather than assumed, so a value containing spaces is quoted and arrives without them. That matters because [`example.env`](./example.env) is also sourced by a shell for the other destination, where an unquoted value would run everything after the first space as a command. + +## The GitHub Environments + +Held on the `production` and `staging` environments. The deploy workflow reads no file. + +| Value | Kind | Names | +| --- | --- | --- | +| `HUGO_BASEURL` | variable | the base URL, used twice: the site is built with it and `check-live-urls.sh` is pointed at it | +| `DEPLOY_SSH_HOST` | variable | the deploy endpoint | +| `DEPLOY_SSH_USER` | variable | the confined deploy account | +| `DEPLOY_SSH_KNOWN_HOSTS` | variable | the pinned host key. A variable rather than a secret, deliberately, since it is public by nature | +| `DEPLOY_SSH_PRIVATE_KEY` | secret | the deploy key, held behind an `rrsync` forced command | +| `PANGOLIN_ACCESS_TOKEN_ID` | secret | as above, for an environment behind the gate | +| `PANGOLIN_ACCESS_TOKEN` | secret | as above | + +**`HUGO_BASEURL` being read twice is the trap worth knowing.** A wrong value bakes the wrong address into every canonical tag and then runs the full URL contract against that same wrong address, so the deploy verifies itself and passes. + +**A host rebuild regenerates the SSH host keys and the pinned value stops matching**, which fails every deploy closed and blocks the rollback path at the same moment a rebuild makes both matter. Replace `DEPLOY_SSH_KNOWN_HOSTS` on **both** environments before the first deploy after a rebuild. + +Two repository-level secrets are unrelated to deployment and exist for the merge bot: `CODEGEN_APP_CLIENT_ID` and `CODEGEN_APP_PRIVATE_KEY`. + +## Per-invocation knobs + +Set on the command line for one run rather than stored anywhere. + +| Value | Effect | +| --- | --- | +| `ENV_FILE` | which environment file to source. Defaults to `secrets/local.production.env` | +| `REQUIRE_BROTLI=1` | fail rather than shipping gzip-only. CI sets it | +| `NO_LINK_DEST=1` | full copy instead of hard-linking from the previous release | +| `KEEP_RELEASES` | how many releases `make-release.sh` leaves behind | +| `EXPECT_RELEASE` | the release id `check-live-urls.sh` requires the live site to report, which is what makes a rollback verifiable rather than merely exiting zero | + +## Two credentials to the VPS, and why they are separate + +`DEPLOY_SSH_USER` reaches a confined account behind an `rrsync` forced command that can write one release tree and read nothing else. `VPS_SSH_HOST` is the ordinary administrative login used for reading logs, reading the archive directory, and moving the channel files. Reaching for the deploy account to read a log fails in a way that reads like an outage, and reaching for the admin account to deploy grants far more than the deploy needs. + +## The one place indirection is wrong + +The two channel transfers under [`OPERATIONS.md`](./OPERATIONS.md) "The Channel Between the Two Sides" spell out the host and directory rather than using `VPS_SSH_HOST` and `VPS_COMMS_DIR`. The permission allowlist matches the text of a command rather than what it expands to, so substituting the variables turns an allowed transfer into one that prompts, while looking like a tidy-up that changed nothing. The same rule is why neither may be chained behind `cd` or `&&`. + +## Rules + +- **One name per thing.** A value that appears on two sides is spelled identically on both, so neither side needs translating into the other. +- **No value naming a machine reaches git.** Not in a script default, not in a unit, not in a template. The `.example` files carry placeholders, and the real values live in `secrets/` or on the host. +- **A description belongs here and a reference belongs everywhere else.** A `.example` file says what the format is, and this file says what the value means. +- **Nothing sources some of these, and that is recorded rather than hidden.** A value kept only so a rebuild does not depend on memory is still worth holding, but a reader should not have to discover that no code reads it. diff --git a/OPERATIONS.md b/OPERATIONS.md index 9e87c85..22b83b8 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -48,26 +48,19 @@ This site has served the same domain across earlier platforms, so its whole oper ## The Migration Record -**The migration is documented once, as a post on the site, and that post is the artifact to reference.** [`content/posts/2026/08/01/moving-this-blog-from-wordpress-to-hugo.md`](./content/posts/2026/08/01/moving-this-blog-from-wordpress-to-hugo.md) holds how the URL surface was captured, why the contract splits into a render half and a redirect half, why the Blogger permalink map needs more entries than the posts it covers, why the media had to come from the export tar and be hash-verified, and which Hugo taxonomy default moves every archive to a new address without reporting anything. +**The procedure and the facts live in the directory READMEs.** [`capture/README.md`](./capture/README.md) is the authority on how the inputs were captured and what is derived from them, [`checks/README.md`](./checks/README.md) on the URL contract, and [`deploy/README.md`](./deploy/README.md) on how the redirects are expressed. Each sits beside the thing it describes, which is what keeps it true. -**Read it before changing anything under [`checks/`](./checks/) or [`deploy/maps/`](./deploy/maps/).** Both hold values that no code derives and no test explains, and the reasoning behind them is in the post rather than beside them. Cite the post rather than restating it. This file is the procedure and the post is the account of how the procedure came to be, so where the two disagree this file governs what to do while the post explains why the check exists. +**The migration also has an account of itself, as a post on the site.** It is the casual version, what was done and how it went, and it is worth reading before changing anything under [`checks/`](./checks/) or [`deploy/maps/`](./deploy/maps/), because those hold values that no code derives. + +**The direction between them is one-way.** A post may cite a README. A README never cites the post. A doc that sends a reader to published prose for an operational fact has put the fact where it cannot be kept current, and where correcting it means editing something people have already read. **The post is content, so it sits under the URL contract.** Editing it moves nothing. Renaming it or taking it down breaks an address the site serves. A fact in it that proves wrong is corrected in the post rather than footnoted here. ### Rebuilding from the Exports -Everything derived is in this repository. Everything it was derived *from* is in a capture directory outside it, which is where a rebuild starts. **The capture path is `CAPTURE_ROOT` in `secrets/local.production.env`**, recorded alongside the other values that name a machine rather than the project, so it is read from there rather than searched for. The capture is not a git repository, so it has no history to revert to, and it is read-only in normal use. - -| Under the capture | Holds | Recoverable | -| --- | --- | --- | -| `export/raw/` | the WordPress content export, WXR XML | yes, from the WordPress account while it exists | -| `export/media-tar/` | the media export, the only trustworthy copy of the images | yes, from the same place | -| `mirror/` | a crawl of the old platform as it served, including the media it linked from other hosts | no, once the old hosting ends | -| `inventory/` | the URL and media inventories derived from that crawl | no, for the same reason | +Everything derived is in this repository. Everything it was derived *from* is a capture directory outside it, at `CAPTURE_ROOT`, which is where a rebuild starts. The capture is not a git repository, so it has no history to revert to, and it is read-only in normal use. -The two exports are the only inputs a person has to fetch, and `EXPORT-INSTRUCTIONS.md` at the root of the capture records which two menu items produce them and the counts each has to reconcile against. The counts are the point, because a partial export is the common way a migration loses posts without reporting anything. - -[`checks/build-redirects.py`](./checks/build-redirects.py) takes the capture directory as its one argument and rebuilds everything under `deploy/maps/` from it. It selects the export **by content** rather than by filename and fails unless exactly one candidate holds published posts, because the capture also holds a media-only export whose zero posts produce empty maps that are indistinguishable from working ones until the redirects are live. +**[`capture/README.md`](./capture/README.md) holds the procedure**: what is under the capture and which parts of it can be fetched again, the two exports and the counts they must reconcile against, the ordered rebuild, and the results that look like success and are not. ## Local Verification Before a Pull Request @@ -153,7 +146,7 @@ HUGO_BASEURL= deploy/make-release.sh "$(git rev-parse -- checks/check-live-urls.sh ``` -The deploy root and the base URL are the only host-specific values. A local run reads them from an untracked file under `secrets/`, one per environment, copied from [`deploy/env.example`](./deploy/env.example), and CI passes both explicitly. The whole `secrets/` directory is gitignored, so no address, path, or container name belonging to one machine reaches the published history. +The deploy root and the base URL are the only host-specific values. A local run reads them from an untracked file under `secrets/`, one per environment, copied from [`example.env`](./example.env), and CI passes both explicitly. The whole `secrets/` directory is gitignored, so no address, path, or container name belonging to one machine reaches the published history. **Always set `HUGO_BASEURL` for anything that is not production.** The base URL is baked into the canonical tag, the feed links, and every absolute permalink, so a mirror built without it serves pages that all point back at the production address. Nothing downstream catches this, because the pages render at the right paths and the build gate passes. The effective value is printed on every build for that reason. @@ -196,6 +189,45 @@ The script asserts both halves of that rather than assuming them. It fails when **Nothing prunes on the deploy path, and that is what keeps the deploy key's capability small.** A prune racing a deploy could take the rollback target, where a lingering release only costs disk. This is also why the key needs no delete capability, which is the property "Server Hardening" depends on. The count and the timer belong to the host, so this section records what the host declares rather than holding a second copy of it. See "Who Owns What". +## Working With the VPS + +**Every path and hostname on this page is a value in `secrets/`, never a literal to be remembered or asked for.** The convention is the one "Environments" describes and `CAPTURE_ROOT` already follows: a value naming a machine rather than the project lives in the environment file, is sourced with `set -a`, and is read from there rather than searched for. The VPS values are environment-independent, because there is one such host rather than one per environment, so they sit in the default file alongside `CAPTURE_ROOT`. + +```sh +set -a; . secrets/local.production.env; set +a +ssh "$VPS_SSH_HOST" true && echo reachable +``` + +| Value | Names | Side | +| --- | --- | --- | +| `VPS_SSH_HOST` | the administrative login | the VPS | +| `VPS_TRAEFIK_LOG` | today's live access log, still being appended to | the VPS | +| `VPS_TRAEFIK_LOG_ARCHIVE` | the rotated access logs, and the source of the off-host copy | the VPS | +| `VPS_COMMS_DIR` | the two agent channel files | the VPS | +| `LOG_ARCHIVE_ROOT` | the off-host copy of the rotated logs | the backup host | +| `BACKUP_ARCHIVE_ROOT` | the off-host encrypted archives and the plaintext hostconfig tree beside them | the backup host | + +**There are two credentials to this host and picking the wrong one is the first mistake to avoid.** `DEPLOY_SSH_USER`, held per environment and used only by the deploy, reaches a confined account behind an `rrsync` forced command that can write one release tree and read nothing else. `VPS_SSH_HOST` is the ordinary administrative login used for everything on this page. They are deliberately separate credentials with different blast radii, so reaching for the deploy account to read a log fails in a way that reads like an outage, and reaching for the admin account to deploy grants far more than the deploy needs. + +**The off-host copy is made by a script in this repository, [`ops/vps-backup-pull`](./ops/vps-backup-pull), on a `systemd` timer on the backup host.** It copies three things off the VPS into `BACKUP_ARCHIVE_ROOT` and `LOG_ARCHIVE_ROOT`: the encrypted archives, a plaintext copy of the same non-secret host files, and the rotated access logs. What it does, the three behaviors that look like bugs and are not, how to install it, and how to check it ran are in [`ops/README.md`](./ops/README.md). Read the unit and its last run on the backup host rather than trusting a schedule written down anywhere, including here. + +**It is a pull rather than a push, and nothing on the VPS knows it happens.** That direction is the security property rather than an implementation detail: the backup host holds a key the VPS trusts, and the VPS holds no credential reaching any other system, so a compromise of the web server cannot walk into the backups that exist to survive it. + +**Both sides use one set of names, so there is nothing to reconcile.** The pull writes `BACKUP_ARCHIVE_ROOT` and `LOG_ARCHIVE_ROOT` and the log review reads the same two, spelled the same way, and [`ops/install.sh`](./ops/install.sh) generates the pull's `EnvironmentFile` from this repository's `secrets/` file by copying rather than translating. Every value is described once, in [`ENVIRONMENT.md`](./ENVIRONMENT.md), and [`checks/check-env-docs.py`](./checks/check-env-docs.py) fails if one is declared without a description or described without existing. + +```sh +set -a; . secrets/local.production.env; set +a +ls -d "$LOG_ARCHIVE_ROOT" "$BACKUP_ARCHIVE_ROOT" +``` + +**Today's traffic is never in the off-host copy, and that is deliberate.** Rotation is what makes a file eligible to be pulled, so a live log would be copied as a torn prefix and fetched again on the next run. An analysis covering today therefore reads `VPS_TRAEFIK_LOG` over SSH and everything older from `LOG_ARCHIVE_ROOT`, and treats the two as one series joined on `StartUTC` rather than on which file a line came from. + +**The plaintext `hostconfig` tree under `BACKUP_ARCHIVE_ROOT` is the readable copy of the VPS's own configuration**, carrying the same non-secret files the encrypted archives hold. It exists so a rebuild does not depend on the encryption key, which is not on the backup host and must never be put there, because beside the ciphertext it would make the encryption decorative. What that tree covers is whatever the VPS advertises, read from the host rather than duplicated here, so it tracks the host instead of drifting from a list. + +**The channel transfers are the one exception, and they must stay literal.** The permission allowlist in `.claude/settings.local.json` matches the text of a command rather than what it expands to, so substituting `"$VPS_SSH_HOST:$VPS_COMMS_DIR/..."` into those two `rsync` lines turns an allowed command into one that prompts, while looking like a tidy-up that changed nothing. Use the values above everywhere else, and leave the two commands under "The Channel Between the Two Sides" spelled out exactly as they are written there. + +**What this section does not cover, and where it lives instead.** Reading the logs for content is "Log Review"; exchanging rounds with the agent that owns the host is "The Channel Between the Two Sides"; the boundary of which side fixes what is "Who Owns What"; and what a rebuild restores, including the host-key step that blocks both deploy and rollback, is "Backup and Restore". + ## Log Review **Real traffic is the only source that finds what every check here is blind to.** The URL contract proves the URLs someone thought to list and the redirects derived from the export. It cannot know about a URL nobody recorded, because the lists are their own standard: the gates check the built site and the running server against those lists, never against the old platform that served the addresses. An address the crawl missed is therefore missing from every gate that reads them, and a visitor following a sixteen-year-old link is the one reader who tests for it. @@ -223,8 +255,40 @@ A request crosses the proxy before it reaches the site, so no single log answers **A 404 count taken from Caddy alone is therefore a floor, not a total.** A request the edge refused is a reader who found nothing just as surely, and it appears in no Caddy log. Read the edge for what never arrived and Caddy for what arrived and failed, and treat the two as one answer. +**`ServiceName` is what separates those two cases inside the edge log itself**, which is otherwise a distinction this table draws conceptually and leaves you no way to apply. A Traefik line carrying a service name was routed, so the 404 came from the site. A line with the field absent matched no router at all, so the edge answered and the site never saw the request. The second kind is the one Caddy is structurally blind to, and it is rare enough that it reads as noise in a total and is worth listing individually. + +**When `ServiceName` is absent the edge answered, and `entryPointName` with `RequestScheme` say why**, which is the difference between a finding and a fault. The common cause is a cleartext request to the TLS port: the `websecure` router carries `tls` and therefore matches TLS requests only, so a plaintext request to 443 matches nothing and Traefik answers its own 404 with a body of a few dozen bytes. That is correct behavior rather than a gap, and it needs no action. Read the two fields together before treating a routerless 404 as a routing problem, because the shape that does deserve investigation is a routerless 404 arriving over **https**, which means a hostname the proxy serves no route for. + Two properties of the Caddy side are worth knowing before parsing it. Its access log is `format console`, so each line is a timestamp, a level, and a logger name followed by a JSON object rather than being JSON itself, and a parser that assumes one object per line reads nothing. And `trusted_proxies` is what makes `client_ip` the reader rather than the proxy, which is the same setting "Serving" describes as a security boundary. Without it every request in the log appears to come from one internal address, and the inward pass cannot distinguish a reader from a health check. +### Reading a 404 list without being fooled by it + +The outward pass is four filters over the edge log, and each one exists because skipping it produced a wrong answer once. + +**Exclude this repository's own deploy gate first.** `check-live-urls.sh` requests the whole URL contract on every deploy, so an unfiltered day is mostly a recording of our own `curl`. Filter on user agent: on 2026-08-08, 9,285 of 9,996 requests were `curl/8.5.0` and the 711 that remained are the entire real dataset. A count that omits this step is measuring the pipeline rather than the readers, and it will be an order of magnitude too large. + +**A referer does not implicate this site unless it points somewhere else.** The rule worth applying is that a 404 carrying a referer is a broken link and a 404 without one is a typed or probed address, and it fails on scanners, which set `Referer` to the request URL itself. Every one of the 36 referer-bearing site-host 404s on 2026-08-08 was self-referential, so the unrefined rule reported three dozen broken links on a site that had none. Discard the matches before counting, and **normalize the scheme rather than comparing it**, because a scanner reaching an HTTPS site routinely sends an `http://` referer for the same address. Comparing against the request's own scheme therefore matches nothing and leaves every false positive in place: on the 2026-08-08 data the naive form kept all 36 where the normalized form kept none. + +```sh +# Narrow to this site's own 404s, then keep only referers pointing somewhere else. +# No null guard is needed anywhere here: == and + both tolerate a null host. Swap the +# equality for startswith and one becomes mandatory, which is the trap described below. +jq -c 'select(.DownstreamStatus == 404) + | select(.RequestHost == "blog.example.com") + | select((.["request_Referer"] // "") != "") + | select((.["request_Referer"] | sub("^https?://"; "")) != (.RequestHost + .RequestPath))' +``` + +Widen `== 404` to `>= 400` for the whole non-200 sweep the table above describes. The 404 list is the half with an action, which is why it is the default here. + +**Filter the scanner shapes by shape, never by investigating them.** A site that used to run WordPress attracts probes for `.env` and its dozen variants, `wp-config.php`, `.git/config`, `phpinfo.php`, cloud credential files, and framework config paths. They dominate the raw list and none is ever a finding. What is left after the three filters above is small enough to read line by line, which is the point of running them. + +**Then cross-reference what remains against the contract**, because that is the only step with an action. A surviving 404 whose path appears in [`checks/golden-urls.txt`](./checks/golden-urls.txt) or in [`deploy/maps/`](./deploy/maps/) is a redirect that is not working. A surviving 404 shaped like real content and present in neither is the case this whole pass exists to find, and it is added to the golden list with a redirect per that file's maintenance rules. A run where nothing survives is the expected result and should be recorded as one. + +**A missing `Host` header is the third way this data breaks a filter.** `RequestHost` is null on a request that sends none, which router exploits do. String functions reject that where arithmetic tolerates it, so `.RequestHost | startswith(...)` fails with `startswith() requires string inputs` and takes the whole run with it, while `.RequestHost + .RequestPath` quietly yields the path alone. The failure is loud but partial, which is the worst combination, since it aborts part way through a file having already printed real output. Guard the string functions with `// ""`. Equality and concatenation both tolerate a null, so a guard on those is inert and reads as protection that is not there. + +**Two `jq` mistakes each read as a plausible answer rather than as an error.** A hyphenated key parses as subtraction, so `.request_User-Agent` silently is not the field you meant and `.["request_User-Agent"]` is, and the same holds for `Referer`. And `jq 'select(...)'` with no projection pretty-prints each match across many lines, so piping it to `wc -l` counts lines rather than records and overstates by roughly the width of the object. It reported 37 and 1,332 where the true counts were 1 and 36. Project with `@tsv` or pass `-c` before counting anything. + ### Retention Is the Prerequisite, and It Belongs to the Host **On the VPS the reviewable record is Traefik's access log**, at `/var/log/traefik/access.log`, one JSON object per line, one line per request, across every hostname the host serves. `RequestPath` carries the query string, so the legacy `/?p=` traffic is visible as itself. Request headers are dropped except `Referer` and `User-Agent`, which is what keeps the Pangolin resource access token out of a file that is retained and copied, and query strings are logged in full, so treat an extract as sensitive. @@ -239,7 +303,9 @@ Two properties of the Caddy side are worth knowing before parsing it. Its access **The off-host copy of the access log exists, and the schedule that maintains it is younger than the copy.** The pull to the backup host is installed as a `systemd` timer running daily at 09:00 UTC, chosen to sit behind both producers on the VPS rather than beside them, and its first copy was made by hand rather than by the timer. Read the unit and its last run on the backup host rather than trusting this paragraph, for the same reason retention is read from the VPS: a claim about a schedule is only worth what the machine says. -**A rename on the VPS does not propagate to that copy, and nothing reports the divergence.** The pull passes no `--delete` for the logs, deliberately, since an append-only record must never be removed by a transfer. So a file **the VPS** renames, merges, or re-compresses after it has been pulled keeps its old name **on the backup host** forever, alongside the new one, and a count that walks that archive by filename double-counts the overlap. This has already happened once, to two archives whose names were a day ahead of their contents. **Read a date from a line's `StartUTC` rather than from the filename that holds it**, and treat a rename on the VPS as something the channel has to carry, because no transfer will. +**A rename on the VPS does not propagate to that copy, and nothing reports the divergence.** The pull passes no `--delete` for the logs, deliberately, since an append-only record must never be removed by a transfer. So a file **the VPS** renames, merges, or re-compresses after it has been pulled keeps its old name **on the backup host** forever, alongside the new one, and a count that walks that archive by filename double-counts the overlap. This has already happened once, to two archives whose names were a day ahead of their contents. **Read a date from a line's `StartUTC` rather than from the filename that holds it.** The reconciliation itself now travels with the data: the VPS keeps an append-only `RECONCILE.md` **inside the archive directory**, so the pull carries it automatically and a rename does not depend on someone rereading a channel file. It records what a file contained rather than what it was called, and it is counted among the pulled log files. **The VPS keeps a `MANIFEST.txt` in the same directory**, so expect the count to exceed the number of logs by two rather than by one, and expect any further explanatory file the host side adds to raise it again. Read the count as logs-plus-prose rather than as a number with a fixed offset. + +**A journal with one entry is not evidence of one copy.** The pull can be run directly as well as by its timer, and a direct run writes no service record. Directory mtimes on the backup host are the copy times, where the file mtimes are the VPS's, so those are what to read when establishing when something arrived. ## Who Owns What @@ -271,6 +337,8 @@ rsync -a root@:/srv/agent-comms/vps-agent.md comms/vps-agent.md rsync -a --no-o --no-g --chmod=F644 comms/blog-agent.md root@:/srv/agent-comms/blog-agent.md ``` +**Spell both commands out rather than reading the host and directory from `secrets/`**, which is the opposite of the rule "Working With the VPS" sets for every other path, and is deliberate. These two are allowlisted in `.claude/settings.local.json`, and an allow rule matches the text of the command rather than the value it expands to, so replacing the literals with `$VPS_SSH_HOST` and `$VPS_COMMS_DIR` turns an allowed transfer into one that prompts. The same rule is why neither may be chained behind `cd` or `&&`: an allow rule matches a standalone command only. + **The push suppresses owner and group deliberately.** `-a` implies `-o` and `-g`, and the transfer connects as root, so a plain `rsync -a` carries this workstation's numeric uid onto a host that has no such user and leaves the file owned by a number. Four rules, each covering a way the channel has already failed or could: @@ -345,7 +413,7 @@ Ordering is load-bearing, so every redirect lives in a single `route` block. Out | Blogger label archives | `labels.map`, defaulting to the archive index | | Term archives the generator does not build | `terms.map` | -The maps are generated by `checks/build-redirects.py` from the source export, which lives outside this repository. It is a provenance script rather than a CI step, and its outputs are committed. It selects the export by content and refuses to run unless exactly one contains published posts, because the capture holds a full export and a media-only one, and reading the wrong one yields empty maps that are indistinguishable from working ones until the redirects are live. +The maps are generated by [`capture/build-redirects.py`](./capture/build-redirects.py) and the generated files are committed, so a deploy never regenerates them. How it selects its input, and why that selection is the part to get right, is in [`capture/README.md`](./capture/README.md). ## Server Hardening diff --git a/README.md b/README.md index 10cb4dd..b9f406a 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Deployment is a release directory plus a symlink. A build is installed alongside The site has served the same domain since 2008, across three platforms: Blogger until 2012, WordPress until 2026, and Hugo from then on. Converting the posts took an afternoon. Preserving sixteen years of inbound links was the work, and it is why this repository carries a URL contract and gates it rather than trusting the build. -The account of that migration is a post on the site, [Moving This Blog From WordPress to Hugo][migration-post]. It covers what a WordPress export holds and what it leaves out, why the sitemap named barely a tenth of the addresses the site was actually serving, how the Blogger-era permalinks resolve through a lookup table rather than a pattern, why media fetched over HTTP is not the same bytes as the media in the export and only a content hash tells them apart, and which Hugo default moves every taxonomy archive to a new address without reporting anything. +How it was done is in [`capture/README.md`][capture-readme], and the account of it is a post on the site, [Moving This Blog From WordPress to Hugo][migration-post]. It covers what a WordPress export holds and what it leaves out, why the sitemap named barely a tenth of the addresses the site was actually serving, how the Blogger-era permalinks resolve through a lookup table rather than a pattern, why media fetched over HTTP is not the same bytes as the media in the export and only a content hash tells them apart, and which Hugo default moves every taxonomy archive to a new address without reporting anything. ## How a Change Reaches the Site @@ -152,8 +152,11 @@ flowchart LR | [`hugo.yaml`][hugo-config] | site configuration, taxonomy URLs, and the feed name | | [`checks/`][checks] | the URL contract and the gates that enforce it | | [`deploy/`][deploy] | the release script, the web-server config, and the redirect maps | +| [`capture/`][capture] | the migration's provenance tooling, and how the site was derived from the old platform's exports | +| [`ops/`][ops] | the pull that copies the server's backups and access logs off it, and its schedule | +| [`ENVIRONMENT.md`][environment] | every configuration value, described once | -Deploy paths, environment variables, and the server layout are documented in [OPERATIONS.md][operations]. +Every configuration value is described in [ENVIRONMENT.md][environment]. The deploy procedure and the server layout are in [OPERATIONS.md][operations]. ## Questions or Issues @@ -179,7 +182,7 @@ deploy/make-release.sh checks/check-live-urls.sh "$HUGO_BASEURL" ``` -The deploy root and the base URL come from an untracked file per environment under `secrets/`, named `..env`, copied from [deploy/env.example][env-example] and selected with `ENV_FILE`. `secrets/local.production.env` is the one read when `ENV_FILE` is unset. The whole `secrets/` directory is gitignored, so host-specific values stay out of the published history. +The deploy root and the base URL come from an untracked file per environment under `secrets/`, named `..env`, copied from [example.env][env-example] and selected with `ENV_FILE`. `secrets/local.production.env` is the one read when `ENV_FILE` is unset. The whole `secrets/` directory is gitignored, so host-specific values stay out of the published history. ## 3rd Party Tools @@ -212,10 +215,14 @@ Licensed under the [MIT License][license]\ [checks]: ./checks/ [commits-link]: https://github.com/ptr727/Blog/commits [deploy]: ./deploy/ +[capture]: ./capture/ +[capture-readme]: ./capture/README.md +[ops]: ./ops/ +[environment]: ./ENVIRONMENT.md [deploy-readme]: ./deploy/README.md [discussions-link]: https://github.com/ptr727/Blog/discussions [issues-link]: https://github.com/ptr727/Blog/issues -[env-example]: ./deploy/env.example +[env-example]: ./example.env [history]: ./HISTORY.md [hugo-config]: ./hugo.yaml [license]: ./LICENSE diff --git a/TODO.md b/TODO.md index 6795791..e5dc8ec 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,6 @@ # TODO -Running backlog for this repo, kept in a committed file so the work survives across sessions. How the migration was done is a [post on the blog][migration-post] rather than a section here. +Running backlog for this repo, kept in a committed file so the work survives across sessions. How the migration was done is in [`capture/README.md`](./capture/README.md), with a casual account of it as a [post on the blog][migration-post]. ## State @@ -18,8 +18,8 @@ The site is built, gated in CI, and deployed to staging by pipeline. It is not y | Fleet conformance | cataloged in the hub registry, audited, and carrying the current canonical | | Deploy pipeline | `deploy-site.yml` is dispatchable and has deployed staging from CI end to end, through a transport retested against the real host | | VPS staging | live at `blog.vps.insanegenius.net`, behind the auth gate, serving a pipeline release | -| VPS production | **M7a done 2026-08-08.** Serving release `20260808-041050` at `blog.insanegenius.net`, answering `200` unauthenticated, verified 9/9 from the host side with the built `baseURL` read from the deployed bytes. DNS for the public name is still on the old platform | -| Operations | started, and neither half has completed a **scheduled** run. The off-host log pull is installed, armed for 09:00 UTC daily, and has copied once, started by hand, so the timer itself has never fired and 2026-08-09 is its first scheduled run. The periodic log review has not run at all | +| VPS production | **M7a done 2026-08-08.** Serving release `20260808-154717` at `blog.insanegenius.net`, answering `200` unauthenticated, deployed from `main` by pipeline with the 1,245-URL contract verified against the live site. `/robots.txt` answers 200 carrying a `.net` sitemap line, and the gallery fix is live. DNS for the public name is still on the old platform | +| Operations | started, and neither half has completed a **scheduled** run. The off-host log pull is installed, armed for 09:00 UTC daily, and has copied once, started by hand, so the timer itself has never fired and 2026-08-09 is its first scheduled run. The periodic log review has run once by hand, the outward pass only: it read 2026-08-08 traffic and found nothing to add to the URL contract, the inward pass has not run, and neither pass is on the cadence **Recurring operations** sets | ## Blocked on the maintainer @@ -29,18 +29,19 @@ The site is built, gated in CI, and deployed to staging by pipeline. It is not y ## Next, in dependency order - **Prove a rollback through the pipeline.** A forced mid-deploy failure, then a flip back to the previous release, verified by `EXPECT_RELEASE` rather than by the transport exiting zero. The server side has been measured at well under a second by hand; what is unproven is that a **pipeline** run leaves the site serving when its deploy fails part way. -- **Production is deployed, which the VPS agent calls M7a, done 2026-08-08.** `blog.insanegenius.net` serves release `20260808-041050`, answering `200` unauthenticated on a Let's Encrypt certificate issued 2026-08-07. The host side verified it independently, 9/9 unauthenticated with the built `baseURL` read from the deployed bytes rather than from this repo's config, across a 3,095-request gate run with no unexplained 404s. What remains is **M7b, the `.com` cutover**, and the sub-items below are where this repo stands against it, two of them owed and one already answered. The VPS agent's §19, §20, §23 and §24 carry the detail and that file is not in the repository, so pull it first per [`OPERATIONS.md`](./OPERATIONS.md) "The Channel Between the Two Sides": +- **Production is deployed, which the VPS agent calls M7a, done 2026-08-08.** `blog.insanegenius.net` serves release `20260808-154717`, answering `200` unauthenticated on a Let's Encrypt certificate issued 2026-08-07, read from the `X-Blog-Release` header rather than from a pipeline's exit code. The host side verified the first production release independently, 9/9 unauthenticated with the built `baseURL` read from the deployed bytes rather than from this repo's config, across a 3,095-request gate run with no unexplained 404s. What remains is **M7b, the `.com` cutover**, and the sub-items below are where this repo stands against it, two of them owed and one already answered. The VPS agent's §19, §20, §23 and §24 carry the detail and that file is not in the repository, so pull it first per [`OPERATIONS.md`](./OPERATIONS.md) "The Channel Between the Two Sides": - **`HUGO_BASEURL` on the `production` environment is set to `https://blog.insanegenius.net/`**, done 2026-08-07. It held `https://blog.insanegenius.com/`, the live WordPress address, which is what the workflow both builds with and points the live check at, so a deploy would have baked the old platform's address into every canonical tag, feed link and `sitemap.xml` and then run 1,245 requests at the live site to verify it. **Setting it back to `.com` at M7b is the other half and is not done.** - **Production emits `X-Robots-Tag: noindex, nofollow` for the length of the rehearsal**, deliberately, because `.net` serves a public duplicate of a live site and Certificate Transparency publishes the hostname. Where a check asserts `index, follow`, make the expected value a parameter rather than flipping a literal, since it reverts at M7b and a hardcoded literal is one more thing to remember at the wrong moment. - **The two questions in §19.3 are answered.** `HUGO_BASEURL` holds the interim `.net` name, per the item above. Exactly one place hardcodes `blog.insanegenius.com`: `baseURL` on line 1 of `hugo.yaml`, which is the production default every environment overrides through `HUGO_BASEURL`. Nothing under `checks/`, `deploy/`, `layouts/`, or `.github/` carries it. -- **`robots.txt` is decided and built, 2026-08-08, and what remains is that production has not been redeployed since.** The site emits one now, `enableRobotsTXT` is set, and the theme's template derives the `Sitemap:` line from the built `baseURL`, so it names `.net` during the rehearsal and `.com` after the cutover with nothing to remember at M7b. `/robots.txt/` redirects to the real file rather than to the home page, and `check-url-parity.py` gates all of it. The record below is kept because the reasoning is what the next decision about crawl directives will need, and because production still answers 404 until a deploy carries this. - - **The first deploy did not fix the 404, and that is what turned this from a gap into a decision.** The VPS agent raised it in §22.10 and both halves were measured rather than assumed: the site emitted no `robots.txt` at all, because `hugo.yaml` set no `enableRobotsTXT`, so the 404 survived the deploy and `X-Robots-Tag` was the only control, while `sitemap.xml` **was** emitted and became fetchable on the interim name at that same deploy. A crawler got a full sitemap and no robots file. `enableRobotsTXT` is now set, so this describes the release production is still serving rather than the current build. +- **`robots.txt` is decided, built, and deployed, 2026-08-08.** The site emits one, `enableRobotsTXT` is set, and the theme's template derives the `Sitemap:` line from the built `baseURL`, so it names `.net` during the rehearsal and `.com` after the cutover with nothing to remember at M7b. `/robots.txt/` redirects to the real file rather than to the home page, and `check-url-parity.py` gates all of it. Verified from the served bytes on release `20260808-154717`: `/robots.txt` answers 200 advertising `https://blog.insanegenius.net/sitemap.xml`, `/robots.txt/` 301s to it, and `sitemap.xml` carries 312 `.net` URLs and zero `.com`. The record below is kept because the reasoning is what the next decision about crawl directives will need. + - **The first deploy did not fix the 404, and that is what turned this from a gap into a decision.** The VPS agent raised it in §22.10 and both halves were measured rather than assumed: the site emitted no `robots.txt` at all, because `hugo.yaml` set no `enableRobotsTXT`, so the 404 survived the deploy and `X-Robots-Tag` was the only control, while `sitemap.xml` **was** emitted and became fetchable on the interim name at that same deploy. A crawler got a full sitemap and no robots file. `enableRobotsTXT` is now set and a deploy has carried it, so this describes the state up to release `20260808-041050` rather than what is served today. - **At the cutover this stops being a gap and becomes a loss, which is the half neither side had checked.** The live `.com` blog **serves a `robots.txt` today, carrying a `Sitemap:` line**. Because this site emitted none, M7b would not have been a return to a previous state, it would have been a move from having crawl directives to having none on a site that has had them for years, and the sitemap pointer would have gone with them. The VPS agent measured this from the outside in §23.3, will not put a file in this repository's bundle, and has made it a decision that blocks step 1 of the M7b checklist rather than one discovered after it. The minimum that preserves today's behavior is `User-agent: *`, no `Disallow`, and the sitemap line, since every `Disallow` the old platform serves names a WordPress path this site does not have. **That is what was chosen**, out of three options: preserve today's behavior, write what this site actually wants, or keep emitting nothing and accept the loss. The sitemap URL is derived from the built `baseURL` rather than typed, which is what makes the choice survive the cutover without a second edit. - **The log reframes the decision, and it is the `Sitemap:` line that carries it rather than any rule.** Across the interim hostname's first full day, `/robots.txt` was requested nine times and answered 404 every time, five of those from real agents on a hostname with no inbound links. **No crawler fetched `sitemap.xml` or `feed.xml` once**: every request to either came from `curl`, the deploy gate's or the host side's. Crawlers do not guess a sitemap's location, they are told it, and the only thing telling them today is the `robots.txt` the old platform serves, which is the file the cutover deletes. So the question is not whether to have crawl directives, it is whether the sitemap stays advertised at all. Measured on the host side in its §26.4 and recorded here because the decision outlives that channel. - - **`/robots.txt/`, with a trailing slash, now redirects to the real file** rather than to the home page, in the same change, since the two are only correct together. The fix is in `build-redirects.py` rather than in the generated map, because the map is rewritten from the capture and a hand edit does not survive the next regeneration. `/osd.xml/` stays pointed at the home page: it was the old platform's OpenSearch description and this site emits no such file. + - **`/robots.txt/`, with a trailing slash, now redirects to the real file** rather than to the home page, in the same change, since the two are only correct together. The fix is in `capture/build-redirects.py` rather than in the generated map, because the map is rewritten from the capture and a hand edit does not survive the next regeneration. `/osd.xml/` stays pointed at the home page: it was the old platform's OpenSearch description and this site emits no such file. - **A wrong `HUGO_BASEURL` is still invisible to every gate here, and the `Sitemap:` line does not change that.** Worth stating because the opposite is easy to believe: the parity check compares the advertised origin against the one on the home page's canonical link, and both come from `baseURL`, so they agree whenever the build is coherent — including when `baseURL` was wrong for the environment. Nothing inside the artifact can see it, which is why the check belongs on the side that knows which host it is serving, and the VPS side does it by reading the origin out of the deployed `sitemap.xml`, `og:url` and `feed.xml`. What the comparison does catch is an origin **written rather than derived**, a committed `static/robots.txt` shadowing the template being the way that happens. -- **Nothing checks that media survived the trip to the server.** The VPS agent noticed in §24.3 that a 3,095-request gate run fetched no image at all, and asked whether `golden-media-legacy.txt` is wired in. It is, but only at build time, in `check-url-parity.py`, against files on disk. The live check requests pages and redirects and never an image, so a media tree lost **between the build and the server**, a partial upload, is caught by neither: the build passed before the loss and the live gate never asks. On a site whose value is eighteen years of posts with images in them, that is the gap worth closing rather than the one that was suspected. A handful of media URLs in the live check would close it, chosen to cover both trees rather than to be exhaustive, since the build gate already proves the set. The mechanism that makes this concrete rather than theoretical is the hard-link trap below: a link carries its inode's mode, so a media file that acquires a bad one rides the chain into every later release, present and correctly named and unreadable to the server, which `is_file()` on the runner cannot see and a check that never requests an image cannot either. -- **Restore file mtimes in CI so `--link-dest` links, and do it after the media check rather than before.** The host side measured zero shared inodes across every release the pipeline has delivered, against 1052 of 3266 on a release built here, and the cause is neither the call site nor the confined rsync: both were tested there and link correctly through a relative symlink. Git stores no mtimes, so a CI checkout writes all 3,272 files inside a 23-second window and the `static/` tree that would otherwise match arrives freshly stamped with everything else. `git-restore-mtime` is the fix and needs no checkout change, since `deploy-site-task.yml` already uses `fetch-depth: 0`, and it is deterministic across runs in exactly the place that matters, because `static/` has stable last-commit times. **The ordering is the part worth writing down.** Today every file arrives as a fresh inode, so `--no-g --chmod=D2755,F644` re-establishes the mode contract on every deploy; make the mtimes honest and about a third of the tree starts arriving as links carrying whatever mode its chain began with, which is the trap above. Harmless as things stand, since every inode in the current chain was made by that same rsync line, and it means the live media check should exist first. Nothing is broken meanwhile: the cost is ~585 MB a release, which the host's prune timer reclaims. +- **Media is checked live now, which unblocks the item below.** [`checks/golden-media-live.txt`](./checks/golden-media-live.txt) is fetched by `check-live-urls.sh` against a running server, covering both media trees and the `@uploads` rule, and asserting status, a non-zero body and an image content type so that a 403 from a bad mode, a 404 from a lost transfer, a truncated file and a soft-404 error page are each caught. Verified against production, and each of the four failure shapes was reproduced rather than assumed. The record of why it was needed follows. +- **~~Nothing checks that media survived the trip to the server.~~ Closed 2026-08-08, by the item above.** The VPS agent noticed in §24.3 that a 3,095-request gate run fetched no image at all, and asked whether `golden-media-legacy.txt` is wired in. It is, but only at build time, in `check-url-parity.py`, against files on disk. The live check requests pages and redirects and never an image, so a media tree lost **between the build and the server**, a partial upload, is caught by neither: the build passed before the loss and the live gate never asks. On a site whose value is eighteen years of posts with images in them, that is the gap worth closing rather than the one that was suspected. A handful of media URLs in the live check would close it, chosen to cover both trees rather than to be exhaustive, since the build gate already proves the set. The mechanism that makes this concrete rather than theoretical is the hard-link trap below: a link carries its inode's mode, so a media file that acquires a bad one rides the chain into every later release, present and correctly named and unreadable to the server, which `is_file()` on the runner cannot see and a check that never requests an image cannot either. +- **Restore file mtimes in CI so `--link-dest` links. The media check it waited on now exists, so this is unblocked.** The host side measured zero shared inodes across every release the pipeline has delivered, against 1052 of 3266 on a release built here, and the cause is neither the call site nor the confined rsync: both were tested there and link correctly through a relative symlink. Git stores no mtimes, so a CI checkout writes all 3,272 files inside a 23-second window and the `static/` tree that would otherwise match arrives freshly stamped with everything else. `git-restore-mtime` is the fix and needs no checkout change, since `deploy-site-task.yml` already uses `fetch-depth: 0`, and it is deterministic across runs in exactly the place that matters, because `static/` has stable last-commit times. **The ordering is the part worth writing down.** Today every file arrives as a fresh inode, so `--no-g --chmod=D2755,F644` re-establishes the mode contract on every deploy; make the mtimes honest and about a third of the tree starts arriving as links carrying whatever mode its chain began with, which is the trap above. Harmless as things stand, since every inode in the current chain was made by that same rsync line, and it means the live media check should exist first. Nothing is broken meanwhile: the cost is ~585 MB a release, which the host's prune timer reclaims. - Lower the `blog` A-record TTL to 60s a day ahead, then flip it to the VPS, unproxied. - **Publish a release from `main`, once the pipeline has soaked.** `1.0.11` is the newest release from `main` and was cut on 2026-08-01, ahead of every deploy change, so the next one is the first that would describe a site actually serving its public address. The mechanism is proven and is not what this waits on: it waits on the switchover being trusted rather than merely green, which is what the log review under **Recurring operations** establishes and no gate can. A release cut before that names a state that has not held yet. - Add the weekly non-blocking external-link-check workflow, which is the one gate that cannot be blocking because it fails on other people's outages. @@ -52,7 +53,15 @@ The site is built, gated in CI, and deployed to staging by pipeline. It is not y - **Review the logs for non-200s**, daily for the first week after cutover, then monthly. Real traffic finds what the golden list missed and the crawl that produced the list cannot. Append anything new to `checks/golden-urls.txt` and add a redirect. Read the edge as well as Caddy: a request the proxy refused never reaches the site's log, so a count taken from Caddy alone is a floor, and a staging probe for `/wp-login.php` answered by the auth gate rather than by the site is the shape of what Caddy never sees. The procedure, the three tiers and what each is blind to, and the inward pass that names content nobody has ever requested are in [`OPERATIONS.md`](./OPERATIONS.md) "Log Review". - **Pull the log off the VPS, on a schedule**, which is [#53][issue-53]. The access log is deliberately outside the nightly encrypted archives, because those are fourteen full copies with no dedupe and an append-only file would be multiplied by fourteen for no recovery benefit, so the VPS's 400-day window was the **only** copy until this ran. **It is installed**, as `vps-backup-pull.timer` at 09:00 UTC daily with `Persistent=true`, and a first copy exists: 42 archives and 4 log files, pulled 2026-08-08 12:59 UTC. **That run was started by hand, so the timer has never fired**, which is the distinction worth keeping until 2026-08-09 09:00 UTC proves the schedule rather than the script. One copy is a fact; "backed up daily" is still a unit file. - - **A rename on the VPS does not reach this copy, and nothing detects that it did not.** The pull deliberately passes no `--delete` for the logs, since that flag exists to mirror the VPS's fourteen-archive window and must never touch an append-only file. So when the host side renamed and merged its two mis-dated archives, the pre-fix name survived here: `access.log-2026-08-08`, 52 lines, every one of them 2026-08-07 traffic and every one already inside the merged `access.log-2026-08-07.gz`, which holds 58. Verified a strict subset with `comm -23` rather than assumed. **A line count over the off-host archive therefore returns 110 lines where 58 exist, half of them filed under a date whose traffic they are not** — which is exactly the defect the host side fixed, surviving on the copy the log review will read once the VPS's window rolls past what it needs. The general form is that any rename, merge, or re-compression of an already-pulled log leaves the old name here permanently, and the only propagation mechanism is a note in the channel. Removing that one file is the maintainer's call, since it is a deletion inside a backup tree. + - **A rename on the VPS does not reach this copy, and nothing detects that it did not.** The pull deliberately passes no `--delete` for the logs, since that flag exists to mirror the VPS's fourteen-archive window and must never touch an append-only file. So when the host side renamed and merged its two mis-dated archives, the pre-fix name survived here: `access.log-2026-08-08`, 52 lines, every one of them 2026-08-07 traffic and every one already inside the merged `access.log-2026-08-07.gz`, which holds 58. Verified a strict subset with `comm -23` rather than assumed. **A line count over the off-host archive therefore returned 110 lines where 58 exist, half of them filed under a date whose traffic they are not** — which is exactly the defect the host side fixed, surviving on the copy the log review will read once the VPS's window rolls past what it needs. The general form is that any rename, merge, or re-compression of an already-pulled log leaves the old name here permanently. **That one file is deleted and the archive reads 58**, and the general case now has a mechanism: the host side keeps an append-only `RECONCILE.md` **inside the archive directory**, so the pull carries it alongside the data it explains rather than relying on a note in a channel file nobody rereads. It records what a file contained rather than what it was called, and it will be counted among the pulled log files. + - **Read a date from a line's `StartUTC` rather than from the filename holding it.** That is the durable form of the lesson, and it is in [`OPERATIONS.md`](./OPERATIONS.md) "Log Review" as well. + - **An off-host copy also predates the timer.** The directory mtimes on the backup host are copy times where the file mtimes are the VPS's, and they show a pull at 2026-08-08 03:31 UTC that the service journal has no record of, because the script was run directly rather than through `systemd`. So a journal with one entry is not evidence of one copy. The whole set was audited both ways afterwards and nothing else had diverged: logs identical, 38 archives shared and identical in size, four newer on the VPS because they postdate the pull, four older retained off-host because the pull passes no `--delete`. + - **Both halves were exercised 2026-08-08 between 16:18 and 16:22 UTC, deliberately without running the service.** `systemctl start` would have written the second journal entry that 2026-08-09 is supposed to prove, so the transport was exercised with `--dry-run` instead and the timer was read rather than triggered: `LAST` is `-` and the journal still holds exactly one entry, with `NEXT` inside the 15-minute randomized window after 09:00 UTC. **Read `NEXT` rather than remembering it**, because `systemctl enable` redraws that offset: it moved from 09:08:45 to 09:00:12 UTC when the unit was installed. **If a second entry exists before that time, someone ran it by hand and the schedule is still unproven.** The dry run reached the VPS over SSH and all three legs planned cleanly. Pending for the first timed run: five encrypted archives dated 2026-08-08, plus `RECONCILE.md` **and** `MANIFEST.txt`, so the log-file count rises by two non-log files rather than the one recorded above. No rotated access log is pending, which is correct, because the 00:00 UTC rotation that produces `access.log-2026-08-08` has not happened yet. + - **`--dry-run` named nothing, which made it a connectivity test wearing a preview's name.** `RSYNC_OPTS` carried only `-a --human-readable --info=stats1`, so a dry run printed transfer totals and not one filename, and "what will tomorrow's run bring" was unanswerable by the flag that exists to answer it. Fixed by adding `--itemize-changes` alongside `--dry-run`, which is how the pending set above was read. **The fix is committed at [`ops/vps-backup-pull`](./ops/vps-backup-pull) and installed 2026-08-08**, verified byte-identical to the committed copy. Installing it is a maintainer step, and it stacks with the unshipped change [#53][issue-53] already owes the VPS canonical at `/usr/local/share/pangolin-maint/vps-pull.sh`. + - **The VPS's older copy of the script is not a source, and reconciling the two is [#53][issue-53].** The committed copy carries the whole access-log leg, the `tell()` fix and the `VERIFIED` counter. The VPS's carries an install block and a no-sudo rationale that this side lacked, now folded in. Neither direction is a safe overwrite, so #53 is a merge rather than a copy, and copying the VPS's over the committed one would delete the log pull that [`OPERATIONS.md`](./OPERATIONS.md) "Log Review" runs on. + - **The header's own install command pointed at `/usr/local/sbin/vps-backup-pull`, which nothing runs.** `vps-backup-pull.service` runs `/usr/local/bin/vps-backup-pull` and `/usr/local/sbin/` is empty, so following the instruction would have written a second copy nobody executes while `scp` and `chmod` both reported success and the timer went on running the old one. The canonical had already corrected this to a `sudo install` into `bin/`, and states the reason `bin/` is deliberate. **The stale block is replaced in the patch copy** with the canonical's wording plus an explicit refusal to run that `scp` until the divergence above is reconciled. It is another reason a plain overwrite in either direction is the wrong merge. + - **The outward pass ran end to end on 2026-08-08 traffic and found nothing to add.** 9,996 edge requests, 9,285 of them this repo's own deploy gate. Of the 711 that remain, 101 were site-host 404s across 73 distinct paths, and every one was a scanner shape. Only `/` and `/robots.txt` intersect the URL contract at all, and both are explained rather than open: `/robots.txt` 404ed until the 15:47:17 deploy and has answered 200 since 15:48:45, and `/` 404ed twice at the edge, below. **No legacy content URL 404ed, so `checks/golden-urls.txt` needs no addition from this run**, which is the expected result and is recorded because an unrecorded clean pass is indistinguishable from a pass nobody ran. + - **Two site-host 404s came from the edge rather than the site, and belong to the VPS side.** `/` and `/favicon.ico` at 2026-08-08T15:39:01, one client, same second, both carrying no `ServiceName` at all where the other 99 carried `1-Blog-Production-service@http`. No router matched, so Traefik answered and the blog never saw the request, and requests to `/` seventeen seconds later were routed normally. It sits inside the window the host side was reconfiguring Pangolin in, which is a plausible cause and not a measured one. Raise it in the channel rather than diagnosing it from this side, and note that Caddy is structurally blind to it: a 404 count taken from the site's own log would report zero of these. ## Owed to the hub @@ -69,9 +78,13 @@ The reference leaf the hub now ships carries one step this repo's deploy does no ## Open decisions -- **Where the operational tooling lives, given that today it lives nowhere.** `vps-backup-pull`, its `systemd` units, and the environment variables naming both ends of the copy are an operational asset built from another agent's instructions, and they exist only on the Proxmox host. That host is the machine the backup runs *from*, so losing it loses both the copies and the means of making them, and the instructions that produced them are in a channel file this repository deliberately does not carry. Two candidate homes, and the choice is open: **here**, beside the deploy tooling the same host runs, or **the home-automation config repository**, with the rest of that host's configuration. The argument for the second is that nothing about the pull is specific to this site; the argument for the first is that [`OPERATIONS.md`](./OPERATIONS.md) "Log Review" is the thing that stops working without it. - - **The same question covers the migration toolchain in the capture directory**, which is fourteen scripts: the `wp2hugo` run, the content restructure and clean passes, external-media localization, the crawl and mirror, the golden-URL build, and the media inventory. Some are worth keeping only if generalized, and some are cheaper to rewrite than to maintain, so this is a per-script call rather than one decision. - - **One of them is already three copies with two of them stale**, which is the concrete version of this risk rather than a hypothetical one. `build-redirects.py` exists at the capture root, again under the capture's own `checks/`, and here at [`checks/build-redirects.py`](./checks/build-redirects.py). The two capture copies are identical to each other at 115 lines; the copy in this repository is the maintained one at 225. Nothing detects that, because the capture is not a git repository and is read-only in normal use. +- **Resolved for the backup pull, 2026-08-08: it is in this repository at [`ops/`](./ops/).** The script, both `systemd` units, an `EnvironmentFile` template naming every path it uses, and a README covering what it does and how to check it. [`OPERATIONS.md`](./OPERATIONS.md) "Working With the VPS" names it and states which of its variables pair with which of this repo's. The reasoning below stands as the record of why, and the same question is still open for everything under it. **Installed 2026-08-08 with `ops/install.sh`**, which derives the address, both destinations, the account, the group and the mount from `secrets/local.production.env`, so nothing is typed twice. Verified after the fact rather than from the installer's own output: `systemd` resolves `User=pieter`, `Group=users` and `RequiresMountsFor=/data/backup` from the drop-in, and the environment file is `600 root:root`. The running script is byte-identical to the committed one. Re-running the installer after the shell-gate reformat also exercised its idempotent path, which reported both config files already correct and replaced only the script, so a changed value is applied by running it again rather than by editing anything on the host. The root guard was exercised and refused. **The journal still holds exactly one entry and the timer's `LAST` is still `-`**, so installing did not spend the evidence that 2026-08-09 is the first scheduled run. Separately, [#53][issue-53] reconciles the VPS's older copy in both directions rather than by overwriting either. +- **Where the rest of the operational tooling lives, given that today it lives nowhere.** `vps-backup-pull`, its `systemd` units, and the environment variables naming both ends of the copy were an operational asset built from another agent's instructions, and they existed only on the Proxmox host. That host is the machine the backup runs *from*, so losing it loses both the copies and the means of making them, and the instructions that produced them are in a channel file this repository deliberately does not carry. Two candidate homes, and the choice is open: **here**, beside the deploy tooling the same host runs, or **the home-automation config repository**, with the rest of that host's configuration. The argument for the second is that nothing about the pull is specific to this site; the argument for the first is that [`OPERATIONS.md`](./OPERATIONS.md) "Log Review" is the thing that stops working without it. + - **The pull itself is resolved and the reasoning is kept because it applies to everything still listed here.** What made it urgent was measured: the copy protected everywhere was the VPS's older one, while the copy that actually ran, carrying the log leg the review depends on, was in no snapshot and no repository. Committing it is what closed that, not the backup host's own off-site copy, which never reached the script. + - **The directory holding it is named as though it were disposable.** `~/vps-backup-pull-patch` reads as a patch staged against a source, and there is no source: it is the most complete copy of the script in existence. A directory named for a temporary artifact is the one a cleanup deletes, and nothing here would notice until a restore produced the wrong script. + - **The same reasoning points at the home-automation configuration repository for anything that is purely this host's**, since that is where the rest of the backup host's service configuration already lives. The pull is here instead because [`OPERATIONS.md`](./OPERATIONS.md) "Log Review" is what stops working without it. Revisit if a second unrelated host service ends up here. + - **Resolved for the migration toolchain: the durable scripts are in this repository at [`capture/`](./capture/).** The fourteen split by whether they can ever run again rather than by whether they look reusable. A re-export re-runs the conversion chain, so `run-wp2hugo.sh`, `clean-content.py`, `restructure-content.py`, `localize-external.py` and `enumerate-media.py` are carried. `build-golden.py` and `classify.py` are carried as the record behind an append-only contract, and say so, since they cannot run once the old hosting ends. The crawl pair and the three fetchers are named in [`capture/README.md`](./capture/README.md) rather than carried, because their outputs are the durable artifact and the scripts hit a site that will be gone. + - **The three copies of `build-redirects.py` are down to one.** The maintained copy moved from `checks/` to [`capture/`](./capture/build-redirects.py), which is where it belonged: it generates rather than gates, and `checks/` is defined as the contract and the gates that enforce it. Both stale copies in the capture are named by path in the capture README and marked stale, so finding one is not mistaken for finding a source. - **What `robots.txt` says, which is undecided and is the last non-mechanical item before M7b.** Recorded under "Next" above, where it blocks the cutover. - `/osd.xml/` sits in `slugs.map` pointing at `/`, and stays there unless this site ever emits an OpenSearch description. `/robots.txt/` was the other half and is resolved, above. - Content is capped at a fixed 720px on every screen, because PaperMod's width is four CSS variables with no responsive term and no Hugo parameter. The prose measure is right and should stay; images and galleries inheriting the same cap is the part that costs something on a wide display. The knobs, the override location, and the `--gap` trap are documented under "Customization points" in [`themes/README.md`](./themes/README.md). @@ -147,7 +160,7 @@ The URL contract lives in this repo and is the thing CI enforces. | [`checks/README.md`](./checks/README.md) | how the contract was derived | | [`deploy/README.md`](./deploy/README.md) | the release mechanics and the redirect design | -The provenance store holds the raw exports, the media tar, and the crawl. It lives outside this repo, is never published, and is passed to `checks/build-redirects.py` as an argument, which is why that script is a provenance tool rather than a CI step. +The capture holds the raw exports, the media tar, and the crawl. It lives outside this repo at `CAPTURE_ROOT` and is never published, because it carries commenter email addresses and IP addresses that the conversion drops rather than scrubs. [`capture/README.md`](./capture/README.md) is the procedure. Secrets and variables, per environment. The App-token pair is repository-scoped rather than per-environment. diff --git a/capture/README.md b/capture/README.md new file mode 100644 index 0000000..76e5e8a --- /dev/null +++ b/capture/README.md @@ -0,0 +1,122 @@ +# Provenance Capture + +Everything in this repository is derived. What it was derived *from* is a capture directory that lives outside it, and this directory holds the scripts that read that capture. + +The capture's path is `CAPTURE_ROOT`, recorded alongside the other values that name a machine rather than the project. It is environment-independent, so unlike a deploy root it belongs in `secrets/local.production.env` alone rather than in a copy per environment: there is one capture, and four copies of its path is four chances for three of them to be wrong. It is not a git repository, so it has no history to revert to, and it is read-only in normal use. Nothing here writes into it except the steps below that say they do. + +**None of this runs in CI, and none of it runs on a schedule.** These are provenance tools, run by hand, and their outputs are committed. That is the whole difference between this directory and [`checks/`](../checks/), which holds gates that run on every change. + +## What is under the capture + +| Under the capture | Holds | Recoverable | +| --- | --- | --- | +| `export/raw/` | the WordPress content export, WXR XML | yes, from the WordPress account while it exists | +| `export/media-tar/` | the media export, the only trustworthy copy of the images | yes, from the same place | +| `mirror/` | a crawl of the old platform as it served, including the media it linked from other hosts | no, once the old hosting ends | +| `inventory/` | the URL and media inventories derived from that crawl | no, for the same reason | + +**The mirror spans more hosts than the blog.** It was taken across the blog itself, the platform's image CDN, and the Google hosts that served the hotlinked images, which is why it holds media that was never in the media library. That matters beyond the migration: [`checks/README.md`](../checks/README.md) adjudicates orphaned media against this tree, and an adjudication is only as good as what the crawl reached. + +**The capture holds personal data and is never committed.** The WXR carries commenter email addresses and IP addresses, and the mirror carries the same data rendered into HTML. The conversion resolves this by dropping comments entirely rather than scrubbing them, so there is no partial to get wrong, and the site this repository builds has none. The capture itself keeps them, which is one more reason it stays outside git. + +## The two exports, and the counts they must reconcile + +Two downloads from the WordPress account, both from Tools then Export. + +1. **The content export.** Choose **All content** and do not filter by date or type. One WXR XML file. This carries the post and page bodies in raw form, the comments, the categories and tags, and pointers to the media. It is the primary input and the one thing that cannot be fetched from outside the account. +2. **The media library export.** A separate menu item on the same page, producing a `.tar` organized into year and month folders. Not strictly required, because the crawl captures what the posts reference, but worth taking for two reasons: it includes media that was uploaded and never embedded in a post, which the XML omits entirely, and it avoids rate-limiting during conversion. + +**Verify the export against what the live site reported before trusting it.** A partial export is the common way a migration loses posts without reporting anything, so a mismatch is a stop-and-investigate rather than a rounding error. These are a **measurement of the old platform taken 2026-07-29**, not values this repository can check, and they are recorded because nothing else will ever be able to state them again: + +| | Measured | +| --- | --- | +| Posts | 108 | +| Pages | 2 | +| Comments | 397 | +| Categories | 12 | +| Tags | 183 | +| Media assets | 941, being 675 in the library and 266 hotlinked | + +**Do not cancel the old plan, delete the site, or change DNS until the cutover is done and has held.** The conversion fetches media over HTTP from the live site, so cancelling early loses images, and deleting the site destroys the ability to re-export. The DNS change *is* the cutover and comes last. + +## Rebuilding from the exports + +In order. Each Python step is a dry run by default and takes `--apply` to write. + +Everything here is standard library except `clean-content.py`, which needs PyYAML to read front matter. It names the package if it is missing rather than raising an import error. + +```sh +set -a; . secrets/local.production.env; set +a + +capture/run-wp2hugo.sh # convert, into $CAPTURE_ROOT/converted/ +capture/clean-content.py --apply # drop comments, reduce the front matter +capture/restructure-content.py --apply # reshape the tree to match the URLs +capture/localize-external.py --apply # pull the hotlinked images local +capture/build-redirects.py # regenerate deploy/maps/ +``` + +`build-redirects.py` is the only one that writes into this repository, and what it writes is committed. The other four write into the capture. + +**The export is selected by content, never by filename.** An account holds several exports and a media-only one carries the attachments and no posts, so a glob would choose by filesystem order and converting the wrong one yields a site that builds and is empty. `build-redirects.py` makes the choice, and `run-wp2hugo.sh` asks it with `--print-export` rather than repeating the logic, so the conversion and the maps are provably built from the same file. + +## What the conversion does not carry + +**Hotlinked images are in no export.** Images embedded from other hosts during the Windows Live Writer and Blogger era were never in the media library, so they are absent from the WXR, and the converter skips them as non-relative links. Left alone they stay a permanent dependency on someone else serving fifteen-year-old URLs. `localize-external.py` downloads them, names each by a hash of its source URL, because the originals carry characters that do not survive a filesystem, then rewrites every reference and records the mapping in the capture's inventory. + +**Comments are dropped rather than scrubbed.** The blog is read-only going forward, so the data file is deleted outright. There is no partial scrub to get wrong and nothing to carry. + +**Front matter is reduced to what drives the site.** The converter carries every WordPress custom field through, most of them platform internals that no template reads. `clean-content.py` keeps a short list and drops the rest. The list is in the script rather than restated here, so a reader can diff it against a new export. + +## Three results that look like success and are not + +- **A Picasa or ggpht URL whose size segment ends in `-h` returns HTTP 200 with an HTML page**, not an image, and a naive fetch writes a few hundred bytes of markup with a successful status. `localize-external.py` parses the wrapper and follows the `` it names, which is the platform's own answer rather than a guess. +- **Any non-image body is a failure regardless of status.** The magic bytes are checked, and anything that is not an image is reported rather than written. +- **Selecting the export by filename yields empty maps that pass every check** until the redirects are live. See the note above on selection by content. + +## How the URL contract was captured + +The contract in [`checks/`](../checks/) was not derived from the content tree. It was measured against the live old platform, by union of three sources and then verified URL by URL: + +| Script | Did | +| --- | --- | +| `enumerate-media.py` | pulled every post and page from the platform's REST API in **rendered** form, so shortcodes were expanded and a media reference was seen as a browser sees it, and split the results by host to separate library media from hotlinks | +| `build-golden.py` | unioned the crawl, the sitemap and a derivation of the archive and pagination shapes, then requested every candidate against the live site and recorded what it answered | +| `classify.py` | split the verified set into what must render and what must redirect, using the sitemap as the discriminator for bare one-segment URLs, which are ambiguous between a real page and an attachment page | + +**These three cannot run again once the old hosting ends.** They are carried as the record behind an append-only contract, not as a step anyone re-runs. An append to `golden-urls.txt` is only reviewable against the derivation that produced the original, which is why the derivation is here rather than lost with the platform. + +**They write into the capture and never into `checks/`.** The committed lists are grown by hand from a log finding, per that directory's own maintenance rules. A script that could rewrite them would be able to replace a verified contract with an inferior re-derivation. + +## Two properties of the maps that are easy to break + +**The Blogger map holds more entries than there are Blogger-era posts.** That platform truncated an auto-generated slug at a fixed length on a whole-word boundary, so a long title was served at the truncated address and that is the address in search indexes. The importer registered the full slug, both answer, and both are mapped. The limit is a named constant in `build-redirects.py`, and changing it changes how many entries the map has. + +**A term archive the old platform served is not always one Hugo builds.** Where it does not, the URL still has to answer, which is why the maps carry terms that no longer exist as pages. + +## What lived in the capture and is not carried + +Seven scripts stayed behind, and each is named here so nobody goes looking for something that was deliberately left. + +| In the capture | Did | Why it is not here | +| --- | --- | --- | +| `crawl/spider.sh` | enumerated every URL the live site served, politely and with an identifying user agent | A short flag list against a site that will be gone. Its output, the crawl log, is the durable artifact and is in the capture. | +| `crawl/mirror.sh` | took the offline snapshot, spanning the blog and the image hosts | Same, and its span-host list is recorded above because the orphan adjudication depends on it. | +| `inventory/fetch-at-risk.sh` | downloaded every hotlinked image with its size and SHA-256 | The manifest it wrote is the durable artifact. Re-verification reads the manifest and never re-fetches. | +| `inventory/fetch-missing.sh` | fetched library media the crawl did not reach | Same shape, same reason. | +| `inventory/fetch-unattached.sh` | fetched media listed in the export but never embedded in a post | Same shape, same reason. | +| `build-redirects.py` at the capture root | an older copy of the generator | **Stale.** Superseded by the copy here. | +| a second copy under the capture's own `checks/` | byte-identical to the one above | **Stale.** Superseded by the copy here. That directory belongs to the capture and is not this repository's `checks/`. | + +**The two stale copies are named by path on purpose**, so that finding one is not mistaken for finding a source. Neither has the fixes this copy carries. + +## Variables + +Every path and address is a variable, so nothing here names a machine. [`example.env`](../example.env) lists them and [`ENVIRONMENT.md`](../ENVIRONMENT.md) describes them. + +A required value that is unset stops the script and names what is missing, rather than falling back to something plausible. A wrong-but-valid capture directory produces empty maps that are indistinguishable from working ones until the redirects are live, which is the failure this rule exists for. + +`CAPTURE_SOURCE_URL` is the **old** platform. It holds the same string as `HUGO_BASEURL` after the cutover and means something different, so merging the two would point a verification run at the new site while every check still passed. + +## This directory is the source + +Edit the copy here and run it from here. A copy found in the capture is an artifact of when it ran, not a place to make a change. diff --git a/capture/build-golden.py b/capture/build-golden.py new file mode 100755 index 0000000..8cd0045 --- /dev/null +++ b/capture/build-golden.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Build the golden URL list: every URL the live WordPress site actually serves. + +Three sources, unioned, then every candidate verified against the live site: + + 1. The recursive crawl - finds linked pages, but misses anything unlinked. + 2. The sitemap - only 111 URLs, misses taxonomy terms entirely. + 3. Derivation - date archives and pagination are served but linked from nowhere, + so no crawl can find them. Computed from post dates and taxonomy post counts. + +Verification is what makes the list authoritative: a candidate is kept only if the +live site answers for it. The output is split by how Hugo must satisfy each URL: + + golden-urls.txt - must render as a page (200) + redirect-urls.txt - must redirect (301/308), or is a WordPress-ism with no Hugo + equivalent (attachment pages, per-post comment feeds) that we + choose to redirect to the parent post rather than reproduce. +""" + +import json +import math +import os +import re +import sys +import threading +import urllib.error +import urllib.request +from collections import Counter +from pathlib import Path +from queue import Queue + +def env(name: str) -> str: + """A required capture value, refused rather than guessed when unset. + + Duplicated across the scripts here rather than shared, because these get copied out to + a scratch directory to run, and an import would break the moment one travelled alone. + """ + v = os.environ.get(name, "") + if not v: + sys.exit(f"{name} is not set -- see example.env and ENVIRONMENT.md") + return v + + +BASE = env("CAPTURE_SOURCE_URL") +API = env("CAPTURE_SOURCE_API") +PER_PAGE = 10 # measured against the old platform: the post count paginated to /page/11/ +# The capture, never this script's directory. Both outputs below are capture artifacts and +# must not be able to land in the repository's checks/, whose lists are append-only and are +# grown by hand from a log finding rather than regenerated. +ROOT = Path(env("CAPTURE_ROOT")) +UA = {"User-Agent": "Mozilla/5.0 (compatible; blog-migration-audit/1.0)"} + + +def api(path): + req = urllib.request.Request(f"{API}/{path}", headers=UA) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + + +def norm(p): + p = re.sub(r"^https?://[^/]+", "", p) + p = p.split("#")[0] + if not p.startswith("/"): + p = "/" + p + return p if p.endswith("/") else p + "/" + + +def pages_for(count): + """Pagination URLs beyond page 1 for an archive holding `count` posts.""" + return range(2, math.ceil(count / PER_PAGE) + 1) if count > PER_PAGE else [] + + +def build_candidates(): + cand = {"/"} + + # 1. Crawl + log = (ROOT / "crawl" / "spider.log").read_text(encoding="utf-8", errors="replace") + # Built from CAPTURE_SOURCE_URL rather than written in, so the pattern and the site the + # crawl actually ran against cannot drift apart. + crawled = {norm(u) for u in re.findall(re.escape(BASE) + r"[^\s]*", log)} + cand |= {u for u in crawled if not u.startswith("/wp-content/")} + print(f" crawl: {len(crawled)} paths") + + # 2. Sitemap + try: + req = urllib.request.Request(f"{BASE}/sitemap.xml", headers=UA) + with urllib.request.urlopen(req, timeout=30) as r: + sm = {norm(u) for u in re.findall(r"([^<]+)", r.read().decode())} + cand |= sm + print(f" sitemap: {len(sm)} URLs") + except Exception as e: # noqa: BLE001 - the sitemap is a nice-to-have source + print(f" sitemap: FAILED ({e})", file=sys.stderr) + + # 3. Derivation - the part no crawl can reach. + posts = json.loads((ROOT / "inventory" / "posts.json").read_text(encoding="utf-8")) + derived = set() + + ym = Counter() + for p in posts: + d = p.get("date", "") + if len(d) >= 7: + ym[(d[:4], d[5:7])] += 1 + years = Counter() + for (y, m), n in ym.items(): + derived.add(f"/{y}/{m}/") + years[y] += n + for i in pages_for(n): + derived.add(f"/{y}/{m}/page/{i}/") + for y, n in years.items(): + derived.add(f"/{y}/") + for i in pages_for(n): + derived.add(f"/{y}/page/{i}/") + + for i in pages_for(len(posts)): + derived.add(f"/page/{i}/") + + for kind, key, field in (("category", "categories", "categories"), ("tag", "tags", "tags")): + data = api(f"{field}?number=500")[key] + for t in data: + derived.add(f"/{kind}/{t['slug']}/") + for i in pages_for(t["post_count"]): + derived.add(f"/{kind}/{t['slug']}/page/{i}/") + + derived.add("/feed/") + cand |= derived + print(f" derived: {len(derived)} URLs (date archives + pagination, unlinked)") + return sorted(cand) + + +def verify(urls, workers=6): + """HEAD every candidate. Redirects are recorded, not followed.""" + out, q, lock = {}, Queue(), threading.Lock() + for u in urls: + q.put(u) + + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *a, **k): + return None + + opener = urllib.request.build_opener(NoRedirect) + + def worker(): + while True: + try: + u = q.get_nowait() + except Exception: # noqa: BLE001 - empty queue ends the worker + return + code, loc = 0, "" + for _ in range(3): + try: + req = urllib.request.Request(BASE + u, headers=UA, method="HEAD") + with opener.open(req, timeout=25) as r: + code, loc = r.status, r.headers.get("Location", "") + break + except urllib.error.HTTPError as e: + code, loc = e.code, e.headers.get("Location", "") + break + except Exception: # noqa: BLE001 - transient, retry + continue + with lock: + out[u] = (code, loc) + if len(out) % 200 == 0: + print(f" verified {len(out)}/{len(urls)}", flush=True) + q.task_done() + + threads = [threading.Thread(target=worker, daemon=True) for _ in range(workers)] + for t in threads: + t.start() + for t in threads: + t.join() + return out + + +def main(): + print("Building candidate set:") + cand = build_candidates() + print(f" UNION: {len(cand)} candidates\n") + + print("Verifying against the live site:") + res = verify(cand) + + # An attachment page is /YYYY/MM/DD/post/attachment/ - a WordPress-ism with no Hugo + # equivalent. The image itself stays at its wp-content path; the page redirects home + # to its parent post. Per-post /feed/ endpoints get the same treatment. + attach = re.compile(r"^/\d{4}/\d{2}/\d{2}/[^/]+/[^/]+/$") + feed = re.compile(r"/feed/$") + + golden, redirect, dropped = [], [], [] + for u, (code, loc) in sorted(res.items()): + if code in (301, 302, 307, 308): + redirect.append((u, code, loc)) + elif code == 200: + (redirect if (attach.match(u) or (feed.search(u) and u != "/feed/")) else golden).append( + (u, code, "") + ) + else: + dropped.append((u, code, loc)) + + (ROOT / "checks").mkdir(exist_ok=True) + (ROOT / "checks" / "golden-urls.txt").write_text("".join(f"{u}\n" for u, _, _ in golden), encoding="utf-8") + (ROOT / "checks" / "redirect-urls.txt").write_text("".join(f"{u}\n" for u, _, _ in redirect), encoding="utf-8") + with (ROOT / "checks" / "url-verification.tsv").open("w", encoding="utf-8") as fh: + fh.write("class\tstatus\turl\tlocation\n") + for name, rows in (("golden", golden), ("redirect", redirect), ("dropped", dropped)): + for u, c, loc in rows: + fh.write(f"{name}\t{c}\t{u}\t{loc}\n") + + print(f"\n GOLDEN (must render): {len(golden)}") + print(f" REDIRECT: {len(redirect)}") + print(f" dropped (4xx/5xx/0): {len(dropped)}") + print("\n golden by shape:") + shape = Counter() + for u, _, _ in golden: + if u == "/": + shape["home"] += 1 + elif re.match(r"^/\d{4}/\d{2}/\d{2}/[^/]+/$", u): + shape["post"] += 1 + elif "/page/" in u: + shape["pagination"] += 1 + elif u.startswith("/tag/"): + shape["tag archive"] += 1 + elif u.startswith("/category/"): + shape["category archive"] += 1 + elif re.match(r"^/\d{4}/(\d{2}/)?$", u): + shape["date archive"] += 1 + else: + shape["page/other"] += 1 + for k, v in shape.most_common(): + print(f" {k:20} {v}") + if dropped: + print(f"\n sample dropped: {[u for u, _, _ in dropped[:5]]}") + + +if __name__ == "__main__": + main() diff --git a/checks/build-redirects.py b/capture/build-redirects.py similarity index 78% rename from checks/build-redirects.py rename to capture/build-redirects.py index 3d94171..fa7eb25 100755 --- a/checks/build-redirects.py +++ b/capture/build-redirects.py @@ -2,9 +2,14 @@ """Generate the Caddy redirect maps from the WordPress export. Reads the export and the capture inventory, which live outside this repo, and writes the -committed maps under deploy/maps/. See OPERATIONS.md for when to run it. +committed maps under deploy/maps/. See capture/README.md for when to run it. + +The capture directory comes from CAPTURE_ROOT, and a first argument wins over it. This +generates rather than gates, which is why it sits in capture/ beside the other scripts +that read the capture rather than in checks/ beside the gates. """ +import os import pathlib import re import sys @@ -12,7 +17,11 @@ NS = {"wp": "http://wordpress.org/export/1.2/"} -CHECKS = pathlib.Path(__file__).resolve().parent +# Anchored on the repository rather than on this file's own directory, because the two +# lists below live in checks/ and the maps in deploy/maps/, and neither follows this +# script if it moves again. +REPO = pathlib.Path(__file__).resolve().parent.parent +CHECKS = REPO / "checks" # Blogger truncates an auto-generated slug at this many characters, on a whole-word boundary. # For a longer slug the truncated form is the URL Blogger served, and so the one in search indexes. @@ -52,17 +61,40 @@ def blogger_truncate(slug, limit=BLOGGER_SLUG_LIMIT): def write_map(path, pairs): path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("".join(f"{k} {v}\n" for k, v in pairs)) + path.write_text("".join(f"{k} {v}\n" for k, v in pairs), encoding="utf-8") return len(pairs) def main(argv): - if len(argv) != 2: - print(__doc__.strip().splitlines()[0], file=sys.stderr) - print(f"usage: {argv[0]} ", file=sys.stderr) + # CAPTURE_ROOT is the default and an argument wins over it, the same shape DEPLOY_ROOT + # uses. Unset and unsupplied is refused rather than guessed: a wrong-but-plausible + # capture yields maps that are empty and indistinguishable from working ones. + # --print-export names the selected export and writes nothing. run-wp2hugo.sh calls it + # rather than reimplementing the choice, so the conversion and the maps are provably + # built from the same file. A shell reimplementation cannot match this anyway: the test + # is that ONE item carries both post_type=post and status=publish, and two greps over a + # whole file would accept a media-only export that happens to contain both words. + args = [a for a in argv[1:] if a != "--print-export"] + print_export = "--print-export" in argv[1:] + # An unknown option is refused rather than taken as a path. Without this, `--help` is + # read as a capture directory and the run fails with "no export XML found under --help", + # which sends the reader looking for a missing file rather than a mistyped flag. + unknown = [a for a in args if a.startswith("-")] + if unknown: + print(f"unknown option: {unknown[0]}", file=sys.stderr) + print(f"usage: {argv[0]} [--print-export] [capture-dir]", file=sys.stderr) + return 2 + if len(args) > 1: + print(f"usage: {argv[0]} [--print-export] [capture-dir]", file=sys.stderr) + return 2 + root = args[0] if args else os.environ.get("CAPTURE_ROOT", "") + if not root: + print(f"usage: {argv[0]} [--print-export] [capture-dir]", file=sys.stderr) + print("CAPTURE_ROOT is not set and no capture directory was given", file=sys.stderr) + print("see example.env and ENVIRONMENT.md", file=sys.stderr) return 2 - capture = pathlib.Path(argv[1]) - out = pathlib.Path(__file__).resolve().parent.parent / "deploy" / "maps" + capture = pathlib.Path(root) + out = REPO / "deploy" / "maps" # An account holds several exports, and a media-only one carries the attachments but no posts. # Filesystem order decides which a glob returns, and the wrong one yields empty maps that look valid. @@ -84,6 +116,9 @@ def main(argv): print(f" {path}", file=sys.stderr) return 1 src, root = with_posts[0] + if print_export: + print(src) + return 0 print(f"export: {src}") posts, attachments, blogger, terms = {}, [], [], [] @@ -129,7 +164,7 @@ def main(argv): # Reading the render list also resolves a slug that exists as both a tag and a category. # Map lookups are case-sensitive, so each slug is emitted alongside a capitalized variant. rendered = set() - for line in (CHECKS / "golden-urls.txt").read_text().splitlines(): + for line in (CHECKS / "golden-urls.txt").read_text(encoding="utf-8").splitlines(): if m := re.match(r"^/(tag|category)/([^/]+)/$", line.strip()): rendered.add((m.group(2), m.group(1))) destinations = {} @@ -166,7 +201,7 @@ def main(argv): r"|^/p/[^/]+\.html$" # Blogger static pages ) # The repo's list, not the capture's, which is a frozen snapshot from before the contract grew. - redirects = (CHECKS / "redirect-urls.txt").read_text().splitlines() + redirects = (CHECKS / "redirect-urls.txt").read_text(encoding="utf-8").splitlines() needed = [u for u in (line.strip() for line in redirects) if u and u.count("/") == 2 and not covered.match(u)] # Attachments with a real post_parent resolve directly. @@ -178,7 +213,7 @@ def main(argv): file_to_post = {} inv = capture / "inventory" / "media-urls.tsv" if inv.exists(): - for line in inv.read_text().splitlines()[1:]: + for line in inv.read_text(encoding="utf-8").splitlines()[1:]: parts = line.split("\t") if len(parts) >= 4: stem = pathlib.PurePosixPath(parts[1].split("?")[0]).stem.lower() diff --git a/capture/classify.py b/capture/classify.py new file mode 100755 index 0000000..0c52eba --- /dev/null +++ b/capture/classify.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Split the verified URL set into what Hugo must render and what must redirect. + +Reads the cached verification results so this can be re-run without re-hitting the +live site. The discriminator that matters: WordPress serves an attachment page for +every uploaded image, at BOTH /YYYY/MM/DD/post/attachment/ and a bare /attachment/. +A bare one-segment URL is therefore ambiguous with a real page, and the sitemap is +the authority - it lists exactly the posts and pages, so a one-segment URL absent +from it is an attachment page, not content. +""" + +import os +import re +import sys +import urllib.request +from collections import Counter +from pathlib import Path + + + +def env(name: str) -> str: + """A required capture value, refused rather than guessed when unset. + + Duplicated across the scripts here rather than shared, because these get copied out to + a scratch directory to run, and an import would break the moment one travelled alone. + """ + v = os.environ.get(name, "") + if not v: + sys.exit(f"{name} is not set -- see example.env and ENVIRONMENT.md") + return v + + +# The capture, never this script's directory. Both outputs below are capture artifacts and +# must not be able to land in the repository's checks/, whose lists are append-only. +ROOT = Path(env("CAPTURE_ROOT")) +BASE = env("CAPTURE_SOURCE_URL") +UA = {"User-Agent": "Mozilla/5.0 (compatible; blog-migration-audit/1.0)"} + +req = urllib.request.Request(f"{BASE}/sitemap.xml", headers=UA) +with urllib.request.urlopen(req, timeout=30) as r: + sitemap = { + re.sub(r"^https?://[^/]+", "", u).rstrip("/") + "/" + for u in re.findall(r"([^<]+)", r.read().decode()) + } + +rows = [ + line.rstrip("\n").split("\t") + for line in (ROOT / "checks" / "url-verification.tsv").read_text(encoding="utf-8").splitlines()[1:] +] +status = {r[2]: (int(r[1]), r[3] if len(r) > 3 else "") for r in rows} + +# The author archive and its pagination are served but linked from nowhere, and on a +# single-author blog they duplicate the home archive exactly. Verified live: /page/11/ +# is the last one. +# +# Optional, and skipped loudly rather than silently. An account slug is not this +# repository's to carry, and a run without it produces a list short by the author URLs +# rather than a wrong one. Silence would be indistinguishable from a site that never +# served them. +AUTHOR_SLUG = os.environ.get("CAPTURE_AUTHOR_SLUG", "") +if AUTHOR_SLUG: + for i in range(2, 12): + status.setdefault(f"/author/{AUTHOR_SLUG}/page/{i}/", (200, "")) + status.setdefault(f"/author/{AUTHOR_SLUG}/", (200, "")) +else: + print("CAPTURE_AUTHOR_SLUG is not set: skipping the author archive backfill", file=sys.stderr) + +POST = re.compile(r"^/\d{4}/\d{2}/\d{2}/[^/]+/$") +NESTED_ATTACH = re.compile(r"^/\d{4}/\d{2}/\d{2}/[^/]+/[^/]+/$") +DATE = re.compile(r"^/\d{4}/(\d{2}/)?$") +DATE_PAGED = re.compile(r"^/\d{4}/(\d{2}/)?page/\d+/$") +HOME_PAGED = re.compile(r"^/page/\d+/$") +TERM = re.compile(r"^/(tag|category)/[^/]+/(page/\d+/)?$") +AUTHOR = re.compile(r"^/author/[^/]+/(page/\d+/)?$") + +golden, redirect, dropped = [], [], [] +reason = Counter() + +for url, (code, _loc) in sorted(status.items()): + if code not in (200, 301, 302, 307, 308): + dropped.append(url) + continue + if code != 200: + redirect.append(url) + reason["already a redirect on WordPress"] += 1 + elif NESTED_ATTACH.match(url): + redirect.append(url) + reason["attachment page (nested)"] += 1 + elif url.endswith("/feed/") and url != "/feed/": + redirect.append(url) + reason["per-post or per-term feed"] += 1 + elif url == "/feed/": + redirect.append(url) + reason["site feed -> /index.xml"] += 1 + elif AUTHOR.match(url): + redirect.append(url) + reason["author archive (single author, duplicates home)"] += 1 + elif DATE.match(url) or DATE_PAGED.match(url): + # Decision (2026-07-29): Hugo has no built-in year/month archive, and these are + # absent from the sitemap and linked from nowhere on the live site. Building them + # would be the only custom templating in the migration, for URLs nothing points at. + # One Caddy pattern rule sends them home instead. + redirect.append(url) + reason["date archive -> / (no Hugo equivalent, unlinked)"] += 1 + elif POST.match(url) or HOME_PAGED.match(url) or TERM.match(url) or url == "/": + golden.append(url) + elif url.rstrip("/").count("/") == 1: + # One segment: a real page if the sitemap lists it, otherwise an attachment. + if url in sitemap: + golden.append(url) + else: + redirect.append(url) + reason["attachment page (root level)"] += 1 + else: + golden.append(url) + +(ROOT / "checks" / "golden-urls.txt").write_text("".join(f"{u}\n" for u in sorted(golden)), encoding="utf-8") +(ROOT / "checks" / "redirect-urls.txt").write_text("".join(f"{u}\n" for u in sorted(redirect)), encoding="utf-8") + +shape = Counter() +for u in golden: + if u == "/": + shape["home"] += 1 + elif POST.match(u): + shape["post"] += 1 + elif TERM.match(u) and "/page/" in u: + shape["taxonomy pagination"] += 1 + elif u.startswith("/tag/"): + shape["tag archive"] += 1 + elif u.startswith("/category/"): + shape["category archive"] += 1 + elif DATE.match(u): + shape["date archive"] += 1 + elif DATE_PAGED.match(u) or HOME_PAGED.match(u): + shape["date/home pagination"] += 1 + else: + shape["page"] += 1 + +print(f"GOLDEN (Hugo must render): {len(golden)}") +for k, v in shape.most_common(): + print(f" {k:24} {v}") +print(f"\nREDIRECT: {len(redirect)}") +for k, v in reason.most_common(): + print(f" {k:46} {v}") +print(f"\ndropped (4xx): {len(dropped)}") diff --git a/capture/clean-content.py b/capture/clean-content.py new file mode 100755 index 0000000..a20205d --- /dev/null +++ b/capture/clean-content.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Reduce the converted site to content only. + +Two jobs, both decided deliberately: + +1. **Drop comments entirely.** The blog is read-only going forward - no comments, no + interaction. Deleting the data file rather than scrubbing it means there is no PII + question at all, no partial to write, and nothing to carry. + +2. **Strip front matter to what actually drives the site.** wp2hugo carries every + WordPress custom field through, which is 156 distinct keys across 108 posts, 127 of + them WordPress internals (`_edit_last`, `_oembed_`, `_jetpack_*`, `_coblocks_*`, + `_publicize_*`). None of it is content. + +Kept, with the reason each earns its place: + + title required + date required, drives the permalink and ordering + url the exact WordPress permalink - keeping it guarantees URL preservation + independent of any hugo.yaml permalink config + categories taxonomy, 12 terms + tags taxonomy, 183 terms + post_id drives the ?p= shortlink redirect map - inbound-link surface + cover featured image, {alt, image}, on 33 posts + +Dropped on purpose: `author` (single-author blog, the site knows), `guid` (GUID +preservation was dropped, and wp2hugo rewrote the scheme to https anyway, which would +have broken RSS), `parent_post_id` (null on every post), `blogger_*` (provenance from an +earlier Blogger migration), `publicize_*` (dead social-share links), `geo_*` (no template +consumes it), and every `_`-prefixed WordPress internal. +""" + +import os +import re +import shutil +import sys +import pathlib +from pathlib import Path + +try: + import yaml +except ModuleNotFoundError: # the only third-party import in capture/ or checks/ + sys.exit("PyYAML is required by this script: pip install PyYAML (Debian: apt install python3-yaml)") + +KEEP = ["title", "date", "url", "categories", "tags", "post_id", "cover"] +FM = re.compile(r"^---\n(.*?)\n---\n(.*)$", re.S) + + +def capture_root() -> pathlib.Path: + """CAPTURE_ROOT, refused rather than guessed when unset. + + Duplicated in each script here rather than shared, because these get copied out to a + scratch directory to run against a copy of the capture, and an import would break the + moment one of them travelled alone. + """ + root = os.environ.get("CAPTURE_ROOT", "") + if not root: + sys.exit("CAPTURE_ROOT is not set -- see example.env and ENVIRONMENT.md") + return pathlib.Path(root) + + +def converted_site(argv) -> pathlib.Path: + """The converted site: a first argument, else the one generated-* under the capture. + + Ambiguity aborts rather than picking, the same rule build-redirects.py applies to the + export. wp2hugo stamps the directory with a run timestamp, so there is no fixed name + to default to and a glob that matched two would otherwise choose by filesystem order. + """ + if len(argv) > 1 and not argv[1].startswith("--"): + return pathlib.Path(argv[1]) + found = sorted((capture_root() / "converted").glob("generated-*")) + if len(found) != 1: + sys.exit( + f"expected exactly one converted site under {capture_root()}/converted, found {len(found)}" + + "".join(f"\n {p}" for p in found) + + "\npass one as the first argument" + ) + return found[0] + + +def main(root: Path, apply: bool): + stats = {"files": 0, "keys_before": 0, "keys_after": 0, "no_title": []} + + data = root / "data" + if data.exists(): + n = sorted(p.name for p in data.iterdir()) + print(f"data/ to delete: {n}") + if apply: + shutil.rmtree(data) + + for p in sorted(root.joinpath("content").rglob("*.md")): + raw = p.read_text(encoding="utf-8") + m = FM.match(raw) + if not m: + continue + fm = yaml.safe_load(m.group(1)) or {} + body = m.group(2) + stats["files"] += 1 + stats["keys_before"] += len(fm) + + out = {k: fm[k] for k in KEEP if k in fm and fm[k] not in (None, "", [])} + if not out.get("title"): + # Recover a title from the slug rather than shipping an untitled page. + slug = p.parent.name if p.name == "index.md" else p.stem + out["title"] = slug.replace("-", " ").title() + out = {"title": out["title"], **{k: v for k, v in out.items() if k != "title"}} + stats["no_title"].append(f"{p.relative_to(root)} -> {out['title']!r}") + stats["keys_after"] += len(out) + + # sort_keys=False preserves the KEEP order, which reads better than alphabetical. + new = "---\n" + yaml.dump(out, sort_keys=False, allow_unicode=True, width=10**6) + "---\n" + body + if apply: + p.write_text(new, encoding="utf-8") + + print(f"\nfiles processed : {stats['files']}") + print(f"front-matter keys: {stats['keys_before']} -> {stats['keys_after']}" + f" ({stats['keys_before'] - stats['keys_after']} removed)") + if stats["no_title"]: + print(f"\ntitle recovered from slug ({len(stats['no_title'])}):") + for t in stats["no_title"]: + print(" ", t) + print("\nAPPLIED" if apply else "\nDRY RUN - pass --apply to write") + + +if __name__ == "__main__": + main(converted_site(sys.argv), "--apply" in sys.argv) diff --git a/capture/enumerate-media.py b/capture/enumerate-media.py new file mode 100755 index 0000000..f62adbe --- /dev/null +++ b/capture/enumerate-media.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Enumerate every media URL referenced by the live WordPress.com site. + +Pulls all posts and pages from the old platform's REST API, named by CAPTURE_SOURCE_API, +and extracts +every media reference from the rendered HTML. The rendered form is deliberate: +shortcodes are already expanded, so this sees what a reader's browser sees. + +The critical output is the by-host breakdown. Media on the wp.com subdomain is in +the media library and will appear in the WXR export; media on lh*.ggpht.com and +googleusercontent.com is hotlinked from the Windows Live Writer era, is NOT in the +library, and will NOT be in the export. That subset is the migration's biggest +silent-data-loss risk, so it gets enumerated explicitly. + +Writes: media-urls.tsv (host, url, post_id, post_url), posts.json (raw archive). +""" + +import json +import os +import re +import sys +import urllib.parse +import urllib.request +from collections import Counter, defaultdict +from pathlib import Path + +def env(name: str) -> str: + """A required capture value, refused rather than guessed when unset. + + Duplicated across the scripts here rather than shared, because these get copied out to + a scratch directory to run, and an import would break the moment one travelled alone. + """ + v = os.environ.get(name, "") + if not v: + sys.exit(f"{name} is not set -- see example.env and ENVIRONMENT.md") + return v + + +API = env("CAPTURE_SOURCE_API") +# Written into the capture, never beside this script. Anchoring on __file__ would put the +# inventory inside the repository the moment this file moved into it, which it now has. +OUT = Path(env("CAPTURE_ROOT")) / "inventory" + +# src/href targets that are media rather than navigation. +MEDIA_EXT = re.compile( + r"\.(?:jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|mp4|m4v|mov|webm|mp3|m4a|wav|ogg|pdf|zip|7z|txt|csv|xlsx?|docx?)" + r"(?:[?#]|$)", + re.IGNORECASE, +) +# Attributes that can carry a media URL, including responsive-image sets. +ATTR = re.compile(r"""(?:src|href|data-orig-file|data-large-file|data-medium-file|poster)\s*=\s*["']([^"']+)["']""", re.I) +SRCSET = re.compile(r"""srcset\s*=\s*["']([^"']+)["']""", re.I) + + +def fetch(url): + req = urllib.request.Request(url, headers={"User-Agent": "blog-migration-audit/1.0"}) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + + +def all_items(kind, extra=""): + """Page through a collection until exhausted. + + Pages go through here too, with type=page, rather than a single number=100 request. + That request was correct for this site and silently wrong for any site with more than + a hundred pages, which is the shape of loss this whole capture exists to catch. + """ + items, page = [], 1 + while True: + data = fetch(f"{API}/{kind}/?number=100&page={page}{extra}&fields=ID,URL,slug,title,date,content") + found = data.get("posts", []) + if not found: + break + items.extend(found) + if len(items) >= data.get("found", 0): + break + page += 1 + return items + + +def extract(html): + urls = set() + for m in ATTR.finditer(html or ""): + urls.add(m.group(1).strip()) + for m in SRCSET.finditer(html or ""): + for candidate in m.group(1).split(","): + part = candidate.strip().split() + if part: + urls.add(part[0]) + return {u for u in urls if MEDIA_EXT.search(u)} + + +def main(): + posts = all_items("posts") + pages = all_items("posts", extra="&type=page") + everything = posts + pages + print(f"fetched {len(posts)} posts + {len(pages)} pages", file=sys.stderr) + + OUT.mkdir(parents=True, exist_ok=True) + (OUT / "posts.json").write_text(json.dumps(everything, indent=1), encoding="utf-8") + + rows, by_host, per_post = [], Counter(), defaultdict(set) + for item in everything: + for u in extract(item.get("content", "")): + absolute = urllib.parse.urljoin(item["URL"], u) + host = urllib.parse.urlparse(absolute).netloc.lower() + by_host[host] += 1 + per_post[item["ID"]].add(absolute) + rows.append((host, absolute, str(item["ID"]), item["URL"])) + + with (OUT / "media-urls.tsv").open("w", encoding="utf-8") as fh: + fh.write("host\turl\tpost_id\tpost_url\n") + for row in sorted(set(rows)): + fh.write("\t".join(row) + "\n") + + unique = {r[1] for r in rows} + print(f"\n{len(rows)} media references, {len(unique)} unique URLs\n") + print(f"{'host':45} refs") + for host, n in by_host.most_common(): + print(f"{host:45} {n}") + + # The at-risk subset, called out explicitly. + risky = sorted(u for u in unique if re.search(r"(ggpht|googleusercontent)\.com", u)) + (OUT / "media-external-hotlinked.txt").write_text("\n".join(risky) + "\n", encoding="utf-8") + print(f"\nNOT in the media library (absent from any WXR export): {len(risky)} unique URLs") + print(" -> inventory/media-external-hotlinked.txt") + + +if __name__ == "__main__": + main() diff --git a/capture/localize-external.py b/capture/localize-external.py new file mode 100755 index 0000000..bfc8f12 --- /dev/null +++ b/capture/localize-external.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Download every externally-hosted image and rewrite the content to point at local copies. + +wp2hugo skips these entirely ("non-relative link (skipped for download)"), and they are +absent from the WordPress export too, because they were never in the media library - they +are hotlinks from the Windows Live Writer and Blogger era. If they are not localized they +remain a permanent dependency on Google serving 15-year-old URLs. + +Two traps this handles, both of which return HTTP 200 and look like success: + +1. **The `-h` wrapper.** A Picasa/ggpht URL whose size segment ends in `-h` (`/s1600-h/`) + serves an HTML *page* containing an `` tag, not the image. Fetching it naively + yields a 400-byte HTML file with a 200 status. The fix is to parse the wrapper and + follow the `` it names, which is Google's own answer rather than a guess. +2. **Non-image bodies generally.** Anything whose magic bytes are not an image is a + failure regardless of status code, and is reported rather than written. + +Writes images to static/external/ named by a hash of the source URL (these URLs carry +percent-encoded brackets and other characters that do not survive a filesystem), records +the mapping in external-media-map.tsv for auditability, and rewrites every reference. +""" + +import hashlib +import pathlib +import os +import re +import sys +import urllib.error +import urllib.request +from collections import Counter + +UA = {"User-Agent": "Mozilla/5.0 (compatible; blog-migration-audit/1.0)"} +EXT_HOST = re.compile(r"^https?://[a-z0-9.-]*\.(?:ggpht|googleusercontent)\.com/", re.I) +URL_IN_CONTENT = re.compile(r"https?://[a-z0-9.-]*\.(?:ggpht|googleusercontent)\.com/[^\s\"'\)\]<>]+", re.I) +IMG_IN_WRAPPER = re.compile(rb' pathlib.Path: + """CAPTURE_ROOT, refused rather than guessed when unset. + + Duplicated in each script here rather than shared, because these get copied out to a + scratch directory to run against a copy of the capture, and an import would break the + moment one of them travelled alone. + """ + root = os.environ.get("CAPTURE_ROOT", "") + if not root: + sys.exit("CAPTURE_ROOT is not set -- see example.env and ENVIRONMENT.md") + return pathlib.Path(root) + + +def converted_site(argv) -> pathlib.Path: + """The converted site: a first argument, else the one generated-* under the capture. + + Ambiguity aborts rather than picking, the same rule build-redirects.py applies to the + export. wp2hugo stamps the directory with a run timestamp, so there is no fixed name + to default to and a glob that matched two would otherwise choose by filesystem order. + """ + if len(argv) > 1 and not argv[1].startswith("--"): + return pathlib.Path(argv[1]) + found = sorted((capture_root() / "converted").glob("generated-*")) + if len(found) != 1: + sys.exit( + f"expected exactly one converted site under {capture_root()}/converted, found {len(found)}" + + "".join(f"\n {p}" for p in found) + + "\npass one as the first argument" + ) + return found[0] + + +def sniff(data: bytes): + for sig, ext in MAGIC: + if data.startswith(sig): + return ext + if data[:4] == b"RIFF" and data[8:12] == b"WEBP": + return ".webp" + return None + + +def fetch(url: str, depth: int = 0): + """Return (bytes, ext) or (None, reason). Follows one level of `-h` HTML wrapper.""" + try: + req = urllib.request.Request(url, headers=UA) + with urllib.request.urlopen(req, timeout=45) as r: + body = r.read() + except urllib.error.HTTPError as e: + return None, f"HTTP {e.code}" + except Exception as e: # noqa: BLE001 - network is the expected failure here + return None, type(e).__name__ + + ext = sniff(body) + if ext: + return body, ext + + # Not an image. If it is Google's `-h` wrapper page, it names the real image. + # A wrapper page is HTML, and HTML does not reliably start with . A doctype, a + # comment or leading whitespace are all ordinary, and requiring the tag meant a wrapper + # that opened with was reported as "not an image" rather than followed. + head = body.lstrip()[:64].lower() + if depth == 0 and (head.startswith(b" /2024/05/29/slug/ + +The `url:` front matter is the authority, not the `date:` field - it is WordPress's own +permalink, and the two can legitimately disagree if a post was ever re-dated. + +Standalone pages move to the content root, which is Hugo's idiomatic place for them: + + content/pages/2012/07/about/index.md -> content/about.md -> /about/ + +They were nested under a date only because wp2hugo's date-folder option applies to pages +as well as posts, which is meaningless for a page. A `content/pages/` section would also +make Hugo publish a `/pages/` listing URL, the same unwanted extra as `/posts/`. +""" + +import collections +import os +import pathlib +import re +import shutil +import sys + +FM = re.compile(r"^---\n(.*?)\n---\n", re.S) +URL = re.compile(r"^url:\s*(\S+)\s*$", re.M) +DATED = re.compile(r"^/(\d{4})/(\d{2})/(\d{2})/([^/]+)/$") + + +def capture_root() -> pathlib.Path: + """CAPTURE_ROOT, refused rather than guessed when unset. + + Duplicated in each script here rather than shared, because these get copied out to a + scratch directory to run against a copy of the capture, and an import would break the + moment one of them travelled alone. + """ + root = os.environ.get("CAPTURE_ROOT", "") + if not root: + sys.exit("CAPTURE_ROOT is not set -- see example.env and ENVIRONMENT.md") + return pathlib.Path(root) + + +def converted_site(argv) -> pathlib.Path: + """The converted site: a first argument, else the one generated-* under the capture. + + Ambiguity aborts rather than picking, the same rule build-redirects.py applies to the + export. wp2hugo stamps the directory with a run timestamp, so there is no fixed name + to default to and a glob that matched two would otherwise choose by filesystem order. + """ + if len(argv) > 1 and not argv[1].startswith("--"): + return pathlib.Path(argv[1]) + found = sorted((capture_root() / "converted").glob("generated-*")) + if len(found) != 1: + sys.exit( + f"expected exactly one converted site under {capture_root()}/converted, found {len(found)}" + + "".join(f"\n {p}" for p in found) + + "\npass one as the first argument" + ) + return found[0] + + +def url_of(p: pathlib.Path): + m = FM.match(p.read_text(encoding="utf-8")) + if not m: + return None + u = URL.search(m.group(1)) + return u.group(1).strip().strip("\"'") if u else None + + +def main(site: pathlib.Path, apply: bool): + content = site / "content" + moves, problems = [], [] + + # --- posts: content/posts///.md -> content/posts////.md + for p in sorted((content / "posts").rglob("*.md")): + # A section index is Hugo structure rather than a post. It carries no url: because + # it is not served, and treating it as unplaceable would make a healthy tree fail. + if p.name.startswith("_index."): + continue + u = url_of(p) + if not u: + problems.append((p, "no url: front matter")) + continue + m = DATED.match(u) + if not m: + problems.append((p, f"url not /Y/M/D/slug/: {u}")) + continue + y, mo, d, slug = m.groups() + dest = content / "posts" / y / mo / d / f"{slug}.md" + if dest != p: + moves.append((p, dest)) + + # --- pages: anywhere under content/pages -> content/.md + for p in sorted((content / "pages").rglob("*.md")): + u = url_of(p) + if not u: + problems.append((p, "no url: front matter")) + continue + slug = u.strip("/").split("/")[-1] + moves.append((p, content / f"{slug}.md")) + + print(f"posts and pages to relocate: {len(moves)}") + for src, dst in moves[:4]: + print(f" {src.relative_to(content)}\n -> {dst.relative_to(content)}") + if problems: + print(f"\nPROBLEMS ({len(problems)}):") + for p, why in problems: + print(f" {p.relative_to(content)}: {why}") + + # shutil.move replaces an existing destination without a word, and this applies moves in + # bulk, so a collision would destroy a post and report success. Two ways it can happen: + # two sources resolving to one destination, which flattening pages to content/.md + # makes possible, and a destination that already exists from a half-finished earlier run. + # Both are refused before anything moves, rather than discovered half way through. + collisions = collections.Counter(dst for _, dst in moves) + clashes = sorted(d for d, n in collisions.items() if n > 1) + occupied = sorted(dst for src, dst in moves if dst.exists() and dst != src) + if clashes or occupied: + print(f"\nREFUSING TO MOVE ({len(clashes)} collisions, {len(occupied)} occupied destinations):") + for d in clashes: + print(f" {d.relative_to(content)} <- {sum(1 for _, x in moves if x == d)} sources") + for d in occupied: + print(f" {d.relative_to(content)} already exists") + return 1 + + if apply: + for src, dst in moves: + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(src), str(dst)) + # Remove directories left empty by the move. + for _ in range(6): + for d in sorted(content.rglob("*"), key=lambda x: -len(x.parts)): + if d.is_dir() and not any(d.iterdir()): + d.rmdir() + print("\nAPPLIED") + else: + print("\nDRY RUN - pass --apply") + + # A problem is a file this cannot place: no url: front matter, or a url that is not the + # dated shape. The valid moves still happen, because leaving them undone helps nobody, + # but the run is incomplete and must not report success. Contrast clean-content.py, + # whose recovered titles are a repair rather than a skip and correctly exit zero. + if problems: + print(f"{len(problems)} file(s) could not be placed, listed above") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(converted_site(sys.argv), "--apply" in sys.argv)) diff --git a/capture/run-wp2hugo.sh b/capture/run-wp2hugo.sh new file mode 100755 index 0000000..175c4b6 --- /dev/null +++ b/capture/run-wp2hugo.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Convert the WordPress export to Hugo. Media is downloaded here because that is what +# drives absolute->relative URL rewriting, but the bytes are replaced afterwards from the +# official media tar - WordPress.com serves optimized derivatives over HTTP for some +# images, one of them at a fraction of the original's dimensions. +# +# The export is chosen by build-redirects.py --print-export rather than by a glob here, so +# the conversion and the redirect maps are provably built from the same file. An account +# holds several exports and a media-only one carries the attachments and no posts, and +# converting that one yields a site that builds and is empty. The selection cannot be +# reimplemented in shell: the test is that ONE item carries both post_type=post and +# status=publish, where two greps over a whole file would accept the media-only export. +set -Eeuo pipefail + +# A `go install`ed wp2hugo lands here. $HOME rather than a literal path. +export PATH="$HOME/.local/bin:$PATH" + +: "${CAPTURE_ROOT:?CAPTURE_ROOT is not set -- see example.env and ENVIRONMENT.md}" +[ -d "$CAPTURE_ROOT" ] || { + echo "CAPTURE_ROOT is not a directory: $CAPTURE_ROOT" >&2 + exit 1 +} +# Resolved to absolute before anything else, because this script cd's into it and the +# export path is worked out beforehand. A relative CAPTURE_ROOT would yield a relative +# export path that stops resolving the moment the cd happens, which reads as a missing +# export rather than as a path problem. +CAPTURE_ROOT="$(cd "$CAPTURE_ROOT" && pwd)" +export CAPTURE_ROOT + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export_xml="$("$here/build-redirects.py" --print-export)" +echo "==> export: $export_xml" + +cd "$CAPTURE_ROOT" +wp2hugo \ + --source "$export_xml" \ + --output converted \ + --download-media \ + --download-all \ + --continue-on-media-download-error \ + --content-date-folder-structure year-month \ + --color-log-output=false +echo "WP2HUGO-EXIT-OK" diff --git a/checks/README.md b/checks/README.md index 89d866a..1b30119 100644 --- a/checks/README.md +++ b/checks/README.md @@ -54,24 +54,38 @@ An **attachment page** is the page the old platform generated per uploaded image ## Legacy URL shapes worth knowing -Two properties of the maps are non-obvious and easy to break when regenerating them. - -**`blogger.map` carries 59 entries for 48 posts.** Blogger truncated an auto-generated slug at 40 characters on a whole-word boundary, so for the 11 posts with a longer slug the URL actually served, and therefore the one in search indexes and in other people's links, is the truncated form. Both forms are live redirects. A map holding only the full slug keeps the URL that never existed and drops the one that did. - **`/search/label/