From aed71c4034224c51b3ae2bb51a27231a9d6c1013 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sun, 2 Aug 2026 12:57:54 -0700 Subject: [PATCH 01/12] Add the merge-bot workflow (#38) Dependabot opens pull requests against both branches here, and nothing merged them, so every dependency bump waited on a manual merge and the action pins went stale between sweeps. Carry the two jobs the contract requires. The fleet canonical also carries merge-codegen and merge-upstream-version, and this repository runs neither, so both are left out rather than carried as conditions that can never match. The App-token secrets the workflow needs are already configured in both the Actions and Dependabot stores. Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/merge-bot-pull-request.yml | 95 ++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 .github/workflows/merge-bot-pull-request.yml diff --git a/.github/workflows/merge-bot-pull-request.yml b/.github/workflows/merge-bot-pull-request.yml new file mode 100644 index 0000000..4ee4f25 --- /dev/null +++ b/.github/workflows/merge-bot-pull-request.yml @@ -0,0 +1,95 @@ +name: Merge bot pull request action + +# Auto-merges in-repo bot pull requests: enable on opened/reopened, disable on a maintainer push. +# - Merge method by base: develop = squash, main = merge. +# - App token, not GITHUB_TOKEN: it fires downstream workflows and can write on a Dependabot PR. +# - pull_request_target, not pull_request: jobs hold the App key. +# The workflow and action SHAs then resolve from the trusted base, not the pull request head. +# No job checks out pull request code, since each runs gh pr merge by URL. +# +# The fleet canonical also carries merge-codegen and merge-upstream-version jobs. +# This repository runs neither codegen nor an upstream-version tracker, so both are N/A here. +# They are left out rather than carried as conditions that can never match. +on: + pull_request_target: + types: [opened, reopened, synchronize] + +# Concurrency keys on the pull request number rather than on github.ref. +# Under pull_request_target that ref is the base branch, serializing every bot pull request. +# Cancelling is off, so a follow-up synchronize cannot kill an opened run mid-enable. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: false + +jobs: + + merge-dependabot: + name: Merge dependabot pull request job + runs-on: ubuntu-latest + # Dependabot pull requests from this repository, not forks. + # Only on opened or reopened, so the disable job stays sticky. + if: >- + (github.event.action == 'opened' || github.event.action == 'reopened') && + github.event.pull_request.user.login == 'dependabot[bot]' && + github.event.pull_request.head.repo.full_name == github.repository + permissions: + contents: write + pull-requests: write + + steps: + + - name: Generate GitHub App token step + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ secrets.CODEGEN_APP_CLIENT_ID }} + private-key: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }} + + # Auto-merge every tier, semver-major included. + # The required checks are the gate, not the size of the bump. + - name: Merge pull request step + run: | + set -Eeuo pipefail + case "${{ github.event.pull_request.base.ref }}" in + develop) method=--squash ;; + main) method=--merge ;; + *) + echo "::error::Unsupported base branch: ${{ github.event.pull_request.base.ref }}" + exit 1 + ;; + esac + gh pr merge --auto "$method" "$PR_URL" + env: + PR_URL: ${{ github.event.pull_request.html_url }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + + disable-auto-merge-on-maintainer-push: + name: Disable auto-merge on maintainer push job + runs-on: ubuntu-latest + # Fires when a maintainer pushes to a bot's branch, meaning synchronize with a non-bot actor. + # Disables auto-merge so the maintainer's commits do not merge with the bot's. + # They re-enable it by hand. + # The disable call is idempotent. + if: >- + github.event.action == 'synchronize' && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login == 'dependabot[bot]' && + github.actor != github.event.pull_request.user.login + permissions: + pull-requests: write + + steps: + + - name: Generate GitHub App token step + # App token, because a Dependabot pull request's GITHUB_TOKEN is read-only whoever triggered it. + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ secrets.CODEGEN_APP_CLIENT_ID }} + private-key: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }} + + - name: Disable auto-merge step + run: gh pr merge --disable-auto "$PR_URL" + env: + PR_URL: ${{ github.event.pull_request.html_url }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} From 5aa4b131ae507c2bd50e3a0b9569dd835e06040c Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sun, 2 Aug 2026 14:41:58 -0700 Subject: [PATCH 02/12] Add OPERATIONS.md (#37) * Add OPERATIONS.md The hub requires this file of every repo, checked for presence only, so its content is entirely this repository's own. It exists because the convention was emerging unevenly and the same operational material was landing under ad-hoc names. Carry the five declared headings with real content rather than a stub: the local gates that mirror CI, the dispatch-driven release, the runtime dependency verify has on Docker and on a database of its own, what the three exit codes distinguish, and why exiftool validation warnings are logged at debug level only. Co-Authored-By: Claude Opus 5 (1M context) * Say how the CI invocation differs from the local one The runbook claimed CI runs the same commands, and it does not: locally csharpier formats and writes, while CI runs it in check mode and only verifies. A reader following the text would expect CI to fix formatting. Co-Authored-By: Claude Opus 5 (1M context) * Describe the commands this branch actually has The file documented the verify command, the exiftool -validate flag, and a per-file-failure exit code of 2. None of those exist on develop: they are on the unmerged verify branch, and this branch was cut from develop. Describe what is here instead. Exit codes are 0 and 1, a per-file failure does not change them, exiftool is invoked without -validate, and the application needs no Docker daemon at runtime. The verify operational content follows once that work merges. Co-Authored-By: Claude Opus 5 (1M context) * Restore the line endings on OPERATIONS.md Co-Authored-By: Claude Opus 5 (1M context) * Name the two places CI differs, and the lint step the snippet omitted The text claimed CI runs dotnet test unchanged, and it adds coverage collection so coverlet can emit the report Codecov consumes. The lint snippet listed three of the four Lint tasks, leaving out actionlint, so following it ran less than the local surface it claimed to match. Co-Authored-By: Claude Opus 5 (1M context) * Say that the local linters and CI's are the same rules, not the same builds The local commands pull :latest while CI reaches three of the four through SHA-pinned wrappers, so a local result can differ from CI once an upstream release lands ahead of the pin. Name CI as authoritative and the difference as a version gap. Co-Authored-By: Claude Opus 5 (1M context) * Recommend the key file rather than naming it the only mechanism The CLI accepts an inline --apikey too, and the two are mutually exclusive by validator, so stating the file form as the only one was wrong. Recommend it and say why. Drop the trailing claim that a local Docker linter run matches what CI resolved, which contradicted the paragraph above it saying local pulls latest while CI reaches three of the four through pinned wrappers. Co-Authored-By: Claude Opus 5 (1M context) * Name the parse error among the exit-1 causes BypassStartup short-circuits when the parse result carries errors, so a command-line mistake exits 1 before any work starts. The list named only cancellation and unhandled exceptions. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- OPERATIONS.md | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 OPERATIONS.md diff --git a/OPERATIONS.md b/OPERATIONS.md new file mode 100644 index 0000000..22adc3d --- /dev/null +++ b/OPERATIONS.md @@ -0,0 +1,87 @@ +# Operations + +How this repository is run. It ships a .NET console application and a multi-architecture Docker image, so its operations are the local gates that mirror CI, the release pipeline, and the external tools the application drives at runtime. + +## Runbooks + +### Run the gates the way CI runs them + +Local and CI runs read the same committed configuration, but they invoke it differently: locally the formatter writes, and in CI it only verifies. The [`.NET Format`](./.vscode/tasks.json) task is the local clean-compile chain, meaning `dotnet csharpier format`, then `dotnet build`, then the style verify. Run the chain and the suite before committing, since the chain never runs the tests and a change that compiles and formats cleanly can still be broken: + +```sh +dotnet csharpier format --log-level=debug . +dotnet build +dotnet format style --verify-no-changes --severity=info --verbosity=detailed +dotnet test +dotnet husky run +``` + +CI differs in two places. It substitutes `dotnet csharpier check .` for the format step, after a `dotnet tool restore`, and it runs the suite as `dotnet test --collect:"XPlat Code Coverage" --results-directory ./coverage` so coverlet emits the report the Codecov upload consumes. The style verify is identical. So a local run that formats a file leaves CI clean, while an unformatted commit fails there rather than being fixed. + +The lint set runs in containers, matching the `Lint:` tasks in [`.vscode/tasks.json`](./.vscode/tasks.json): + +```sh +docker run --rm --pull=always -v "$PWD":/workdir -w /workdir davidanson/markdownlint-cli2:latest "**/*.md" +docker run --rm --pull=always -v "$PWD":/workdir -w /workdir ghcr.io/streetsidesoftware/cspell:latest --no-progress README.md HISTORY.md +docker run --rm --pull=always -v "$PWD":/repo -w /repo rhysd/actionlint:latest -color +docker run --rm --pull=always -v "$PWD":/check -w /check mstruebing/editorconfig-checker:latest +``` + +Those four are the whole local lint surface, matching the four `Lint:` tasks. CI checks the same four rules from the same committed configuration, but not through the same tools: it reaches markdownlint, cspell and actionlint through SHA-pinned action wrappers, and only editorconfig-checker runs as a container there. The local commands pull `:latest` deliberately, so a local result can legitimately differ from CI once an upstream release lands ahead of the pinned wrapper. Treat CI as authoritative when the two disagree, and read the difference as a version gap rather than a rule change. + +The prose gate lives in the hub rather than here, so it is consumed from a hub checkout and reads only the lines a change touches: + +```sh +python3 [path-to-hub]/scripts/prose_lint.py . --diff origin/develop +``` + +### Cut a release + +Publishing never happens as a side effect of a merge. A release is a `workflow_dispatch` of [`publish-release.yml`](./.github/workflows/publish-release.yml), and the same workflow runs on a weekly schedule so the image picks up base-image and tool updates. Merging to `main` publishes nothing on its own. + +## Backup and Recovery + +The repository is the record, and GitHub holds it. Nothing here keeps state outside git. + +The application writes state the user owns rather than the repository: the SQLite databases named by `--db` and `--trashdb`, and the `.bak` files that `process` creates. `undo` restores those `.bak` files, and it is the recovery path for a processing run that went wrong. Running `process --skipbackup` makes that recovery impossible, which is the trade the flag names. + +A deleted branch is recoverable from any full clone that still has the commit: + +```sh +git push origin [sha]:refs/heads/[branch] +``` + +Never use `--depth 1` on a clone that will amend or force-push, because a shallow clone severs the merge base and orphans the branch. + +## Logs and Debugging + +Workflow runs are the CI log. `gh run list --branch [branch]` and `gh run view [id] --log-failed` reach them, and a local gate above reproduces a CI failure exactly, so reproduce locally before reading workflow logs. + +The application logs to the console and to the file named by `--logfile`. Raise the level with `--loglevel debug` when a file is rejected and the reason is not obvious. + +A calling script branches on the exit code rather than on output: + +- `0`: the command ran to completion. +- `1`: the command could not run, meaning a command-line parse or validation error, a cancellation, or an unhandled exception. A parse error short-circuits before any work starts, so nothing was touched. + +Note that a per-file failure does not currently change the exit code, so `0` means the command finished rather than that every file succeeded. Read the log to tell those apart. + +## Tool Usage + +The application shells out to external tools rather than reimplementing them, so their versions decide its behavior: + +- **exiftool** reads and writes metadata, invoked through `MediaUtilities.GetExifToolJsonAsync`. It is installed in the Docker image, and a native run needs it on `PATH`. +- **ffmpeg** handles video, and is installed in the image alongside exiftool. + +The application itself needs no Docker daemon at runtime. Docker here is a packaging and tooling concern only, meaning the shipped image and the containerized linters above. + +The Immich API key can be given inline with `--apikey` or read from a file with `--apikey-file`, and the two are mutually exclusive. Prefer the file: an inline key lands in shell history and is visible in the process list for as long as the command runs. + +## Configuration Layout + +- [PhotoCleaner/](./PhotoCleaner/) is the console application. +- [PhotoCleanerTests/](./PhotoCleanerTests/) is the xUnit suite, and [PhotoCleanerBenchmarks/](./PhotoCleanerBenchmarks/) is the benchmark project. +- [Docker/](./Docker/) holds the multi-architecture `Dockerfile` and the Docker Hub README. +- [repo-config/](./repo-config/) holds the branch rulesets and the apply script. It sits outside `.github/`, which is Actions-owned. +- [.github/workflows/](./.github/workflows/) holds the CI and release pipelines, with `validate-task.yml` as the single validation gate that the pull request check and the release both call. +- Analyzer and package configuration is central: `Directory.Build.props` carries the analyzer set, and `Directory.Packages.props` pins every package version. From 4a7403d205777a479368d8610c4bac7c7200a2c8 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sun, 2 Aug 2026 15:42:54 -0700 Subject: [PATCH 03/12] Consume the fleet prose rules from the hub (#36) * Consume the fleet prose rules from the hub The prose rules that govern this repository live in the hub, and until now nothing here ran them, so a comment or a sentence breaking a documented rule reached main with every linter green. Consume the hub's composite action rather than vendoring its checker, so a rule change lands in one place instead of in every repository holding a copy. A develop-targeted run reads the rules from hub develop, so an unpromoted change is exercised here before it is promoted, and every other run uses the copy bundled at the pinned commit so a released build stays reproducible. The gate reports only lines a change touches, so the repository's existing prose backlog blocks nothing and is corrected as each file is next edited. Checkout gains full history in the lint job, because diffing against the base branch needs that branch present. Co-Authored-By: Claude Opus 5 (1M context) * Point the prose gate at the branch it merges into The gate runs on push, because this repository has no pull_request trigger, so there is no event base to read and the first run diffed against an empty ref. Unresolvable, it reported the whole repository instead of the lines this change touches. Name the base explicitly as the branch being merged into, and skip main, which only receives promotion merges already gated on develop. Co-Authored-By: Claude Opus 5 (1M context) * Skip the prose gate on a publish run A publish reaches validate-task through build-release-task, where the content was already gated when it was pushed. Re-reading it there would diff the whole unpromoted delta against main and could fail a release on prose that already passed. Co-Authored-By: Claude Opus 5 (1M context) * Cut the publish-run comment to one sentence per line Co-Authored-By: Claude Opus 5 (1M context) * Describe what this workflow does, and mark the pin temporary The comment described the action's main-run behavior, which this workflow never reaches because it skips main outright. A maintainer reading it would assume a prose gate runs on main. The pin targets an unmerged hub commit, and that intent lived only in the pull request description, which does not survive the merge. State it inline instead, including that Dependabot cannot bump a pin resolving to no tag, so the repoint is manual. Co-Authored-By: Claude Opus 5 (1M context) * Only pay for full history on the runs that use it The lint job always full-cloned, including publish runs where the prose gate is skipped, which is time and network a release does not need. Tie the fetch depth to the same condition the gate uses, so the two cannot drift apart. Repoint the pin to the hub branch head, which carries the fix for a multi-line paths input that scanned only its first entry. Co-Authored-By: Claude Opus 5 (1M context) * Quote the fetch depths so the true branch is not falsy An unquoted 0 is falsy in an Actions expression, so the ternary collapsed to 1 whenever the condition held and every run shallow-cloned, leaving the prose gate no base branch to diff against. Co-Authored-By: Claude Opus 5 (1M context) * Cut the wrapped comment sentences to one per line Co-Authored-By: Claude Opus 5 (1M context) * Repoint the prose gate at the hub commit that carries it The pin named the head of an unmerged hub branch, which would have gone unreachable once that branch was squashed and deleted, breaking this repository's gate later with nothing here to explain it. ptr727/ProjectTemplate#520 has landed, so point at the commit on hub develop instead. It still carries no release tag, so Dependabot cannot bump it yet and the note says so. Co-Authored-By: Claude Opus 5 (1M context) * Describe the pin as a SHA rather than as a branch A pin that calls itself develop stops being true the moment develop moves. Name what it contains instead. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/validate-task.yml | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index 626e12b..2c90e16 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -58,6 +58,16 @@ jobs: - name: Checkout code step uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + # The prose gate diffs against the base branch, so it needs that branch's history. + # Only the runs that reach the gate pay for it, since a publish run skips the gate. + # A full clone there is time and network a release does not need. + # The condition matches the gate's own below, so the two cannot drift apart silently. + # + # The depths are quoted, and must stay quoted. + # An unquoted 0 is falsy, so `cond && 0 || 1` collapses to 1 whenever cond is true. + # Every run would then shallow-clone, leaving the gate no base to diff against. + fetch-depth: ${{ (github.event_name == 'push' && github.ref_name != 'main') && '0' || '1' }} - name: Check C# formatting step run: | @@ -86,3 +96,26 @@ jobs: - name: Check EditorConfig step run: docker run --rm -v "$PWD":/check --workdir /check mstruebing/editorconfig-checker:latest + + # The fleet prose rules live in the hub, so this repo consumes them rather than vendoring them. + # The rules are read from hub develop, so a rule change is exercised here before promotion. + # Only lines a change touches are reported, so the existing backlog blocks nothing. + # + # The base is the branch this one merges into. + # The gate runs on push, so there is no pull_request event to read a base from. + # Runs on main are skipped, since it only receives promotion merges already gated on develop. + # + # A publish run reaches this workflow too, through build-release-task, and is skipped. + # Its content was already gated when it was pushed. + # Re-reading it would diff the whole unpromoted delta and could fail a release on prose. + - name: Check prose step + if: ${{ github.event_name == 'push' && github.ref_name != 'main' }} + # A SHA pin, containing the prose gate from ptr727/ProjectTemplate#520. + # It was hub develop when pinned, and says nothing about where develop is now. + # A pin that claims to be a branch goes stale the moment that branch moves. + # It carries no release tag yet, so Dependabot cannot compare it and will not bump it. + # Repoint it by hand at the next hub release. + # Dependabot tracks it like any other action pin from then on. + uses: ptr727/ProjectTemplate/.github/actions/prose-gate@cbd5eb3c9e079b640d5461400d89b390de2f0780 # hub develop + with: + base: origin/${{ github.ref_name == 'develop' && 'main' || 'develop' }} From 39c896b55c36101a58332e0bcfd8ed70a51cec2f Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 3 Aug 2026 08:32:57 -0700 Subject: [PATCH 04/12] Add a verify command that confirms Immich can render each file (#35) * Add a verify command and always-on exiftool validation Adds `verify`, a standalone pipeline step that answers whether Immich can generate a preview for each file, and turns on exiftool `-validate` in the metadata read that `process` and `import` already perform. Fixes #25, where ~8,200 HEIC files were byte-complete, passed every existing check, uploaded successfully, and then failed thumbnail generation forever. An `iloc` box re-encoded from version 1 to version 0 loses `construction_method`, so the `grid` descriptors Apple stores in `idat` are read as absolute file offsets and land on the `ftyp` header. That incident is only one of three classes. Two more are open against Immich: DNGs libraw cannot parse, and RAW from cameras it does not yet recognize. Neither is corruption, so no parser can predict them - only running the decoder Immich runs can. `verify` therefore has two passes: - A structural ISO-BMFF box walk, in-process and free, reporting extents past EOF, an `iloc` v0 alongside an `idat` box, and `grid` geometry that disagrees with the `dimg` reference count. `--quick` stops here and needs no Docker. - A decode pass that calls Immich's own compiled `MediaRepository`, `defaults`, and `ThumbnailConfig` inside the `immich-server` image, batching paths over stdin. Calling Immich's code rather than reimplementing its pipeline means behavior tracks Immich across releases. A preflight runs before any file is judged, so an unreachable Docker or a missing image exits Error rather than condemning the collection. Measured on a real collection, `-validate` costs nothing on top of the existing exiftool call, but ~75% of healthy files carry warnings, so only an error count fails a file; warnings are logged at debug level. Commands now share one exit-code contract: 0 success, 1 could not run, 2 ran to completion with per-file failures. This is a behavior change - `process` previously exited 0 while logging failed files. Co-Authored-By: Claude Opus 5 (1M context) * Document the verify database behavior No behavior change. The verified state is a single bit rather than a record of which check produced it, and two consequences of that were undocumented: - A --quick run after a full run skips every file the full run passed, so it can legitimately do nothing. Correct, since a full run subsumes the structural checks, but surprising without a note. --reprocess forces it. - The bit lives in the same is_processed column that process writes, so pointing verify --db at a Process.db silently skips every file as "already verified" when they were only processed. Nothing detects this, so the requirement to use a separate Verify.db is now stated rather than implied. Co-Authored-By: Claude Opus 5 (1M context) * Bring comments in the verify code to house style The verify code was written with XML doc blocks on internal members and multi-sentence prose comments, matching neither the documented rules nor the surrounding code. GOVERNANCE.md bans class/type/file header summary blocks, calls for one sentence per line with no wrapping, and defaults to a single line. CODESTYLE.md scopes XML documentation to public surfaces, and everything here is internal. The pre-existing PhotoCleaner sources carry two XML summaries in total. Removes every XML doc block from the new files, collapses the header blocks to the single non-obvious point each type actually needs, and rewrites the wrapped prose comments as one sentence per line. Net 196 lines of comment removed with no code change; tests unchanged at 367 passing. Co-Authored-By: Claude Opus 5 (1M context) * Parallelize the verify partition pass PartitionAsync hashed every file in a sequential foreach while every other bulk path in the codebase uses Parallel.ForEachAsync bounded by --threads. On a real 264k-file tree that measured 446 files/min, putting roughly ten hours of single-threaded hashing in front of the first verification. Now matches IndexTask.ExecuteAsync: parallel, with a ConcurrentBag for the candidate list and Interlocked counters. Measured 908 files/min on the same storage. The remaining ceiling is the disk, not the code, since hashing reads every byte of the tree off spinning disks. Co-Authored-By: Claude Opus 5 (1M context) * Verify per file in quick mode and per batch in full mode Verification ran as three global barriers: hash every file, structurally check every file, then decode in batches, recording database state only at the very end. On a 264k-file tree that meant roughly five hours of hashing before the first file was verified, every file read twice with nothing left in cache between the two reads, and no progress retained if the run was interrupted. Quick mode now processes one file at a time in parallel, matching the shape import and process already use. Full mode does the same work per batch: hash the batch, decode it in one container, then act on the results and record them. Batches still run concurrently under --threads, so the container round trip is amortized without holding the whole tree in front of the first result. Two container flags follow from running this against a live Docker host. The image's healthcheck cannot pass under an overridden entrypoint, so every container reported unhealthy and alerted anything watching Docker; disable it. And the image may be the same one an existing container already runs, so the containers this tool starts carry a label and are never selected by image. Co-Authored-By: Claude Opus 5 (1M context) * Cut wrapped prose comments back to one sentence per line A sweep across every comment syntax the project ships found prose sentences wrapped mid-thought throughout the C# sources, which the comment rules disallow. Rewrites 32 of them as one sentence per line, cutting each block back to its load-bearing point, and drops the two remaining XML documentation blocks from non-public members. Comments-only, no behavior change. Label-and-example blocks are left alone, since a lead line followed by examples is not a wrapped sentence. The workflow YAML and repo-config carry the same pattern but are verbatim fleet content, so they are left for the hub rather than forked here. Co-Authored-By: Claude Opus 5 (1M context) * Decide the exiftool verdict from its output, not its exit code The metadata read ran with CliWrap's default exit-code validation, but exiftool exits non-zero on exactly the files whose verdict reports an error, while still writing the JSON that carries it. The call therefore threw before the verdict could be parsed, the file was counted as a pipeline failure, and the fail-on-error branch was unreachable. Disables exit-code validation for that read and throws only when exiftool returns no JSON at all, so a genuine tool failure still surfaces. The error text now rides along in the log rather than only a count. Reverting the validation change fails the new test, so the case proves the fault it names. Co-Authored-By: Claude Opus 5 (1M context) * Translate paths across the container boundary The decode pass mounted the media tree at its own absolute host path and wrote host paths to the container. A container path must be Linux-style, so that mount is invalid on a Windows host even though the repo ships and documents a Windows build. Mounts at a fixed container path instead and translates each path across the boundary, mapping the verdicts back by the same table. This drops the same-path assumption rather than branching on the operating system, so one code path serves every host. Co-Authored-By: Claude Opus 5 (1M context) * Log container start and stop for the decode pass The batch size trades container startup against how long a file waits between being hashed and being decoded, and neither side of that trade was measurable. Adds paired start and stop debug lines around each container invocation, so a debug-level log carries the duration of every launch without a timer in the code. The preflight carries no files, so its own pair isolates the fixed cost. Measured over 200 synthetic launches: about 3.8s median fixed overhead, of which roughly 2.3s is Immich's module graph rather than the container itself. That part is a floor, since loading those modules is what makes this Immich's decoder rather than an approximation. At the current batch size the work outruns startup by about seventy times. Co-Authored-By: Claude Opus 5 (1M context) * Correct what quick mode records in the database Both the code comment and the README claimed a quick run records nothing. It records content hashes through the shared index path whenever --db is given, and those hashes are then cached for a later run. What it withholds is the verified bit, so a full run still decodes the file. Documentation only, no behavior change. Co-Authored-By: Claude Opus 5 (1M context) * Count an unreadable file as failed rather than verified The structural check caught every exception, logged a warning, and returned true. A file it could not open was therefore counted as verified, so a permission error or an I/O fault produced a clean result for a file nothing had actually checked. For a command whose whole purpose is to answer whether a file is good, a false clean is the worst possible outcome. Now follows the per-file pattern the other tasks use: a file that no longer exists is logged and uncounted, since that races with any other run, and any other exception is an error against the failed count. Neither path reports the file as verified, and neither reports it as invalid, because being unable to read a file is not evidence of damage. Restoring the old catch fails the new test on Verified being 1 rather than 0. Co-Authored-By: Claude Opus 5 (1M context) * Say that exit 2 also covers files that could not be verified The exit code table claimed a 2 meant files were found to be bad, which stopped being true once an unreadable file started counting against the failed total. A run that could not read a file now exits 2 while nothing was actually judged bad, so a script reading that wording would draw the wrong conclusion. Says invalid or could not be verified, and points at the two counts that tell those apart. Co-Authored-By: Claude Opus 5 (1M context) * Raise the decode batch to 1024 files Per-file decode cost spans fiftyfold across real media, from about 20ms for a thumbnail to over a second for a raw frame, while container startup is a flat four seconds. At 256 files a batch of thumbnails spent 37 percent of its time starting the container, a multiplier of less than two. 1024 puts the cheap end at sevenfold and typical photographs near fiftyfold. Going further would risk producing fewer batches than threads on a small tree, which costs more in lost parallelism than it saves in startup. Measured on three real corpora: 20ms per file for thumbnails, 168ms for five megapixel JPEGs, and 1103ms for HEIC and video. Co-Authored-By: Claude Opus 5 (1M context) * Match the missing-file branch on the exception alone The filter also accepted any exception where File.Exists returned false, and File.Exists returns false when the path cannot be read at all, not only when it is absent. A permission error on an unreadable directory therefore landed in the branch meant for files that went away, logged at information and counted nowhere, so a run could exit 0 with files that were never checked. Matching FileNotFoundException and DirectoryNotFoundException keeps that branch to files that genuinely vanished and lets every other error reach the failed count, which is what the surrounding pattern already intends. Restoring the old filter fails the new test on Failed being 0 rather than 1. Co-Authored-By: Claude Opus 5 (1M context) * Narrow the same missing-file filter in the process command Applies the previous commit's correction to the only other place carrying the pattern, so both command paths classify a missing file the same way. Import and index never had the lenient branch, routing every exception to the failed count, so the two are now the whole set. Left as two inline filters rather than a shared predicate. The condition is a single type pattern, and an exception filter reads better where it is caught than behind a call. The mechanism is proven by the verify test, which fails on the broad filter. Reaching it through the process command needs a directory to become unreadable between enumeration and processing, which is a race rather than something a test can stage honestly. Co-Authored-By: Claude Opus 5 (1M context) * Document verify and replace the icloudpd examples with kei Carries the maintainer's 1.1 release notes and rewrites the workflow example around kei, since icloudpd is no longer maintained. kei keeps its settings in a TOML file rather than on the command line, so the example leads with the config and the later forms point at it: an interactive login to store the session, a one-shot sync, and the service form that keeps mirroring on the watch interval. The stack-specific parts of a real deployment are left out so the compose example stays portable. Adds the verify command to the overview, framed around the question metadata checks cannot answer. Also corrects three typos that the spelling gate covers, and adds the kei author's handle to the dictionary. Co-Authored-By: Claude Opus 5 (1M context) * Build the index task once instead of per file Verify constructed a new IndexTask for every file it considered. The class holds no mutable state, so one instance serves the whole run, matching how the index and import commands already build theirs. The saving is an allocation per file against work measured in megabytes of hashing, so this is consistency rather than a measurable gain. Sharing the instance across the parallel loop is safe for the same reason it was safe to rebuild it: nothing on it is written. Co-Authored-By: Claude Opus 5 (1M context) * Refine application description to enhance clarity across documentation and command line interface * Cover the invalid import path with a test Import gained an Invalid outcome for files exiftool reports errors on, but nothing asserted it. The case now drives a real file through ImportTask and checks that invalid is one while failed stays zero, that nothing is copied to the output, and that the source is left alone. Disabling the branch fails it on invalid being 0 rather than 1. Co-Authored-By: Claude Opus 5 (1M context) * Count a vanished file as failed in import and verify Only process rewrites the tree it walks, so only there is a name that has gone missing the expected result of the command's own work. Import copies without renaming its source and verify only reads, so under either of those a file that disappears between indexing and use means something outside the run changed the tree, and the run no longer covers what it was asked to. Verify's lenient branch is removed, leaving one catch that counts everything as failed, which also brings it in line with import and index. Process keeps its branch, and all three now carry a comment saying which case they are and why. Adds a case for a path that is gone by the time it is verified. Co-Authored-By: Claude Opus 5 (1M context) * Fail an item info box that declares more entries than it holds The entry loop stopped once the remaining bytes could not hold another header, then reported success. A container whose iinf promises more entries than it carries was therefore treated as sound, and because the missing entries are where item types live, a grid item could go unrecognized and skip the geometry check that exists to catch it. Running out of box before the declared count is now a malformed structure. Checked against real media before tightening, since a validator that flags a good file is worse than one that misses: unchanged on the regression corpus, and no new findings across roughly fifteen thousand healthy HEIC, MP4 and MOV files from three collections. Also carries the maintainer's workspace edit, sorting the extension recommendations and adjusting the list. Co-Authored-By: Claude Opus 5 (1M context) * Clear the prose gate on the lines this branch changed The spaced hyphen and the mid-sentence semicolon are both out, replaced by commas, separate sentences, or parentheses rather than an em dash, which the character set rules ban in the same terms. Confined to the lines this branch touched, per the rule that existing prose is corrected as a file is next edited rather than swept. In the Copilot instructions that means the repository layout and pipeline sections only, none of the three sections carried from the hub. Co-Authored-By: Claude Opus 5 (1M context) * Describe the real concurrency and drop an unused enum Two comments described verify as working through files and batches one after another, when both paths run in parallel under the thread limit. A reader reasoning about ordering, or about when a defect reaches the log, would have been misled by them. VerifyStatus was left over from an earlier shape of the code and never referenced once the counts record took over. Co-Authored-By: Claude Opus 5 (1M context) * Bring the prose to spec across the repository The gate reports clean on the whole tree, down from 245 findings, ahead of the repository going public. The carried files were stale rather than wrong. Every one of them is clean at the hub, so where a sentence differs only by the correction the hub already made, its wording is adopted rather than reinvented, which brings the two back into step instead of forking them. The runbook sections of the Copilot instructions are re-carried whole, since the hub had rewritten one of them in substance and not only in style. Nothing was copied wholesale. The two sections of GOVERNANCE.md that describe this repository's own tree and devcontainer, and the equivalent sections of the Copilot instructions, are deliberate adaptations, and taking the hub's text would have replaced this repository's layout with the hub's own. All eighteen byte-locked sections were checked against the hub before and after, and match. The rest is this repository's own prose, rewritten by hand. Co-Authored-By: Claude Opus 5 (1M context) * Keep a per-file verify failure from aborting the run An exception thrown inside either parallel loop of VerifyTask escapes Parallel.ForEachAsync and ends the whole command, so a single unreadable file aborts a run that may span an entire library and reports nothing about the files already verified. The hashing call that runs before the structural check when a database is in use is the exposed path. Guard the per-file body of both loops, logging the file and counting it failed, so the command reaches its summary and exits 2 rather than 1. The regression test sets a database, which is why the existing unreadable-file test did not cover this. Co-Authored-By: Claude Opus 5 (1M context) * Compare ISO-BMFF bounds by subtracting so a crafted file cannot overflow Three bounds checks add two attacker-controlled values before comparing. Each addend is already non-negative, but their sum can exceed long.MaxValue and wrap negative, so the comparison passes and a malformed file is reported clean, which is the one outcome a validator must not produce. An extent at offset long.MaxValue with length 2 returned None. Compare by subtracting instead, in the extent bounds check, the grid descriptor range check, and the top-level box walk, which is the only walk reading a 64-bit size and so the only one able to overflow. Co-Authored-By: Claude Opus 5 (1M context) * Describe the types VerifyResult.cs actually declares The architecture map credits the file with a VerifyStatus enum that exists nowhere in the codebase, sending a reader looking for a type that was never written. Name the two types it does declare. Co-Authored-By: Claude Opus 5 (1M context) * Warn on unrecognized ISO-BMFF structure instead of condemning the file The structural pass treated any structure the box walk could not follow as a defect, which conflates two different things. A file it can name a defect in is damaged; a file it merely cannot parse may just be a format the parser has never met, and from the inside those look identical. That asymmetry is measurable. Across 236,797 files the named-defect checks produced 16,394 hits and no false positives, while the generic check produced 7 hits, all false positives, on Samsung MP4s carrying a proprietary SEF trailer after the last box. Every future vendor quirk would arrive the same way, as a corruption report against someone else's valid library. MalformedBoxStructure now counts Suspect, logs a warning, and lets the file through to the decoder, which is the authority on whether Immich can use it. Suspect never affects the exit code. Recognize the SEF trailer specifically so the common case is silent rather than merely non-fatal. Keeping the check advisory rather than dropping it matters, because a truncated file decodes: Immich renders a thumbnail from bad-truncated.heic and reports nothing wrong. This is the only signal that class of damage has. Co-Authored-By: Claude Opus 5 (1M context) * Make the README help blocks match what the CLI prints The blocks are presented as literal --help output, but eight option descriptions across four commands had drifted from the code, so the documented CLI and the real one disagreed. Copilot found one instance; the same class covered process, import, index, and verify. Four descriptions were wrong in the code rather than the docs, because the options are single instances shared by every command that takes them. A shared option cannot say "the directory path to process" when verify, index, trash, and undo do not process anything, so those now read neutrally. The long --quick text moved out of help, where the README option notes already carry it in full. Co-Authored-By: Claude Opus 5 (1M context) * Refuse to map a file outside the mount into a container path ToContainerPath built a container path from a relative path it never checked, so a file outside the mount root would produce a path containing '..' that resolves somewhere else inside the container. Immich would then judge a different file and report the verdict against the original name, which is worse than an error because it looks like a result. The file list is enumerated under the mount root today, so this is an unchecked invariant rather than a live bug. TryToContainerPath now returns false instead, and the caller logs the file and counts it failed rather than throwing, since a throw there would abort the whole run. Co-Authored-By: Claude Opus 5 (1M context) * Fail closed when box nesting outruns the depth guard WalkBoxes returned true past MaxBoxDepth, so a container nesting meta boxes deeper than the guard stopped the walk and reported success. No iloc was ever recorded, so Validate returned None and a file built specifically to outrun the parser got a clean bill of health. Real containers nest two deep, since only meta and iinf recurse and every other box is skipped by declared size, so passing the guard means the structure cannot be followed rather than that the file is unusual. Return false, which now reports Suspect and defers to the decoder rather than condemning the file. Co-Authored-By: Claude Opus 5 (1M context) * Stop the grid check reading a descriptor it cannot locate Construction method 1 means the item offset is relative to idat, so with no idat box in the file there is no base to resolve against. IdatOffset stayed at its default of zero, and the offset was read as absolute, pulling four arbitrary bytes and treating them as an ImageGrid descriptor. Those bytes then decided the tile geometry, so the check could invent a GridTileCountMismatch out of unrelated data. That is the worst direction for this feature, because a mismatch is a named defect that condemns the file. A synthesized case returned GridTileCountMismatch before this change. Report the contradiction instead, which is advisory and leaves the verdict to the decoder. Also require the box walk to consume its whole range, since a remainder too small to hold a header belongs to no box. Measured first: across 20,000 real files from the collection, every single one consumed its range exactly, so this costs nothing on real media. Co-Authored-By: Claude Opus 5 (1M context) * Import System.Globalization in the file that reads it The namespace is a global using, so the file compiled without the import and the import changes nothing at build time. It is kept because a reader seeing NumberStyles and CultureInfo in this file has no local evidence of where they come from, which is what led a reviewer to call the file uncompilable. Verified that no formatter strips it: csharpier, dotnet format style in apply mode, and husky all leave it in place. Co-Authored-By: Claude Opus 5 (1M context) * Stop counting a dropped file twice, and refuse an unlocatable grid descriptor Two independent problems, both reported against the same round. The decode-batch failure handler counted one failure per entry in batch, but a file dropped during path mapping was already counted there and never entered mapped. Adding the mapping filter in 67e9a37 created the gap, since mapped and batch were the same length before it. Count over mapped instead. CheckGridGeometry read any construction method other than 1 as an absolute offset, so method 2, which is relative to another item, pulled four arbitrary bytes and let them decide the tile geometry. A synthesized case returned GridTileCountMismatch, so the file was condemned by a named defect derived from an accidental read. This is the same shape as the missing-idat case fixed in 88455e3, and the remaining branch of it. Report the contradiction instead, which is advisory and leaves the verdict to the decoder. Co-Authored-By: Claude Opus 5 (1M context) * Bound the dimg reference reads to their own box ParseItemReference read from_item_ID and reference_count straight after the dimg header without checking either fits inside the box. A dimg declaring only its 8-byte header carries no fields at all, so both reads continued into whatever followed, and the resulting count went into the map the grid check consults. Where those bytes land decides which way it goes wrong. A synthesized header-only dimg filed the count under an item id nothing references, so the grid item lost its entry and the file passed as clean. Bytes that happen to decode to a real grid item's id give the opposite result, a tile count invented from unrelated data, which the grid check reports as a named defect. Require the box to hold the two fields and the reference list it declares, and report malformed structure otherwise, which is advisory. Co-Authored-By: Claude Opus 5 (1M context) * Make Immich's decoder the only judge, and drop the container parser The hand-rolled ISO-BMFF validator is removed along with the --quick mode it powered. Verify now runs Immich's decoder and nothing else. The parser was a standing liability rather than a one-off bug source. Five consecutive review rounds each found a real defect in it, every one the same shape: a read that was not bounded by the structure it claimed to parse, fed into the check that condemns a file by name. Two of those produced a tile count invented from unrelated bytes, which is the worst outcome available here, since it marks a healthy photo corrupt. The corpus it was tested against is one collection, and the failure mode of a format it has never met is indistinguishable from damage, so shipping it publicly meant condemning other people's media on evidence it could not actually read. Removing it costs the truncation signal, which decodes cleanly and so no decoder reports. That is a real loss, and the deliberate price of not guessing about files this tool has never seen. Verify keeps the existence check, because a file that vanished mid-run still counts as failed rather than damaged, and without it the missing path would reach Immich and come back as unrenderable. The exiftool -validate read is untouched. Verify has no offline mode now, so the tests that drive it skip when the image is absent, and the ones that judged synthetic bytes use real media. Co-Authored-By: Claude Opus 5 (1M context) * Log a file missing during process at debug rather than information Process rewrites the tree it walks, so a name that has gone is usually its own earlier rename. It could equally be an external deletion, and the two are not cheaply told apart, so the message neither fails the run nor claims to know which happened. Debug is the level for something that is expected most of the time and diagnostic the rest. Import and verify keep counting a missing file as failed, since neither modifies its input and a vanished file there can only be external. Verify reaches that verdict through an explicit check rather than an exception, because without a database it never opens the file, so the wording now matches import's and the comments say why the mechanisms differ. Co-Authored-By: Claude Opus 5 (1M context) * Keep a file missing during process at information, not debug Process rewrites the tree it walks, so a name that has gone is usually its own earlier rename, and it neither fails the run nor claims to know whether an external deletion caused it instead. The line stays at information rather than dropping to debug, so a run still accounts for why a file is gone without the reader having to raise the level to find out. Verify and import are deliberately louder: both log an error and count the file as failed, because neither modifies its input, so a vanished file there can only be an external change to the tree mid-run. Counting it failed is what makes the command exit 2, and the error level agrees with that. Co-Authored-By: Claude Opus 5 (1M context) * Stop the preflight failure pointing at an option that no longer exists Removing --quick left the preflight message telling the reader to pass it, so the one message a user sees when Docker is missing offered a way out that the CLI now rejects. It states the actual requirement instead: docker and the Immich image, with no offline mode, because the decoder is the whole check. Also drop the System.Buffers.Binary import, unused since the tests that synthesized box headers went with the parser. Co-Authored-By: Claude Opus 5 (1M context) * Test that an unknown verify option is rejected, without naming one The case names a specific flag that no released version ever accepted, so the name kept a option alive in the codebase that a reader could never have used. What is worth guarding is the general behavior: an option the command does not define is an error rather than something silently ignored. Co-Authored-By: Claude Opus 5 (1M context) * Count an unreadable file as failed whether or not a database is configured With a database the hash read opens every file, so a permission or I/O failure throws and the per-file guard counts it failed. Without one nothing opened the file before the container did, so the same file was handed to Immich, came back unrenderable, and was counted invalid. The verdict for a tooling gap therefore depended on an unrelated flag, and in one of the two paths reported a readable-media problem as damaged media. Probe the file in the no-database path and let the existing guard count the throw, so both paths reach the same verdict. Also rename the launch profile that still said Quick. The two verify profiles differ by whether they pass a database, so the name says that. Co-Authored-By: Claude Opus 5 (1M context) * Count a file that vanished before the decode as failed, not invalid A file removed after its batch was assembled makes the decoder fail, and that failure was reported as "Immich cannot render", so a file that is simply gone was recorded as damaged media. Check the host before believing a rejection, so the vanished case reaches the same failed count as every other place the tree changes mid-run. Also correct two claims in the verify flow. The media directory mounts at the fixed container path rather than at its own absolute path, and paths are translated onto it, which is what lets a host path that is not a valid container path work at all. The command exits 2 for a failed file as well as an invalid one. Co-Authored-By: Claude Opus 5 (1M context) * Split the untested-race note across two comment lines Co-Authored-By: Claude Opus 5 (1M context) * Reject a path carrying a line break before sending it to the container Paths reach the container one per line, so a name containing a newline or a carriage return cannot be expressed in that protocol. Without this the name splits and the container is asked to decode fragments of it, and the run reports the generic "no verification result returned" rather than the reason. The counts are unchanged, because a split path matches no returned verdict and so already counted failed. What this buys is an accurate reason and a well-formed payload, not a different tally. No path under the collection carries either character, so this guards a hazard rather than fixing an observed failure. Co-Authored-By: Claude Opus 5 (1M context) * Skip the Immich preflight when nothing needs decoding Verify preflighted the Docker image before looking at what it had been given, so a directory holding no supported media exited 1 on a host without Docker, and paid four seconds of container startup on one that had it. The command reported an infrastructure failure for work it was never going to do. Preflight now runs only when at least one file carries a supported extension. The partition loop is untouched, so non-media files are still counted ignored and still drive the unknown-extension warnings. Proven end to end with docker removed from PATH: the same directory of text files exits 1 before this change and 0 after. The test is deliberately not gated on the Immich image, because a host without it is the case this protects, which is also what CI is. Co-Authored-By: Claude Opus 5 (1M context) * Mount the media directory in a form that survives the characters in a path The volume spec is colon delimited, so a colon anywhere in the media directory made docker reject the spec outright, and the whole batch failed with "too many colons" against a directory that was perfectly valid. Switching to --mount alone only moves the problem, because that flag is comma delimited and a comma is far the more likely of the two in a photo directory name: this collection holds 285 paths containing a comma and none containing a colon, so the naive swap would trade a hazard that has never occurred for one that already exists. Quoting the whole source field as CSV takes both, with any embedded quote doubled per CSV rules. Verified against directories named with a colon, a comma, both, and both plus a quote. Co-Authored-By: Claude Opus 5 (1M context) * Use a neutral example path in the kei instructions The examples named a real media tree rather than an illustrative one, which puts a reader's own directory layout in front of them and invites anyone following along to copy a path that means nothing on their machine. Co-Authored-By: Claude Opus 5 (1M context) * Defer the Immich preflight until a file actually reaches the decoder Gating the preflight on any supported extension being present left a database-backed run that skips every file still demanding Docker, so an incremental run over an already-verified tree failed on a host without it. The comment claimed nothing to decode means nothing needs Docker while the code only honoured that for a tree holding no media at all. The first batch to find a candidate now triggers the preflight, once, and the rest await it. A run that decodes nothing never starts a container. The earlier reason for not doing this was wrong. Batches are processed as they are chunked, so the wait for a Docker failure is bounded by one batch rather than by hashing the whole tree, which is seconds rather than hours. Verified with docker off PATH: a fully cached run exits 0 having skipped its files, and an uncached one still exits 1 with the message intact. Co-Authored-By: Claude Opus 5 (1M context) * Probe readability for every file that reaches the decoder The probe only ran when no database was configured, on the assumption that the hash read would open the file otherwise. It does not: a row matching on size and mtime returns cached hashes without reading the file, and stat needs no read permission, so an unreadable file with a cached row went straight to the decoder unopened. Immich runs as another user and may well read it, so a gap in this tool's own access came back as a verdict about the media, which is the outcome this branch has been closing off everywhere else. The probe now sits at the end of the skip decision, so it covers exactly the files that get judged and costs nothing on the ignored, missing, and already-verified paths that an incremental run spends its time on. Co-Authored-By: Claude Opus 5 (1M context) * Check readability in the shared metadata call, not only in verify Reading a file's attributes needs no permission on its content, so size and mtime prove nothing about whether the bytes can be reached. Verify was given its own probe for that, which left the same hole open everywhere else. Measured against an unreadable file before this change: process reported Failed 0 and exited 0, and import reported Imported 1. Both claimed success over a file they never read. Index was already correct, because it hashes and therefore opens. The probe now sits in the shared exiftool call, so process and import inherit it, and verify calls the same helper for the path that hands the file to Immich rather than reading it here. Keeping it ahead of exiftool is what makes the distinction work. exiftool reports a file it cannot open and a file it cannot parse the same way, as an error alongside well formed JSON, so keying on that field would have turned damaged media into a tooling failure. Garbage bytes in a readable file still count Invalid, and only an unreadable one counts Failed. Co-Authored-By: Claude Opus 5 (1M context) * Name the no-extension case, and stop the trash comment overstating A file with no extension tracks as an empty string, so the summary printed "Unknown file extension: ''", a quoted nothing that reads as a bug in the message rather than a fact about the tree. All four commands feed the same tracker, so naming the case there covers every one of them. No file in the 264,044 verified carried this, which makes it latent noise rather than an observed defect. The trash comment claimed a page either fetches or throws, so no partial state exists. Pagination also stops early on a NextPage value it cannot use, warning and then returning success, which leaves the database short of the server. The comment now says so. Co-Authored-By: Claude Opus 5 (1M context) * Exit 2 when trash syncs only part of the server Pagination stops early on a NextPage value it cannot use, and the command then reported success over a database holding fewer hashes than the server. That database is what import --trashdb and process --trashdb consult to skip files, so a short one silently re-imports assets that were trashed, and a pipeline gating on the exit code had no way to see it. The stop is now an error rather than a warning, and the command exits 2, which is the existing meaning of a run that completed while carrying a failure. Whatever pages were read are still kept, since a partial database is more useful than none as long as the caller knows it is partial. Co-Authored-By: Claude Opus 5 (1M context) * Drain the docker probe's pipes before waiting on it The image-availability probe redirected both streams and read neither, so the child could fill a pipe buffer and block on the write while the wait blocked on the child. Measured at 5,662 bytes against a 64 KB buffer today, which is why nothing has hung, but the margin belongs to the image rather than to this code. Both streams are now read before the wait, and the probe asks for an empty rendering since only the exit code is wanted, so there is almost nothing to buffer either way. Co-Authored-By: Claude Opus 5 (1M context) * Name every exit code, list verify, and finish the trash comment Three findings from suppressed review comments that had not been answered. CommandRunner logged completion for anything that was not Failed, so an Error would have reported the command complete. Nothing returns Error from the work itself today and the catch blocks return it directly, so this is a guard on a future caller rather than a live defect. The architecture map still described five subcommands and omitted verify, having been written before this branch added it. The trash comment covered pagination stopping early but not a page that throws, which leaves the database short as well, having kept the pages before it. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .github/copilot-instructions.md | 149 +++--- .vscode/launch.json | 39 ++ AUDIT.md | 4 +- CODESTYLE.md | 56 +-- Docker/Dockerfile | 2 +- Docker/README.md | 8 +- HISTORY.md | 15 +- PhotoCleaner.code-workspace | 15 +- PhotoCleaner/CommandLine.cs | 30 +- PhotoCleaner/CommandRunner.cs | 32 +- PhotoCleaner/ExifToolJson.cs | 57 +++ PhotoCleaner/ExitCode.cs | 13 + PhotoCleaner/ImmichVerifyScript.cs | 109 ++++ PhotoCleaner/ImportCommand.cs | 4 + PhotoCleaner/ImportTask.cs | 55 +- PhotoCleaner/IndexCommand.cs | 2 + PhotoCleaner/MediaUtilities.cs | 33 +- PhotoCleaner/ProcessCommand.cs | 34 +- PhotoCleaner/ProcessTask.cs | 41 +- PhotoCleaner/SkippedExtensionTracker.cs | 8 + PhotoCleaner/TrashCommand.cs | 11 +- PhotoCleaner/UndoCommand.cs | 2 +- PhotoCleaner/UndoTask.cs | 23 +- PhotoCleaner/VerifyCommand.cs | 52 ++ PhotoCleaner/VerifyResult.cs | 22 + PhotoCleaner/VerifyTask.cs | 449 +++++++++++++++++ PhotoCleanerTests/CommandLineTests.cs | 57 +++ PhotoCleanerTests/DirectoryCleanerTests.cs | 5 +- PhotoCleanerTests/ExifToolJsonTests.cs | 44 ++ PhotoCleanerTests/ImportTaskTests.cs | 105 +++- PhotoCleanerTests/IndexTaskTests.cs | 12 +- PhotoCleanerTests/ProcessTaskTests.cs | 172 ++++++- PhotoCleanerTests/TempDirectoryFixture.cs | 9 +- PhotoCleanerTests/TrashCommandTests.cs | 45 +- PhotoCleanerTests/UndoTaskTests.cs | 2 +- PhotoCleanerTests/VerifyTaskTests.cs | 560 +++++++++++++++++++++ README.md | 388 ++++++++++---- WORKFLOW.md | 78 +-- cspell.json | 11 +- repo-config/README.md | 26 +- 40 files changed, 2423 insertions(+), 356 deletions(-) create mode 100644 PhotoCleaner/ExitCode.cs create mode 100644 PhotoCleaner/ImmichVerifyScript.cs create mode 100644 PhotoCleaner/VerifyCommand.cs create mode 100644 PhotoCleaner/VerifyResult.cs create mode 100644 PhotoCleaner/VerifyTask.cs create mode 100644 PhotoCleanerTests/VerifyTaskTests.cs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 7476d19..02e75ab 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -18,7 +18,7 @@ Summarized for VS Code's generators. The full rules, rationale, and examples are ## Reviewing Carried Fleet Content -Several of this repository's governance files are carried from a shared template and kept in sync across a fleet of sibling repositories, among them `AGENTS.md`, `GOVERNANCE.md`, `CODESTYLE.md`, `WORKFLOW.md`, this file, and the `repo-config/` rulesets. Most of `GOVERNANCE.md` is universal fleet law: every section that states a rule, as opposed to the two that describe this repository's own directory tree and devcontainer, is byte-locked and verified by an automated byte-for-byte match against the template canonical, not by line-by-line review. The two sections `AGENTS.md` carries are byte-locked on the same terms. +Several of this repository's governance files are carried from a shared template and kept in sync across a fleet of sibling repositories, among them `AGENTS.md`, `CODESTYLE.md`, `WORKFLOW.md`, this file, and the `repo-config/` rulesets. Most of `GOVERNANCE.md` is universal fleet law: every section that states a rule, as opposed to the two that describe this repository's own directory tree and devcontainer, is byte-locked and verified by an automated byte-for-byte match against the template canonical, not by line-by-line review. `AGENTS.md` is the thin router and carries two byte-locked sections of its own, with no repository-specific ones. Two constraints follow when reviewing that content. @@ -27,19 +27,32 @@ Two constraints follow when reviewing that content. ## GitHub Copilot Review Runbook -> This runbook implements the [GOVERNANCE.md "PR Review Etiquette"](../GOVERNANCE.md#pr-review-etiquette) review-loop contract for GitHub Copilot. Without it in-repo, an agent has no pointer to the reliable Copilot mechanics and falls back to known-broken paths (the no-op `POST /requested_reviewers`, the wrong bot-login filter). In the API snippets below, fill the `` placeholder (the PR number). +> This runbook implements the [GOVERNANCE.md "PR Review Etiquette"](../GOVERNANCE.md#pr-review-etiquette) review-loop contract for GitHub Copilot. Without it in-repo, an agent has no pointer to the reliable Copilot mechanics and falls back to known-broken paths (the no-op `POST /requested_reviewers`, the wrong bot-login filter). In the API snippets below, fill the `` / `` / `` placeholders. Use this section for provider-specific mechanics. The expected review loop *contract* (request review on every push, verify head-SHA coverage, triage findings, reply + resolve, escalate when stuck) is defined in [GOVERNANCE.md -> PR Review Etiquette](../GOVERNANCE.md#pr-review-etiquette). This section only describes how to make GitHub Copilot reliably execute it. ### Triggering and Polling -Auto-review on push is configured (via the branch ruleset's `copilot_code_review` rule with `review_on_push: true`) but fires inconsistently in practice - treat it as best-effort, not guaranteed. After every push, **re-request a review programmatically** via the GraphQL `requestReviews` mutation, passing the Copilot reviewer's bot node id in `botIds`. This drives the loop end-to-end without a UI hand-off. +Auto-review on push is configured (via the branch ruleset's `copilot_code_review` rule with `review_on_push: true`) but fires inconsistently in practice, so treat it as best-effort, not guaranteed. After every push, **re-request a review programmatically** via the GraphQL `requestReviews` mutation, passing the Copilot reviewer's bot node id in `botIds`. This drives the loop end-to-end without a UI hand-off. -**A review with no inline comments is still a completed review - not a failure, and not a reason to ask the maintainer to re-trigger.** Copilot very often posts a single formal review (GraphQL `state: COMMENTED`) whose body ends with "...reviewed N of N changed files ... and generated no comments" and adds **zero** inline threads. That review carries the head `commit.oid` and fully satisfies the loop - it is the clean-pass success case. Never read "no inline comments" as "the review didn't run," and never re-request or escalate to the maintainer because comments are absent. +**A review with no inline comments is still a completed review, not a failure, and not a reason to ask the maintainer to re-trigger.** Copilot very often posts a single formal review (GraphQL `state: COMMENTED`) whose body ends with "...reviewed N of N changed files ... and generated no comments" and adds **zero** inline threads. That review carries the head `commit.oid` and fully satisfies the loop, and it is the clean-pass success case. Never read "no inline comments" as "the review didn't run," and never re-request or escalate to the maintainer because comments are absent. -**Round 1 is normally auto-seeded - poll for it before trying to self-trigger.** Auto-review-on-open supplies the first review with no `botIds` call needed, but it can lag one to three minutes. After opening a PR (or the first push), **poll** for a Copilot review on the head SHA (see [Verify Review Covered Current Head](#verify-review-covered-current-head)) before concluding none ran. The `requestReviews` mutation below is for **re-requesting on later pushes** (a new head SHA); by then a prior review exists, so its bot node id is readable. A missing bot node id on round 1 therefore means "the auto-review has not landed yet - wait and poll," **not** "ask the maintainer to kick it off." +**Read the low-confidence findings, which are not inline threads.** A review body can carry a collapsed `
` block of findings Copilot withheld from the inline threads, and those findings appear nowhere in `reviewThreads`, so a loop that polls threads alone never sees them and reports a clean pass. **Match the block on more than one phrasing.** Its heading has appeared both as `Suppressed comments (N)` and as "Comments suppressed due to low confidence", so a filter keyed on either one alone silently reports zero suppressed findings on a review that has them, the same false clean this rule exists to prevent, one level up in the detection. They have been right repeatedly, including a rule stated more broadly than its check enforced and a check that skipped fenced blocks in every rule but one. Read the body of every review, investigate each suppressed finding on the same footing as an inline one, and answer it in the PR conversation, since a suppressed finding has no thread to reply on or resolve. -> **The reviewer login differs by API.** In **GraphQL** (`gh api graphql` and `gh pr view --json reviews`, which is GraphQL-backed) the `Bot.login` is `copilot-pull-request-reviewer` - **no `[bot]` suffix**. In the **REST** API (`gh api repos/.../issues|pulls/...`) the same account's `user.login` is `copilot-pull-request-reviewer[bot]` - **with** the suffix. Each query below uses the correct form for its API; match the API, not a single spelling, when adapting them. +```sh +# `test` with an alternation, not `contains` on one phrasing: the heading wording has changed. +gh api repos///pulls//reviews --jq \ + '.[] | select(.body | test("Suppressed comments|low confidence")) | .body' + +# Scope it to the current head, so an answered finding from an earlier round does not re-open. +PR_HEAD=$(gh pr view --json headRefOid --jq '.headRefOid') +gh api repos///pulls//reviews --jq \ + "[.[] | select(.commit_id==\"$PR_HEAD\") | select(.body | test(\"Suppressed comments|low confidence\"))] | length" +``` + +**Round 1 is normally auto-seeded, so poll for it before trying to self-trigger.** Auto-review-on-open supplies the first review with no `botIds` call needed, but it can lag one to three minutes. After opening a PR (or the first push), **poll** for a Copilot review on the head SHA (see [Verify Review Covered Current Head](#verify-review-covered-current-head)) before concluding none ran. The `requestReviews` mutation below is for **re-requesting on later pushes** (a new head SHA). By then a prior review exists, so its bot node id is readable. A missing bot node id on round 1 therefore means "the auto-review has not landed yet - wait and poll," **not** "ask the maintainer to kick it off." + +> **The reviewer login differs by API.** In **GraphQL** (`gh api graphql` and `gh pr view --json reviews`, which is GraphQL-backed) the `Bot.login` is `copilot-pull-request-reviewer`, with **no `[bot]` suffix**. In the **REST** API (`gh api repos/.../issues|pulls/...`) the same account's `user.login` is `copilot-pull-request-reviewer[bot]`, **with** the suffix. Each query below uses the correct form for its API, so match the API, not a single spelling, when adapting them. ```sh # 1. PR node id + the Copilot reviewer's bot node id (read from any existing @@ -47,7 +60,7 @@ Auto-review on push is configured (via the branch ruleset's `copilot_code_review PR_NODE=$(gh pr view --json id --jq '.id') BOT_ID=$(gh api graphql -f query=' { - repository(owner: "ptr727", name: "PhotoCleaner") { + repository(owner: "", name: "") { pullRequest(number: ) { reviews(first: 50) { nodes { author { __typename login ... on Bot { id } } } } } @@ -65,14 +78,14 @@ mutation($pr: ID!, $bot: ID!) { }' -F pr="$PR_NODE" -F bot="$BOT_ID" ``` -The bot node id is read from an existing Copilot **formal** review (`pullRequest.reviews`), so step 1 needs at least one prior formal review on the PR - the auto-review-on-open normally supplies the first one (it may have **no inline comments**; that still counts, and its bot node id is still readable). Poll for it (give auto-review-on-open a few minutes) before deciding it is missing. +The bot node id is read from an existing Copilot **formal** review (`pullRequest.reviews`), so step 1 needs at least one prior formal review on the PR, and the auto-review-on-open normally supplies the first one (it may have **no inline comments**, which still counts, and its bot node id is still readable). Poll for it (give auto-review-on-open a few minutes) before deciding it is missing. -**Cold start (round 1 not yet landed): read the id repo-wide, not from this PR.** The Copilot reviewer's bot node id is the reviewer bot *account's* node id and is **stable across every PR in the repo**. So a freshly opened PR that has neither a formal review nor an issue comment yet does **not** need UI seeding to bootstrap the id - read it from any prior Copilot review anywhere in the repo, then feed it into the `requestReviews` mutation to drive round 1. Query the **most recent** PRs (`first: 20` with an explicit newest-first order; plain `last: 20` returns the *oldest* PRs, which may predate Copilot on the repo), and **guard for an empty result** - an empty `$BOT_ID` means none of the sampled PRs carry a Copilot review. Widen the window (raise the count or paginate) before concluding the repo has never had one and falling back to UI seeding; never feed an empty id into the mutation: +**Cold start (round 1 not yet landed): read the id repo-wide, not from this PR.** The Copilot reviewer's bot node id is the reviewer bot *account's* node id and is **stable across every PR in the repo**. So a freshly opened PR that has neither a formal review nor an issue comment yet does **not** need UI seeding to bootstrap the id: read it from any prior Copilot review anywhere in the repo, then feed it into the `requestReviews` mutation to drive round 1. Query the **most recent** PRs (`first: 20` with an explicit newest-first order; plain `last: 20` returns the *oldest* PRs, which may predate Copilot on the repo), and **guard for an empty result**, since an empty `$BOT_ID` means none of the sampled PRs carry a Copilot review. Widen the window (raise the count or paginate) before concluding the repo has never had one and falling back to UI seeding; never feed an empty id into the mutation: ```sh BOT_ID=$(gh api graphql -f query=' { - repository(owner: "ptr727", name: "PhotoCleaner") { + repository(owner: "", name: "") { pullRequests(first: 20, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { reviews(first: 20) { nodes { author { __typename login ... on Bot { id } } } } } } @@ -86,21 +99,25 @@ if [ -z "$BOT_ID" ]; then fi ``` -If Copilot posted **only an issue comment** on this PR and no formal review, you can instead read the id from that comment's author (`pullRequest.comments` -> author `... on Bot { id }`). Manual UI seeding is the last resort - needed only for a repo that has **never** had a Copilot review, so no prior id exists anywhere to read; then use the mutation for every subsequent re-request. +If Copilot posted **only an issue comment** on this PR and no formal review, you can instead read the id from that comment's author (`pullRequest.comments` -> author `... on Bot { id }`). Manual UI seeding is the last resort, needed only for a repo that has **never** had a Copilot review, so no prior id exists anywhere to read. Use the mutation for every subsequent re-request. **Do NOT post `@Copilot review` as a PR comment.** That comment triggers the Copilot *coding agent* (`copilot-swe-agent[bot]`), which makes code changes rather than posting a review. -Known non-working request paths (don't rely on them - use the `requestReviews` mutation above instead): +Known non-working request paths (don't rely on them, and use the `requestReviews` mutation above instead): - `POST /requested_reviewers` with `reviewers=[Copilot]` can return 200 but no-op. - `copilot-pull-request-reviewer` as a requested reviewer slug returns 422. -- `requestReviews` with the reviewer's bot node id in **`userIds`** fails with `Could not resolve to User node` - the Copilot reviewer is a **Bot**, so its node id goes in **`botIds`** (as in the mutation above), never `userIds`. -- `suggestedActors(capabilities: [CAN_BE_ASSIGNED])` lists `copilot-swe-agent` (the coding agent), not `copilot-pull-request-reviewer` - do not source the reviewer's bot node id there. Read it from an existing review per step 1 above. -- There is no `removePullRequestFromReviewRequest` mutation, and removing the reviewer to force a fresh pass is unnecessary anyway - `requestReviews` with `union: true` re-fires the review on the current head. +- `requestReviews` with the reviewer's bot node id in **`userIds`** fails with `Could not resolve to User node`, because the Copilot reviewer is a **Bot**, so its node id goes in **`botIds`** (as in the mutation above), never `userIds`. +- `suggestedActors(capabilities: [CAN_BE_ASSIGNED])` lists `copilot-swe-agent` (the coding agent), not `copilot-pull-request-reviewer`, so do not source the reviewer's bot node id there. Read it from an existing review per step 1 above. +- There is no `removePullRequestFromReviewRequest` mutation, and removing the reviewer to force a fresh pass is unnecessary anyway, since `requestReviews` with `union: true` re-fires the review on the current head. ### Verify Review Covered Current Head -Before merging, confirm Copilot reviewed the current PR head SHA. Copilot may respond as either a formal review (carries an exact commit SHA) or an issue comment (no SHA - use the most recent Copilot comment for manual confirmation). Check both. +Before merging, confirm Copilot reviewed the current PR head SHA. Copilot may respond as either a formal review (carries an exact commit SHA) or an issue comment (no SHA, so use the most recent Copilot comment for manual confirmation). Check both. + +**Count matches and compare numerically, so an empty result cannot read as success.** A poll that captures a `gh api --jq` result and exits on `[ "$found" != "0" ]` treats an **empty** string as a landed review, and an empty string is exactly what a mis-written filter returns. Pipe the matches through `wc -l` and test `-gt 0`, so a query that finds nothing and a query that ran wrong both read as "not yet". A `gh` call that fails to run reaches the test the same way, because it writes its message to stderr and prints nothing to stdout, so the `$(...)` around it still yields the empty string. A mistyped or unsupported flag is the usual cause, and `gh` reports one as `accepts 1 arg(s), received 4` rather than as anything resembling a review verdict. + +**Check head coverage before reading merge-state, never the reverse.** A push makes the required checks go green before Copilot re-reviews the new head, so `mergeStateStatus` can read `CLEAN` in the window before any formal review covers the head. A poll that exits on `CLEAN` merges into that gap. Gate on a formal review whose `commit.oid` equals the current head SHA first, then on zero unresolved threads, and only then read merge-state. ```sh PR_HEAD=$(gh pr view --json headRefOid --jq '.headRefOid') @@ -112,33 +129,35 @@ gh pr view --json reviews --jq \ # 2. Issue comment - show the most recent Copilot comment for manual # confirmation. This is the REST API, so the login carries the `[bot]` suffix. -gh api repos/ptr727/PhotoCleaner/issues//comments --jq \ +gh api repos///issues//comments --jq \ '[.[] | select(.user.login=="copilot-pull-request-reviewer[bot]")] | last | {created_at, body: .body[:200]}' ``` -Coverage is confirmed when (1) exits 0 - **a formal review with no inline comments still satisfies path (1)**, because coverage is about the head SHA, not the comment count. For issue comments (path 2), body content is the only reliable signal - `created_at` is not: `git log -1 --format=%cI` is the **commit** timestamp, not the push timestamp, so amended or rebased commits can have an earlier timestamp and an older Copilot comment could satisfy a time check even though Copilot never saw the current head. Treat path (2) as confirmed only when the comment body explicitly refers to the current changes. +Coverage is confirmed when (1) exits 0, and **a formal review with no inline comments still satisfies path (1)**, because coverage is about the head SHA, not the comment count. For issue comments (path 2), body content is the only reliable signal, and `created_at` is not: `git log -1 --format=%cI` is the **commit** timestamp, not the push timestamp, so amended or rebased commits can have an earlier timestamp and an older Copilot comment could satisfy a time check even though Copilot never saw the current head. Treat path (2) as confirmed only when the comment body explicitly refers to the current changes. ### Bounded Retry Workflow -This path is only for a **genuinely missing** review - no Copilot review (formal *or* issue comment) covers the current head SHA after polling. A review that covered the head but produced no comments is a clean pass, not a missing review; do not enter this retry path for it. +This path is only for a **genuinely missing** review, meaning no Copilot review (formal *or* issue comment) covers the current head SHA after polling. A review that covered the head but produced no comments is a clean pass, not a missing review, so do not enter this retry path for it. + +**A slow review is pending, not missing, so poll with backoff and never escalate on a timeout alone.** Copilot can lag far beyond the usual one-to-three minutes when it has been re-requested many times in quick succession, because it throttles under load, and a re-review landing tens of minutes after the request is normal. A poll that times out is therefore evidence only that the review has not landed *yet*, not that Copilot is done or unresponsive. Report the status as "review still pending" and keep polling on a widening interval (for example 20s steps, then a few minutes) rather than stopping. Enter the escalation step below only when the `requestReviews` mutation itself no-ops or errors, or after a genuinely long wait with the request confirmed accepted, never merely because one fixed poll window elapsed. If a review did not run on the current head, retry: 1. Wait briefly and check head-SHA coverage (see above). -1. Re-request the review via the `requestReviews` mutation (see "Triggering and Polling"); fall back to the GitHub PR UI only if the mutation no-ops. +1. Re-request the review via the `requestReviews` mutation (see "Triggering and Polling"), falling back to the GitHub PR UI only if the mutation no-ops. 1. Retry up to two more times (three total). 1. If still missing, mark review as blocked and escalate to the user/maintainer with what was attempted. ### Reply and Thread Resolution Workflow -Every id below is captured from a live query into a variable and passed from there - never hand-typed, guessed, or pasted as a `PRRT_...` literal. A node id resolves globally, so a fabricated or stale id does not fail, it writes to a real thread on an unrelated repository. This runbook implements [GOVERNANCE.md "Repository Boundaries and Write Safety"](../GOVERNANCE.md#repository-boundaries-and-write-safety): write only to this repo, capture every id from a live query, and never suppress a mutation's output. +Every id below is captured from a live query into a variable and passed from there, never hand-typed, guessed, or pasted as a `PRRT_...` literal. A node id resolves globally, so a fabricated or stale id does not fail, it writes to a real thread on an unrelated repository. This runbook implements [GOVERNANCE.md "Repository Boundaries and Write Safety"](../GOVERNANCE.md#repository-boundaries-and-write-safety): write only to this repo, capture every id from a live query, and never suppress a mutation's output. -List unresolved threads. Use `first: 100` with cursor-based pagination; if `hasNextPage` is true, re-run with `after: ""` to retrieve the next page: +List unresolved threads. Use `first: 100` with cursor-based pagination, and where `hasNextPage` is true, re-run with `after: ""` to retrieve the next page: ```sh gh api graphql -f query=' { - repository(owner: "ptr727", name: "PhotoCleaner") { + repository(owner: "", name: "") { pullRequest(number: ) { reviewThreads(first: 100) { nodes { @@ -156,12 +175,12 @@ gh api graphql -f query=' ' ``` -Reply on a thread, then resolve it. Capture the target thread's id into `$TID` from the listing query above - filter to the thread being answered by its `path`, and guard for an empty result so a mutation never runs on a guessed id. When a file carries more than one unresolved thread, `path` alone is ambiguous and `head -n 1` would pick the wrong one, so narrow by first-comment body - the query already fetches `comments(first: 1)` for this - by adding `and (.comments.nodes[0].body | contains(""))` to the `select`: +Reply on a thread, then resolve it. Capture the target thread's id into `$TID` from the listing query above, filtering to the thread being answered by its `path`, and guard for an empty result so a mutation never runs on a guessed id. When a file carries more than one unresolved thread, `path` alone is ambiguous and `head -n 1` would pick the wrong one, so narrow by first-comment body (the query already fetches `comments(first: 1)` for this) by adding `and (.comments.nodes[0].body | contains(""))` to the `select`: ```sh TID=$(gh api graphql -f query=' { - repository(owner: "ptr727", name: "PhotoCleaner") { + repository(owner: "", name: "") { pullRequest(number: ) { reviewThreads(first: 100) { nodes { id isResolved path comments(first: 1) { nodes { body } } } @@ -190,26 +209,27 @@ mutation($threadId: ID!) { }' -F threadId="$TID" ``` -Issue-level Copilot comments (those in `issues//comments`) have no resolution action - GitHub provides no API or UI to resolve them. Reply if the finding warrants it; no resolution step is needed or possible. +Issue-level Copilot comments (those in `issues//comments`) have no resolution action, since GitHub provides no API or UI to resolve them. Reply if the finding warrants it, but no resolution step is needed or possible. ### PR Edits and Merge-State Gotchas -- **`gh pr edit --title/--body` is broken here.** It touches the deprecated Projects-classic `projectCards` GraphQL field and **exits non-zero without applying the change** (a stale PR description then survives review rounds). Edit the title/body via the API and verify it took: GraphQL `updatePullRequest(input: { pullRequestId, title, body })`, or REST `gh api -X PATCH repos/ptr727/PhotoCleaner/pulls/ -F body=@body.md` (the `@` reads the body from a file - name it explicitly, not the literal `file`). -- **`main`/`develop` use rulesets, not classic branch protection.** The classic protection REST endpoint (`repos/.../branches//protection`) 404s - read the ruleset instead. A `mergeStateStatus` of `BLOCKED` on a green PR is usually just **unresolved review threads** (the ruleset requires thread resolution); resolving them moves it to `CLEAN`. (`BLOCKED` is a `mergeStateStatus` value; don't confuse it with the separate `mergeable` field's `MERGEABLE`/`CONFLICTING`, which reports merge conflicts, not review gates.) -- **Push -> head-SHA read race.** A `headRefOid` read taken immediately after a push can return the **old** head; re-read after the push registers, or a coverage poll evaluates the stale SHA. -- **Copilot is sometimes factually wrong** (e.g. it claimed `actionlint -color` "requires a value" - it is a boolean flag). Verify a finding before fixing; decline with evidence when it is wrong - that is distinct from dismissing a still-present finding as stale. +- **`gh pr edit --title/--body` is broken here.** It touches the deprecated Projects-classic `projectCards` GraphQL field and **exits non-zero without applying the change** (a stale PR description then survives review rounds). Edit the title/body via the API and verify it took: GraphQL `updatePullRequest(input: { pullRequestId, title, body })`, or REST `gh api -X PATCH repos///pulls/ -F body=@body.md` (the `@` reads the body from a file, so name it explicitly, not the literal `file`). +- **`main`/`develop` use rulesets, not classic branch protection.** The classic protection REST endpoint (`repos/.../branches//protection`) 404s, so read the ruleset instead. A `mergeStateStatus` of `BLOCKED` on a green PR is usually just **unresolved review threads** (the ruleset requires thread resolution); resolving them moves it to `CLEAN`. (`BLOCKED` is a `mergeStateStatus` value; don't confuse it with the separate `mergeable` field's `MERGEABLE`/`CONFLICTING`, which reports merge conflicts, not review gates.) +- **Push -> head-SHA read race.** A `headRefOid` read taken immediately after a push can return the **old** head, so re-read after the push registers, or a coverage poll evaluates the stale SHA. +- **Copilot is sometimes factually wrong** (e.g. it claimed `actionlint -color` "requires a value" when it is a boolean flag). Verify a finding before fixing, and decline with evidence when it is wrong, which is distinct from dismissing a still-present finding as stale. Reply-body conventions: - Accepted bug/style fix: include fixing commit SHA and a one-line summary. - Declined style comment: cite the rule (GOVERNANCE.md or the CODESTYLE.md language section) and the existing-tree precedent. - Declined architecture proposal: one-sentence rationale. +- Declined false positive on carried fleet content (a broken-link or dead-cross-reference flag inside byte-locked rule text): cite the "Reviewing Carried Fleet Content" section, since the reference is intentional and the text cannot be edited locally. After the final push, sweep-resolve stale older threads for removed code paths. ## When in Doubt -Read [AGENTS.md](../AGENTS.md) to find the section that governs your change, and [GOVERNANCE.md](../GOVERNANCE.md) for the rule text itself. For code-style rules, [`CODESTYLE.md`](../CODESTYLE.md) (its General section plus the relevant language section) is authoritative. Don't restate any of these files' rules in commit bodies or PR descriptions - keep those focused on the change itself. +Read [AGENTS.md](../AGENTS.md) to find the section that governs your change, and [GOVERNANCE.md](../GOVERNANCE.md) for the rule text itself. For code-style rules, [`CODESTYLE.md`](../CODESTYLE.md) (its General section plus the relevant language section) is authoritative. Don't restate any of these files' rules in commit bodies or PR descriptions, and keep those focused on the change itself. If you find a gap in the governance itself (this file, AGENTS.md, or GOVERNANCE.md is out of date, a rule is missing, something bit this repo and would bite the next), fix it in the governance docs as part of your change rather than only working around it locally. @@ -222,30 +242,35 @@ PhotoCleaner is a .NET 10 console application that processes media files in prep ### Project Structure - **Docker/**: Docker configuration - - `Dockerfile`: Two-stage build (SDK Alpine build -> runtime Alpine final); installs `exiftool` and `ffmpeg` in the final stage + - `Dockerfile`: Two-stage build (SDK Alpine build -> runtime Alpine final) that installs `exiftool` and `ffmpeg` in the final stage - **PhotoCleaner/**: Main console application - `Program.cs`: Entry point with logger setup (Main only) - - `CommandLine.cs`: System.CommandLine implementation for CLI parsing (`process`, `undo`, `import`, `index`, and `trash` subcommands) - - `MediaUtilities.cs`: Shared static utilities - `SupportedExtensions` (FrozenSet), `GetUniqueFileName`, `GetExifToolJsonAsync`, `SetCreateDateAsync`, video/duration constants - - `CommandRunner.cs`: Thin wrapper for command start/complete/error logging + - `CommandLine.cs`: System.CommandLine implementation for CLI parsing (`process`, `undo`, `import`, `index`, `trash`, and `verify` subcommands) + - `MediaUtilities.cs`: Shared static utilities, `SupportedExtensions` (FrozenSet), `GetUniqueFileName`, `GetExifToolJsonAsync`, `SetCreateDateAsync`, video/duration constants + - `CommandRunner.cs`: Thin wrapper for command start/complete/error logging, taking `Func>` and returning the command's exit code + - `ExitCode.cs`: The shared exit-code contract, being `Success` (0), `Error` (1, command could not run), and `Failed` (2, ran to completion with per-file failures) - `DatabaseScope.cs`: Generic async DB lifecycle helper (create, init, dispose) - `TrashDatabaseScope.cs`: Same lifecycle helper pattern for `TrashDatabase` - `FileEnumerator.cs`: Parallel file enumeration returning `(IReadOnlyList, int)` - - `DirectoryCleaner.cs`: Static helper that deletes empty subdirectories under a root (deepest-first; root itself is never deleted); used by `import` and `process` when `--deleteempty` is set - - `ProcessCommand.cs`: Process command orchestration - case conflict resolution, reprocessing loop, result reporting + - `DirectoryCleaner.cs`: Static helper that deletes empty subdirectories under a root (deepest-first, and the root itself is never deleted), used by `import` and `process` when `--deleteempty` is set + - `ProcessCommand.cs`: Process command orchestration, case conflict resolution, reprocessing loop, result reporting - `ImportCommand.cs`: Import command orchestration (formerly `OrganizeCommand`) - `IndexCommand.cs`: Index command orchestration - - `TrashCommand.cs`: Trash command orchestration - fetches trashed asset checksums from Immich API, stores SHA-1 hashes in a `TrashDatabase` + - `TrashCommand.cs`: Trash command orchestration, fetches trashed asset checksums from Immich API, stores SHA-1 hashes in a `TrashDatabase` - `UndoCommand.cs`: Undo command orchestration + - `VerifyCommand.cs`: Verify command orchestration, which enumerates, runs `VerifyTask`, and reports counts - `ProcessTask.cs`: Core file processing pipeline (validation, conversion, metadata) - - `UndoTask.cs`: Undo logic - two-pass algorithm that restores `.bak` files - - `ImportTask.cs`: Import logic - copies (default) or moves supported media files from source into date-based subdirectories under `--outpath`. Inserts a row keyed by SOURCE path into Import.db. Optional SQLite deduplication via `Database`. (Formerly `OrganizeTask`.) + - `UndoTask.cs`: Undo logic, two-pass algorithm that restores `.bak` files + - `ImportTask.cs`: Import logic, copies (default) or moves supported media files from source into date-based subdirectories under `--outpath`. Inserts a row keyed by SOURCE path into Import.db. Optional SQLite deduplication via `Database`. (Formerly `OrganizeTask`.) + - `VerifyTask.cs`: Verification logic, a decode pass that runs Immich's own `MediaRepository` inside `ghcr.io/immich-app/immich-server:release` via `docker run`, batching paths over stdin. Preflights the image before judging any file, so an infrastructure failure exits `Error` rather than marking files invalid + - `ImmichVerifyScript.cs`: The Node script run inside the Immich image, as const strings. Calls Immich's own compiled `MediaRepository`, `defaults`, and `ThumbnailConfig` rather than reimplementing the preview pipeline, so behavior tracks Immich across releases + - `VerifyResult.cs`: The AOT-compatible `ImmichVerifyLine` JSON model and its `ImmichVerifyJsonContext` source-generated context, together forming the container's output protocol - `IndexTask.cs`: Common DB upsert logic used by `process` and `index` commands; `IndexFileAsync` (single-file) returns `(IndexStatus, sha256, sha1, wasProcessed)`; `ExecuteAsync` (batch parallel) returns `(inserted, updated, unchanged, ignored, failed)`. When `options.MarkProcessed` is true, newly inserted rows are marked `is_processed=1` (used by `index --processed` to seed Process.db). - `Database.cs`: SQLite wrapper with a single `files` table (`path` PRIMARY KEY, `sha256`, `sha1`, `file_size`, `mtime_ticks`, `is_processed`); indexes on both hash columns; size/mtime caching via `ResolveHashesAsync` to skip rehashing unchanged files. Every write computes both sha256 and sha1 in a single read pass; both columns are non-null - `TrashDatabase.cs`: Simple SQLite wrapper for Immich trash hashes; single `trash_hashes` table (`sha1` PRIMARY KEY); used by `trash`, `import`, and `process` commands - `ImmichApiModels.cs`: AOT-compatible JSON models for Immich API (`ImmichSearchRequest`, `ImmichSearchResponse`, `ImmichAssetDto`) with `ImmichJsonContext` source generation - `DateFromPath.cs`: Static utility class for date inference from filenames/paths - - `ExifToolJson.cs`: JSON model for ExifTool metadata + - `ExifToolJson.cs`: JSON model for ExifTool metadata, including the `ExifTool:Validate` verdict and `ParseValidate` which splits it into error and warning counts - `SkippedExtensionTracker.cs`: Thread-safe tracker for unknown file extensions skipped during processing; used by all commands that filter by `MediaUtilities.SupportedExtensions` (`process`, `import`, `index`) - `HttpClientFactory.cs`: Polly resilience pipeline (retry, circuit breaker) and `SocketsHttpHandler` connection pooling - `AssemblyInfo.cs`: Assembly metadata (app name, version) used by `HttpClientFactory` for User-Agent header @@ -263,6 +288,7 @@ PhotoCleaner is a .NET 10 console application that processes media files in prep - `TrashDatabaseTests.cs`: TrashDatabase tests (8 tests) - `TrashCommandTests.cs`: TrashCommand tests with mock HTTP handler (6 tests) - `DirectoryCleanerTests.cs`: DirectoryCleaner static helper tests (6 tests) + - `VerifyTaskTests.cs`: Verify protocol parsing and script-contract tests (10 tests) ### Core Processing Pipeline @@ -277,11 +303,16 @@ if (!RenameMismatchedMimeExtensions() || !WarnDngVersion()) ``` +Before that chain runs, `CheckExifToolValidation` acts on the `ExifTool:Validate` verdict that +rides along with the metadata read. Only an error count fails the file (`ProcessResult.Invalid`). +Warnings are logged at debug level, because roughly three quarters of healthy files in a real +collection carry at least one. + ### State Management Pattern - **Primary Constructor Parameters**: Command and task classes use C# 12 primary constructors. All task classes take `CommandLine.Options options` as their first parameter, plus any non-option runtime params (e.g., `Database`, shared collections). Command classes take `(CommandLine.Options options, CancellationToken cancellationToken)` and pass `options` directly to task constructors. -- **Command/Task Separation**: Command classes (e.g., `ProcessCommand`) handle orchestration (file enumeration, DB lifecycle, result logging); task classes (e.g., `ProcessTask`) handle per-file business logic -- **Composable Infrastructure**: `CommandRunner`, `DatabaseScope`, and `FileEnumerator` are static helpers freely composed by command classes - no inheritance hierarchy +- **Command/Task Separation**: Command classes (e.g., `ProcessCommand`) handle orchestration (file enumeration, DB lifecycle, result logging), while task classes (e.g., `ProcessTask`) handle per-file business logic +- **Composable Infrastructure**: `CommandRunner`, `DatabaseScope`, and `FileEnumerator` are static helpers freely composed by command classes, no inheritance hierarchy - **Shared Collections**: `ConcurrentBag` for file names, `ConcurrentDictionary` for unknown extensions with case-insensitive comparison - **Parallel Processing**: Files processed using `Parallel.ForEachAsync` with `MaxDegreeOfParallelism` - **External Tool Integration**: Uses `CliWrap` for all external command execution (exiftool, ffmpeg, ffprobe) @@ -293,7 +324,7 @@ if (!RenameMismatchedMimeExtensions() ```csharp BufferedCommandResult result = await Cli.Wrap("exiftool") - .WithArguments(["-groupNames", "-json", _fileInfo.FullName]) + .WithArguments(["-groupNames", "-json", "-validate", "-all", _fileInfo.FullName]) .ExecuteBufferedAsync(); ``` @@ -304,7 +335,7 @@ BufferedCommandResult result = await Cli.Wrap("exiftool") ### Media File Processing Conventions - **FrozenSet Extensions**: Define supported extensions as `FrozenSet` with `StringComparer.OrdinalIgnoreCase` (e.g., `s_remuxExtensions`, `s_jpegExtensions`) -- **Case-Insensitive Matching**: Use FrozenSet `.Contains()` directly without `.ToLower()` - comparer handles case-insensitivity +- **Case-Insensitive Matching**: Use FrozenSet `.Contains()` directly without `.ToLower()`, comparer handles case-insensitivity - **File Type Categorization**: Group operations by file type requirements (remux vs re-encode vs audio-only) - **Single-Pass Optimizations**: Prefer single-loop iterations with early exit over multiple LINQ passes - **Skipped Extension Tracking**: Commands that filter files by `MediaUtilities.SupportedExtensions` pass a shared `SkippedExtensionTracker` instance to their task classes. The tracker collects unknown extensions (thread-safe via `Track()`), and the command calls `LogWarnings()` after processing to log them sorted. Used by `process`, `import`, and `index` commands. @@ -320,7 +351,7 @@ BufferedCommandResult result = await Cli.Wrap("exiftool") ### Date Inference System (DateFromPath.cs) - **Static Internal Methods**: All methods are `internal static` for testability with `InternalsVisibleTo` -- **DateFromPath.InferCreatedDate()**: Main entry point - tries filename first, then path fallback +- **DateFromPath.InferCreatedDate()**: Main entry point, tries filename first, then path fallback - **DateFromPath.ExtractDateFromFilename()**: Supports multiple filename patterns: - `YYYYMMDD_HHMMSS` format (e.g., `20210502_200152957_iOS-1747.jpg`) - `YYYYMMDD` format (e.g., `EX_20030219_3378.jpg`) @@ -332,16 +363,16 @@ BufferedCommandResult result = await Cli.Wrap("exiftool") ### Command Line Interface (CommandLine.cs) - **System.CommandLine Integration**: Uses modern .NET command line parsing -- **Five subcommands**: `process`, `undo`, `import`, `index`, `trash` - each with their own option set +- **Six subcommands**: `process`, `undo`, `import`, `index`, `trash`, `verify`, each with their own option set - **Required `--path` Parameter**: Single directory path using `Option`. Validated with `AcceptExistingOnly()` -- **Optional `--dryrun` Flag**: Non-destructive preview mode (process, undo, import - not index) +- **Optional `--dryrun` Flag**: Non-destructive preview mode (process, undo, import, not index) - **Optional `--threads` Parameter**: Controls parallel processing degree with `DefaultValueFactory = _ => Math.Min(Environment.ProcessorCount, 4)`. Validated to be > 0 and <= Environment.ProcessorCount using `Validators.Add()` (process, import, index) -- **Optional `--skipbackup` Flag** (process only): Skips all `.bak` file creation - originals are deleted/overwritten in-place. Logs a warning at startup. Disables undo. +- **Optional `--skipbackup` Flag** (process only): Skips all `.bak` file creation, originals are deleted/overwritten in-place. Logs a warning at startup. Disables undo. - **Optional `--deleteempty` Flag** (process, import): After the command completes, deletes empty child subdirectories from the target directory (deepest first; target root is never deleted). For `process` the target is `--path` (operated on in-place); for `import` it is `--outpath`. Implemented by `DirectoryCleaner.DeleteEmptyDirectories(root, dryRun)`. - **`import` subcommand** (formerly `organize`): Copies (default) or moves supported media files from `--path` sources into `--outpath/date/filename` directory structure. Date comes from EXIF metadata (falls back to `DateTime.MinValue` -> `"0001/01/01"` bucket when absent). `--format` (default `"yyyy/MM/dd"`) controls subdirectory naming and is validated as a date-only format (no time components). Uses `GetUniqueFileName` for collision handling (`foo_1.jpg` etc.). Parallel via `--threads` (same as `process`). `--deleteempty` (default `false`) deletes empty child subdirectories from `--outpath` after all files are imported. `--move` (default `false`) moves files instead of copying. `--tagpath` (default `false`) splits the source sub-directory path into tokens and writes each token as an `XMP:Subject` tag on the destination file using exiftool; filtered by `s_exiftoolWriteExtensions` (`.3gp`, `.arw`, `.cr2`, `.dng`, `.gif`, `.heic`, `.heif`, `.jpeg`, `.jpg`, `.mov`, `.mp4`, `.nef`, `.orf`, `.png`, `.psd`, `.rw2`, `.tif`, `.tiff`) checked via `meta.FileTypeExtension`; uses `-XMP:Subject-= / -XMP:Subject+=` to prevent duplicates while preserving existing tags. `--tags ` (optional) applies explicit comma-separated `XMP:Subject` tags to every imported file. `--datepath` (default `false`) infers the EXIF creation date from the source file path when no date is already embedded; applies the date to the destination file before restoring mtime. **`--db ` (Import.db) is the source-side dedup DB**: rows are keyed by `path = source_path` (NOT dest path) and hold the source file's hash/size/mtime. On each source file, import calls `GetByPathAsync(source_path)` for source-side hash caching, then `Sha256ExistsAsync(source_hash)` to skip already-imported sources. New imports insert a row at the source path. **No command outside `import` writes to source-keyed rows**, so dedup cannot be clobbered by later runs of `process`/`index`. `--trashdb ` skips files whose **source-file** SHA-1 is in Trash.db (Limitation: when import rewrites the dest via `--tags`/`--tagpath`/`--datepath`, the dest SHA-1 differs from the source SHA-1; Immich stored the dest SHA-1 from a prior upload, so the trash match is missed here and is caught later by `process --trashdb`). `--skipdb ` skips files whose SHA-256 matches a reference DB (read-only). Cross-collection dedup is typically implemented by pointing `--skipdb` at another collection's Import.db. `--rehash` forces recomputation of all hashes ignoring the size/mtime cache. -- **`index` subcommand**: Iterates all files in `--path`, upserts each into the `files` DB table via `IndexTask.ExecuteAsync` (insert new, update if hash changed, skip unchanged). `--db ` is **required**. No `--dryrun` (always writes to DB). Supports `--threads` and `--rehash`. `--processed` (optional) marks newly-INSERTED rows with `is_processed = 1`; useful when seeding a Process.db from existing files so `process` treats them as already-done. The flag does not flip the flag on existing rows. Reports `inserted`/`updated`/`unchanged`/`ignored`/`failed` counts. +- **`index` subcommand**: Iterates all files in `--path`, upserts each into the `files` DB table via `IndexTask.ExecuteAsync` (insert new, update if hash changed, skip unchanged). `--db ` is **required**. No `--dryrun` (always writes to DB). Supports `--threads` and `--rehash`. `--processed` (optional) marks newly-INSERTED rows with `is_processed = 1`, which is useful when seeding a Process.db from existing files so `process` treats them as already-done. The flag does not flip the flag on existing rows. Reports `inserted`/`updated`/`unchanged`/`ignored`/`failed` counts. - **`trash` subcommand**: Syncs trashed asset checksums from an Immich server into a local SQLite trash database. `--url` (Immich server URL, required), `--trashdb ` (trash database, required), and the API key supplied by exactly one of `--apikey` (inline) or `--apikey-file` (path to a file whose trimmed contents are the key). The two API-key options are mutually exclusive and exactly one must be provided; `--apikey-file` must reference an existing, non-empty, readable file (existence enforced by an option validator, non-empty/readable by a command-level validator; read failures are translated to validation errors, never thrown). The key is resolved at parse time by `CommandLine.ResolveApiKey`/`ReadApiKeyFile` (file contents preferred and `.Trim()`-med) and flows into `Options.ImmichApiKey`. Uses `POST /api/search/metadata` with `trashedAfter` to fetch all trashed assets, converts Base64 SHA-1 checksums to hex, and inserts them via `INSERT OR IGNORE`. Full sync (idempotent, append-only). No `--dryrun`. -- **`--trashdb` Flag** (import, process): SQLite database file with Immich trash hashes (synced by `trash`). In `import`, files matching the trash DB are skipped (this prevents re-importing photos the user trashed in Immich); the check is against the **source-file** SHA-1, so files whose dest SHA-1 was mutated by `import` itself (`--tags`/`--tagpath`/`--datepath`) will not match here even though Immich stored the mutated SHA-1 - `process --trashdb` catches those on the next pass. In `process`, matching files are **deleted from disk and from Process.db** before the per-file processing pipeline runs (cleanup of files trashed in Immich after upload, and the safety net for the import source-vs-dest SHA-1 drift). The Trash.db check is the durable safety net beyond Immich's ~30-day trash retention. +- **`--trashdb` Flag** (import, process): SQLite database file with Immich trash hashes (synced by `trash`). In `import`, files matching the trash DB are skipped (this prevents re-importing photos the user trashed in Immich); the check is against the **source-file** SHA-1, so files whose dest SHA-1 was mutated by `import` itself (`--tags`/`--tagpath`/`--datepath`) will not match here even though Immich stored the mutated SHA-1, `process --trashdb` catches those on the next pass. In `process`, matching files are **deleted from disk and from Process.db** before the per-file processing pipeline runs (cleanup of files trashed in Immich after upload, and the safety net for the import source-vs-dest SHA-1 drift). The Trash.db check is the durable safety net beyond Immich's ~30-day trash retention. - **Optional `--skipdb` Flag** (import only): SQLite database of files to skip (read-only SHA-256 check). Files whose SHA-256 matches a record in this DB are skipped without being recorded. Use this to skip files already present in another collection. - **Optional `--rehash` Flag** (process, import, index): Forces SHA-256 recomputation for every file, ignoring the size/mtime cache. SHA-1 is also recomputed when `--trashdb` is in use. Useful after filesystem operations that preserve mtime but change content. - **Optional `--duration` Flag** (process only): Overrides `ShortVideoDuration` (default `1.0`s). Videos in a live-photo-compatible format whose duration is <= this value are always deleted. Must be `> 0`. Stored in `CommandLine.Options.ShortVideoDuration` and read by `DeleteLivePhotosAsync`. @@ -379,18 +410,18 @@ See [`CODESTYLE.md`](../CODESTYLE.md) for build requirements, formatting command ### Video Conversion Logic - **Three-tier approach**: Remux (.mts, .m2ts, .mkv) -> Re-encode (.wmv, .avi, .3gp, .gif) -> Audio-only (.mov/.mp4 with PCM) -- **Backup Strategy**: Original files renamed to `.bak` extension after successful conversion; `BackupFile()` returns the backup path. A `{backup}.out` companion file (e.g. `img.gif.bak.out`) is written alongside the backup containing the full output path - this is needed when `GetUniqueFileName` appended a counter suffix (e.g. `img_1.mp4`) because the canonical name was already taken. When `options.SkipBackup` is true, no `.bak` or `.bak.out` files are created - the original is deleted after conversion. -- **Metadata Preservation**: After every ffmpeg conversion, `exiftool -TagsFromFile -all:all -overwrite_original` copies all source metadata to the output file. `ffmpeg -map_metadata` is not used - it is unreliable for Apple QuickTime-specific tags (e.g. `ContentIdentifier` in the `mdta`/`keys` atom). `TagsFromFile` handles cross-format date mapping, so no separate date-setting step is needed after conversion. +- **Backup Strategy**: Original files renamed to `.bak` extension after successful conversion, and `BackupFile()` returns the backup path. A `{backup}.out` companion file (e.g. `img.gif.bak.out`) is written alongside the backup containing the full output path, this is needed when `GetUniqueFileName` appended a counter suffix (e.g. `img_1.mp4`) because the canonical name was already taken. When `options.SkipBackup` is true, no `.bak` or `.bak.out` files are created, and the original is deleted after conversion. +- **Metadata Preservation**: After every ffmpeg conversion, `exiftool -TagsFromFile -all:all -overwrite_original` copies all source metadata to the output file. `ffmpeg -map_metadata` is not used, it is unreliable for Apple QuickTime-specific tags (e.g. `ContentIdentifier` in the `mdta`/`keys` atom). `TagsFromFile` handles cross-format date mapping, so no separate date-setting step is needed after conversion. - **Re-queue Pattern**: Converted files are added back to processing queue for validation ### Live Photo Detection -- **Short videos** (duration <= `options.ShortVideoDuration`, default `1.0s`; overridable via `--duration`): always deleted regardless of companion file +- **Short videos** (duration <= `options.ShortVideoDuration`, default `1.0s`, overridable via `--duration`): always deleted regardless of companion file - **Companion file search** (`FindCompanionImagePath()`): looks for a HEIC/JPG/JPEG file by: 1. Direct basename match (`IMG_1234.mov` -> `IMG_1234.heic`) - 2. Basename minus `_hevc` suffix (`IMG_1234_HEVC.mov` -> `IMG_1234.heic`) - new iPhone naming + 2. Basename minus `_hevc` suffix (`IMG_1234_HEVC.mov` -> `IMG_1234.heic`), the newer iPhone naming - **ContentIdentifier confirmation**: a candidate pair is only deleted when both files expose a `ContentIdentifier` tag that matches exactly. If either file lacks the tag, or the tags differ, the video is kept. There is no fallback to name-only deletion. -- **Long videos** (>= `LiveVideoDuration` = 4.0s): always kept even with a matching companion; a warning is logged +- **Long videos** (>= `LiveVideoDuration` = 4.0s): always kept even with a matching companion, and a warning is logged ### Undo Architecture (UndoTask.cs) @@ -404,9 +435,9 @@ See [`CODESTYLE.md`](../CODESTYLE.md) for build requirements, formatting command - Derived base: delete current file + all its backups - Non-derived base: delete current file if present, restore `X.bak` -> `X`; then locate the derived conversion output: if `X.bak.out` companion exists read the explicit output path from it and delete that file (handles uniquified names like `img_1.mp4`); otherwise fall back to checking whether `stem.mp4` exists and has no backup (legacy single-run heuristic) - **Internal static helpers** (testable via `InternalsVisibleTo`): - - `IsBackupFile(path)` - matches `.bak\d*$` - - `IsNumberedBackup(path)` - matches `.bak\d+$` - - `GetBackupBase(path)` - strips the `.bak\d*` suffix + - `IsBackupFile(path)`: matches `.bak\d*$` + - `IsNumberedBackup(path)`: matches `.bak\d+$` + - `GetBackupBase(path)`: strips the `.bak\d*` suffix - **Dry run**: logs all intended operations but performs no file I/O - **Known limitation**: extension renames to a previously non-existent filename create no backup and cannot be undone diff --git a/.vscode/launch.json b/.vscode/launch.json index 8925289..48d7908 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -100,6 +100,45 @@ "console": "internalConsole", "stopAtEntry": false }, + { + "name": "Verify", + "type": "coreclr", + "request": "launch", + "preLaunchTask": ".NET Build", + "program": "${workspaceFolder}/PhotoCleaner/bin/Debug/net10.0/PhotoCleaner.dll", + "args": [ + "verify", + "--path=/data/media/PhotoCleaner/Originals", + "--db=/data/media/PhotoCleaner/Verify.db", + "--rehash=false", + "--reprocess=true", + "--threads=4", + "--loglevel=verbose", + "--logfile=/data/media/PhotoCleaner/Verify.log", + "--logclear=true" + ], + "cwd": "${workspaceFolder}/PhotoCleaner/bin/Debug/net10.0", + "console": "internalConsole", + "stopAtEntry": false + }, + { + "name": "Verify No Database", + "type": "coreclr", + "request": "launch", + "preLaunchTask": ".NET Build", + "program": "${workspaceFolder}/PhotoCleaner/bin/Debug/net10.0/PhotoCleaner.dll", + "args": [ + "verify", + "--path=/data/media/PhotoCleaner/Originals", + "--threads=4", + "--loglevel=verbose", + "--logfile=/data/media/PhotoCleaner/Verify.log", + "--logclear=true" + ], + "cwd": "${workspaceFolder}/PhotoCleaner/bin/Debug/net10.0", + "console": "internalConsole", + "stopAtEntry": false + }, { "name": "Undo", "type": "coreclr", diff --git a/AUDIT.md b/AUDIT.md index 4d3b68d..ffbeb7b 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -6,7 +6,7 @@ The audit is read-only: it diffs live state against the committed baseline and r ## Scope -This is a release-model repo: the self-audit covers the `main` and `develop` rulesets, general repository settings, and secret names. Code-project conformance (analyzers, tests, coverage, publish workflows) is CI's job and the fleet hub's fleet-wide audit's, not this self-audit's - see [GOVERNANCE.md "Branching Model"][governance-branching-model] for the model this baseline encodes. +This is a release-model repo: the self-audit covers the `main` and `develop` rulesets, general repository settings, and secret names. Code-project conformance (analyzers, tests, coverage, publish workflows) is CI's job and the fleet hub's fleet-wide audit's, not this self-audit's. See [GOVERNANCE.md "Branching Model"][governance-branching-model] for the model this baseline encodes. ## General Settings @@ -35,7 +35,7 @@ for b in develop main; do done ``` -The result must be exactly two rulesets named `develop` and `main` - a missing ruleset or a divergent payload is a **defect**; a duplicate or stray ruleset is a **drift finding**. +The result must be exactly two rulesets named `develop` and `main`. A missing ruleset or a divergent payload is a **defect**, and a duplicate or stray ruleset is a **drift finding**. ## Secrets diff --git a/CODESTYLE.md b/CODESTYLE.md index 85e16f1..6962f21 100644 --- a/CODESTYLE.md +++ b/CODESTYLE.md @@ -1,6 +1,6 @@ # Code Style and Formatting Rules -This is the single code-style guide for the fleet. The **General** section applies to every language. Each **language section** (.NET, Python) is self-contained: a repo follows only the section(s) for the languages it ships and ignores the rest. A repo keeps the whole file rather than trimming it - an unused-language section costs nothing, the same whole-file model as [`.editorconfig`][root], whose inert `[*.cs]` block a non-.NET repo keeps. +This is the single code-style guide for the fleet. The **General** section applies to every language. Each **language section** (.NET, Python) is self-contained: a repo follows only the section(s) for the languages it ships and ignores the rest. A repo keeps the whole file rather than trimming it. An unused-language section costs nothing, the same whole-file model as [`.editorconfig`][root], whose inert `[*.cs]` block a non-.NET repo keeps. Cross-cutting *process* rules (PR titles, branching, US English, markdown style, comments philosophy, workflow YAML, PR review etiquette) live in [GOVERNANCE.md][governance] and are not repeated here. @@ -10,21 +10,21 @@ These rules apply to every language in the repo. ### Tooling Names and Casing -Use each tool's official casing in task labels, docs, and prose - `.NET` (not `.Net`), `CSharpier`, `ruff`, `pyright`, `uv`. Don't invent personal variants. +Use each tool's official casing in task labels, docs, and prose: `.NET` (not `.Net`), `CSharpier`, `ruff`, `pyright`, `uv`. Don't invent personal variants. ### Clean-Compile Verification -Each language defines a **clean-compile** verification - the combination of build, formatter, linter, and code-analysis tools that must report clean before a commit. It is exposed as one or more **named** VS Code tasks (or, where a language ships no tasks, documented commands), and those definitions are the same across the fleet. The concrete names live in each language section below. +Each language defines a **clean-compile** verification: the combination of build, formatter, linter, and code-analysis tools that must report clean before a commit. It is exposed as one or more **named** VS Code tasks (or, where a language ships no tasks, documented commands), and those definitions are the same across the fleet. The concrete names live in each language section below. -- **Run it after every code change.** The relevant language's clean-compile must pass before you commit; CI runs the same checks as a backstop. -- **The named task definition is the canonical spec** - its exact command sequence, arguments, and strictness. You may run it through the VS Code task **or** by invoking the equivalent native commands directly; either is fine **only if the sequence, arguments, and strictness match exactly**. No shortcuts and no more-lenient options (for example, never drop `--verify-no-changes` or loosen a `--severity`). -- **A local commit/pre-commit gate is the repo's choice.** No single hook runner fits every language (a `dotnet`-tool runner like Husky.Net suits .NET but not Python), so none is mandated - but that is **not** a recommendation against commit gates. CI is the authoritative backstop regardless; a local gate is an additive convenience a repo may wire and keep - Husky.Net (and `dotnet husky run` as a style step) for .NET, `pre-commit` for Python. Keeping a working gate is not drift. +- **Run it after every code change.** The relevant language's clean-compile must pass before you commit, and CI runs the same checks as a backstop. +- **The named task definition is the canonical spec** - its exact command sequence, arguments, and strictness. You may run it through the VS Code task **or** by invoking the equivalent native commands directly, and either is fine **only if the sequence, arguments, and strictness match exactly**. No shortcuts and no more-lenient options (for example, never drop `--verify-no-changes` or loosen a `--severity`). +- **A local commit/pre-commit gate is the repo's choice.** No single hook runner fits every language (a `dotnet`-tool runner like Husky.Net suits .NET but not Python), so none is mandated, but that is **not** a recommendation against commit gates. CI is the authoritative backstop regardless, and a local gate is an additive convenience a repo may wire and keep: Husky.Net (and `dotnet husky run` as a style step) for .NET, `pre-commit` for Python. Keeping a working gate is not drift. ### Analyzer Diagnostics and Suppressions -- **A new port is not a license to silence diagnostics.** Brownfield / just-ported status never justifies relaxing analyzer or linter severities or muting newly surfaced warnings - fix them. (The only brownfield allowance is the one-time git-signing / line-ending migration described in [GOVERNANCE.md][governance] and [README.md][readme], which has nothing to do with code analysis.) +- **A new port is not a license to silence diagnostics.** Brownfield / just-ported status never justifies relaxing analyzer or linter severities or muting newly surfaced warnings. Fix them. (The only brownfield allowance is the one-time git-signing / line-ending migration described in [GOVERNANCE.md][governance] and [README.md][readme], which has nothing to do with code analysis.) - **Suppress only genuine false-positives or deliberate, documented exceptions**, always at the **narrowest scope that fits**, in this order of preference: - 1. An **in-code annotation on the specific symbol**, with a justification - the language's attribute/comment form, never a blanket pragma spanning a region. + 1. An **in-code annotation on the specific symbol**, with a justification, in the language's attribute/comment form, never a blanket pragma spanning a region. 2. The **owning project's local config** when the exception is project-wide for one project (e.g. a test project's own `.editorconfig` / `pyproject.toml`). 3. The **root / shared config** only when the suppression is genuinely applicable to **every** project in the repo. - **Never blanket-relax a batch of rules project-wide** to get a port to build. The per-language mechanics (which attribute, which config key) are in each language section. @@ -33,9 +33,9 @@ Each language defines a **clean-compile** verification - the combination of buil These apply repo-wide, in every directory: -1. **Markdown linting**: All `.md` files must be lint-clean (error and warning free) via the VS Code `markdownlint` extension. [`.markdownlint-cli2.jsonc`][markdownlint-cli2] at the repo root is the single source of truth - the davidanson `markdownlint` extension and a command-line `markdownlint-cli2` run both read it, so the IDE and CLI stay in lock-step. Rules it deliberately disables (e.g. `MD013` line-length) are **intentional** - do not "fix" them. `MD033` inline HTML stays **enabled**: HTML comments are permitted (markdownlint does not flag them), HTML elements are flagged, and anything with a native markdown equivalent uses the markdown. Fix violations at the source rather than disabling rules. -2. **Spelling**: All spelling must be clean via the CSpell VS Code integration; words must be correctly spelled in **US English** (the repo-wide convention - see [GOVERNANCE.md][governance]). The shared `cspell.json` sets `"language": "en-US"` so British spellings are flagged - a bare `"en"` accepts both US and British and silently passes the wrong spelling. Project-specific terms go in the shared `cspell.json` `words` list - it is the single source of truth the extension, CLI, and CI all read. The `.code-workspace` must **not** carry its own `cspell.words`/`cSpell.words` block; when externalizing words into `cspell.json`, delete any word list left in the workspace (a leftover one duplicates the list and silently drifts). -3. **Spelling CI scope**: The enforced CI spell-check gate covers **`README.md` and `HISTORY.md` only** - these are the files every repo visitor sees, so they must be clean. It is deliberately **not** all `**/*.md`: repos carry many markdown files full of technical terms, and gating every one of them would mean endlessly padding `cspell.json` just to keep CI green. Broad, live spell-checking across any file (source, markdown, text) is the **cspell editor extension's** job, so typos still surface to whoever is editing. A repo owner **may** widen their own CI file list, but README + HISTORY are the default; keep the CI workflow, the `Lint: Spelling` VS Code task, and the GOVERNANCE.md cspell one-liner on the same file list. The list is explicit (not a glob), so a repo that ships no `HISTORY.md` (e.g. one with no changelog) must drop it from all three surfaces and gate on `README.md` alone - cspell errors on a listed file that does not exist. Markdown *linting* (item 1) stays repo-wide `**/*.md` - it does not choke on technical terms. +1. **Markdown linting**: All `.md` files must be lint-clean (error and warning free) via the VS Code `markdownlint` extension. [`.markdownlint-cli2.jsonc`][markdownlint-cli2] at the repo root is the single source of truth, since the davidanson `markdownlint` extension and a command-line `markdownlint-cli2` run both read it, so the IDE and CLI stay in lock-step. Rules it deliberately disables (e.g. `MD013` line-length) are **intentional**, so do not "fix" them. `MD033` inline HTML stays **enabled**: HTML comments are permitted (markdownlint does not flag them), HTML elements are flagged, and anything with a native markdown equivalent uses the markdown. Fix violations at the source rather than disabling rules. +2. **Spelling**: All spelling must be clean via the CSpell VS Code integration, and words must be correctly spelled in **US English** (the repo-wide convention, per [GOVERNANCE.md][governance]). The shared `cspell.json` sets `"language": "en-US"` so British spellings are flagged, where a bare `"en"` accepts both US and British and silently passes the wrong spelling. Project-specific terms go in the shared `cspell.json` `words` list, the single source of truth the extension, CLI, and CI all read. The `.code-workspace` must **not** carry its own `cspell.words`/`cSpell.words` block, and when externalizing words into `cspell.json`, delete any word list left in the workspace (a leftover one duplicates the list and silently drifts). +3. **Spelling CI scope**: The enforced CI spell-check gate covers **`README.md` and `HISTORY.md` only**, because these are the files every repo visitor sees, so they must be clean. It is deliberately **not** all `**/*.md`: repos carry many markdown files full of technical terms, and gating every one of them would mean endlessly padding `cspell.json` just to keep CI green. Broad, live spell-checking across any file (source, markdown, text) is the **cspell editor extension's** job, so typos still surface to whoever is editing. A repo owner **may** widen their own CI file list, but README + HISTORY are the default; keep the CI workflow, the `Lint: Spelling` VS Code task, and the GOVERNANCE.md cspell one-liner on the same file list. The list is explicit (not a glob), so a repo that ships no `HISTORY.md` (e.g. one with no changelog) must drop it from all three surfaces and gate on `README.md` alone, since cspell errors on a listed file that does not exist. Markdown *linting* (item 1) stays repo-wide `**/*.md`, which does not choke on technical terms. 4. **`README.md` and `HISTORY.md` share the same header**: the `# PhotoCleaner` title and the one-line description under it match exactly in both files, so a reader landing on the changelog sees the same project identity. A change to one is applied to the other in the same commit, and the description is also what the GitHub About panel and the Docker Hub short description carry (see [GOVERNANCE.md][governance] "Repository Details"). ## .NET @@ -52,15 +52,15 @@ This is the style guide for any **.NET projects** in this repo. 1. **The `.NET Format` clean-compile task** (see [Clean-Compile Verification][clean-compile-verification]) - The .NET clean-compile is the **`.NET Format`** VS Code task, which chains `CSharpier Format` -> `.NET Build` -> `dotnet format style --verify-no-changes`. These three task definitions are carried verbatim in [`.vscode/tasks.json`][vscode-tasks]. - - After any code change it must pass before commit. Run the `.NET Format` task. To run it natively instead, reproduce that task chain from [`.vscode/tasks.json`][vscode-tasks] exactly - `CSharpier Format`, then `.NET Build`, then the `dotnet format style --verify-no-changes --severity=info ...` verify - without dropping or loosening any argument (tasks.json is the canonical command spec). Bare `dotnet format` alone, skipping CSharpier or the build, is not sufficient. + - After any code change it must pass before commit. Run the `.NET Format` task. To run it natively instead, reproduce that task chain from [`.vscode/tasks.json`][vscode-tasks] exactly (`CSharpier Format`, then `.NET Build`, then the `dotnet format style --verify-no-changes --severity=info ...` verify) without dropping or loosening any argument (tasks.json is the canonical command spec). Bare `dotnet format` alone, skipping CSharpier or the build, is not sufficient. 2. **Analyzer configuration** - `true` with `latest-all` and `All` (full analyzer set enabled) - - `true` - any diagnostic surfaced as a warning fails the build, so it must be fixed or deliberately suppressed, not left to accumulate (see [Analyzer Diagnostics and Suppressions][analyzer-diagnostics-and-suppressions]) + - `true`, so any diagnostic surfaced as a warning fails the build, so it must be fixed or deliberately suppressed, not left to accumulate (see [Analyzer Diagnostics and Suppressions][analyzer-diagnostics-and-suppressions]) 3. **CI lint backstop** - CI runs the clean-compile checks on every PR as the authoritative backstop - - Git hooks are optional; a repo may wire a local runner (Husky.Net) for pre-commit enforcement, but CI is the gate that matters + - Git hooks are optional, and a repo may wire a local runner (Husky.Net) for pre-commit enforcement, but CI is the gate that matters #### The Full Post-Change Set @@ -74,14 +74,14 @@ The clean-compile task above is necessary and not sufficient. After every code c Shared MSBuild configuration is centralized at the repository root, never duplicated per project: -- **`Directory.Build.props`** carries the properties every project shares - the analyzer set and `TreatWarningsAsErrors` from the Zero Warnings Policy above, plus `LangVersion`, `TargetFramework` where uniform, and any repo-wide build metadata. A csproj carries only what is genuinely project-specific (`OutputType`, `IsPackable`, project references). +- **`Directory.Build.props`** carries the properties every project shares: the analyzer set and `TreatWarningsAsErrors` from the Zero Warnings Policy above, plus `LangVersion`, `TargetFramework` where uniform, and any repo-wide build metadata. A csproj carries only what is genuinely project-specific (`OutputType`, `IsPackable`, project references). - **`Directory.Packages.props`** enables central package management (`ManagePackageVersionsCentrally` true): every dependency version is declared once as a `PackageVersion` item, and a csproj's `PackageReference` items are versionless. One file to review on a bump, one Dependabot surface, and no version skew between projects. -A repo whose projects still carry per-project analyzer settings or versioned `PackageReference` items is drifted - move the shared property or version up to the root file rather than editing it in place. +A repo whose projects still carry per-project analyzer settings or versioned `PackageReference` items is drifted, so move the shared property or version up to the root file rather than editing it in place. #### Build Tasks -Available VS Code tasks (run them from VS Code's task runner - **Terminal -> Run Task** - or an agent's task-running tool). The three clean-compile tasks below are carried verbatim; a repo adds its own convenience tasks (tool updates, dependency upgrades, benchmarks) on top: +Available VS Code tasks (run them from VS Code's task runner, **Terminal -> Run Task**, or an agent's task-running tool). The three clean-compile tasks below are carried verbatim, and a repo adds its own convenience tasks (tool updates, dependency upgrades, benchmarks) on top: - `.NET Build`: Build with diagnostic verbosity *(clean-compile)* - `CSharpier Format`: Auto-format code with CSharpier *(clean-compile)* @@ -99,7 +99,7 @@ Available VS Code tasks (run them from VS Code's task runner - **Terminal -> Run - `dotnet-outdated-tool`: Dependency update checks - Nerdbank.GitVersioning: Version management -CI is the authoritative lint backstop. Local pre-commit hooks are optional - wire Husky.Net (or another runner) if you want local enforcement. +CI is the authoritative lint backstop. Local pre-commit hooks are optional, so wire Husky.Net (or another runner) if you want local enforcement. #### Editor Baseline @@ -127,7 +127,7 @@ Note: Code snippets are illustrative examples only. Replace namespaces/types to - Top-level statements for console apps - Pattern matching over traditional checks - Collection expressions when types loosely match - - Extension methods - the classic `this`-parameter form, or an `extension() { ... }` block on C# 14+ + - Extension methods, in the classic `this`-parameter form or an `extension() { ... }` block on C# 14+ - Implicit object creation when type is apparent - Range and index operators @@ -206,7 +206,7 @@ Note: Code snippets are illustrative examples only. Replace namespaces/types to - YAML files: 2 spaces - JSON files: 4 spaces -5. **Line endings**: not specified here - governed per repo by `.editorconfig` / `.gitattributes` per the [GOVERNANCE.md][governance] "Line Endings" section. +5. **Line endings**: not specified here, but governed per repo by `.editorconfig` / `.gitattributes` per the [GOVERNANCE.md][governance] "Line Endings" section. 6. **`#region`**: Do not use regions. Prefer logical file/folder/namespace organization. 7. **Member ordering (StyleCop SA1201)**: const -> static readonly -> static fields -> instance readonly fields -> instance fields -> constructors -> public (events -> properties -> indexers -> methods -> operators) -> non-public in same order -> nested types @@ -262,13 +262,13 @@ Follow the scope hierarchy in [Analyzer Diagnostics and Suppressions][analyzer-d #### Error Handling and Logging -1. **Structured logging**: Use structured message templates - Serilog is the **application's** concrete backend; a library never references it (see item 2) +1. **Structured logging**: Use structured message templates. Serilog is the **application's** concrete backend, and a library never references it (see item 2) ```csharp logger.LogError(exception, "{Function}", function); ``` -2. **Libraries log through abstractions, never a concrete backend.** A NuGet **library** depends only on `Microsoft.Extensions.Logging.Abstractions` and exposes an `ILoggerFactory` seam - a settable global factory defaulting to `NullLoggerFactory.Instance` (fallback `NullLogger.Instance`) with `SetFactory`/`TrySetFactory`, and/or an `ILoggerFactory`/`ILogger` parameter in its API. It must **not** reference Serilog or any sink - that forces a logging framework on every consumer and drags in AOT-incompatible dependencies. The consuming **application** owns the concrete logger (Serilog is fine there), bridges it to `ILoggerFactory` (e.g. `SerilogLoggerFactory` from `Serilog.Extensions.Logging`), and injects it. Reference pattern: a `LogOptions` seam in the library; the consuming CLI builds the Serilog-backed factory and injects it via `LogOptions.SetFactory`. +2. **Libraries log through abstractions, never a concrete backend.** A NuGet **library** depends only on `Microsoft.Extensions.Logging.Abstractions` and exposes an `ILoggerFactory` seam: a settable global factory defaulting to `NullLoggerFactory.Instance` (fallback `NullLogger.Instance`) with `SetFactory`/`TrySetFactory`, and/or an `ILoggerFactory`/`ILogger` parameter in its API. It must **not** reference Serilog or any sink, which would force a logging framework on every consumer and drag in AOT-incompatible dependencies. The consuming **application** owns the concrete logger (Serilog is fine there), bridges it to `ILoggerFactory` (e.g. `SerilogLoggerFactory` from `Serilog.Extensions.Logging`), and injects it. Reference pattern: a `LogOptions` seam in the library; the consuming CLI builds the Serilog-backed factory and injects it via `LogOptions.SetFactory`. 3. **CallerMemberName**: Use for automatic function name tracking @@ -288,20 +288,20 @@ Follow the scope hierarchy in [Analyzer Diagnostics and Suppressions][analyzer-d } ``` -5. **Exceptions**: Do not swallow exceptions; log and rethrow or translate to a domain-specific exception +5. **Exceptions**: Do not swallow exceptions, and either log and rethrow or translate to a domain-specific exception #### Code Patterns 1. **Guard clauses**: Prefer early returns for validation and error handling -2. **Async all the way**: Avoid blocking calls (`.Result`, `.Wait()`); use `async`/`await` +2. **Async all the way**: Avoid blocking calls (`.Result`, `.Wait()`) and use `async`/`await` 3. **Cancellation tokens**: Accept `CancellationToken` as the last parameter and pass it through 4. **ConfigureAwait**: In library code, use `ConfigureAwait(false)` unless context is required - Do not call `ConfigureAwait(false)` in xUnit tests (see xUnit1030) -5. **Disposables**: Use `await using` for async disposables; prefer `using` declarations +5. **Disposables**: Use `await using` for async disposables, and prefer `using` declarations 6. **LINQ vs loops**: Use LINQ for clarity, loops for hot paths or allocations -7. **HTTP**: Reuse `HttpClient` via factory; avoid per-request instantiation +7. **HTTP**: Reuse `HttpClient` via factory, never per-request instantiation 8. **Collections**: Prefer `IReadOnlyList`/`IReadOnlyCollection` for public APIs -9. **Immutability**: Prefer immutable records; use init-only setters when records are not suitable; prefer immutable or frozen collections for read-only data +9. **Immutability**: Prefer immutable records, use init-only setters when records are not suitable, and prefer immutable or frozen collections for read-only data 10. **Exceptions as control flow**: Avoid using exceptions for expected flow 11. **Sealing classes**: Seal classes that are not designed for inheritance 12. **Read-only data**: Use immutable or frozen collections for read-only data sets @@ -309,7 +309,7 @@ Follow the scope hierarchy in [Analyzer Diagnostics and Suppressions][analyzer-d #### Testing Conventions -1. **Framework**: **xUnit v3 or later** (the `xunit.v3` package, never the legacy v2 `xunit` package) with **AwesomeAssertions** for every assertion. Native xUnit asserts (`Assert.Equal`, `Assert.True`, ...) are not allowed - use the fluent `.Should()` API. Dynamic test skipping (`Assert.Skip`, `Assert.SkipWhen`) is control flow, not an assertion, and stays native. +1. **Framework**: **xUnit v3 or later** (the `xunit.v3` package, never the legacy v2 `xunit` package) with **AwesomeAssertions** for every assertion. Native xUnit asserts (`Assert.Equal`, `Assert.True`, ...) are not allowed, so use the fluent `.Should()` API. Dynamic test skipping (`Assert.Skip`, `Assert.SkipWhen`) is control flow, not an assertion, and stays native. ```csharp [Fact] diff --git a/Docker/Dockerfile b/Docker/Dockerfile index 842069b..660ce58 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -35,7 +35,7 @@ FROM mcr.microsoft.com/dotnet/runtime:10.0-alpine AS final ARG LABEL_VERSION="1.0.0.0" LABEL name="PhotoCleaner" \ version="${LABEL_VERSION}" \ - description="An application that prepares photos and videos for import into photo managers." \ + description="Utility to prepare photos and videos for import into photo managers." \ maintainer="ptr727" # exiftool: EXIF metadata reader/writer (pulls in Perl) diff --git a/Docker/README.md b/Docker/README.md index 1f114fa..758d23d 100644 --- a/Docker/README.md +++ b/Docker/README.md @@ -1,14 +1,14 @@ # PhotoCleaner -An application that prepares photos and videos for import into photo managers. +Utility to prepare photos and videos for import into photo managers. ## Documentation Refer to the [project page][github] for complete usage and configuration. -- **Source Code**: [GitHub][github] - source code, issues, and CI/CD pipelines. -- **Binary Releases**: [GitHub Releases][releases] - pre-compiled executables for Windows, Linux, and macOS. -- **Docker Images**: [Docker Hub][dockerhub] - container images with exiftool and ffmpeg pre-installed. +- **Source Code**: [GitHub][github], holding the source, the issues, and the CI/CD pipelines. +- **Binary Releases**: [GitHub Releases][releases], carrying pre-compiled executables for Windows, Linux, and macOS. +- **Docker Images**: [Docker Hub][dockerhub], carrying container images with exiftool and ffmpeg pre-installed. ## Docker Tags diff --git a/HISTORY.md b/HISTORY.md index 1fa24bf..bc217c8 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,9 +1,22 @@ # PhotoCleaner -An application that prepares photos and videos for import into photo managers. +Utility to prepare photos and videos for import into photo managers. ## Release History +**Version: 1.1**: + +- Added `verify` command to detect possibly corrupt media: + - I discovered thousands of files in my Immich library that failed to create thumbnails, all these images were [corrupt](https://github.com/ptr727/PhotoCleaner/issues/25) but passed the entire pipeline undetected. + - Runs the Immich decoder inside the Immich docker image to confirm that the file is usable, which makes that decoder the only judge of a file's health. + - Carries no container parser of its own, deliberately, because such a parser condemns whatever it fails to understand, and an unfamiliar but valid format looks the same as a damaged one from the inside. Docker is therefore required. +- The exiftool metadata read now always passes `-validate`: + - Files exiftool reports errors on are marked invalid by `process` and skipped by `import`. + - Validation warnings are logged at debug level only, because many healthy files do produce warnings. +- **Breaking**: + - Commands now exit `2` when they complete with per-file failures, where `process` previously exited `0`. `0` still means success and `1` still means the command could not run. + - `trash` exits `2` when pagination stops early and the trash database ends up short of the server, where it previously reported success. + **Version: 1.0**: - First published release, carrying the multi-arch Docker image and the GitHub release with the Linux and Windows executables attached. diff --git a/PhotoCleaner.code-workspace b/PhotoCleaner.code-workspace index f440ce9..8181455 100644 --- a/PhotoCleaner.code-workspace +++ b/PhotoCleaner.code-workspace @@ -26,20 +26,19 @@ "csharp.debug.symbolOptions.searchNuGetOrgSymbolServer": true, "csharp.debug.symbolOptions.searchMicrosoftSymbolServer": true, "files.encoding": "utf8", - "csharp.debug.justMyCode": false, - "claudeCode.preferredLocation": "sidebar", - "claudeCode.useCtrlEnterToSend": true, + "csharp.debug.justMyCode": false }, "extensions": { "recommendations": [ + "csharpier.csharpier-vscode", "davidanson.vscode-markdownlint", - "ms-dotnettools.csdevkit", - "streetsidesoftware.code-spell-checker", "editorconfig.editorconfig", - "csharpier.csharpier-vscode", + "fanaticpythoner.better-todo-tree", "github.vscode-github-actions", - "eamodio.gitlens", - "fanaticpythoner.better-todo-tree" + "ms-azuretools.vscode-docker", + "ms-dotnettools.csdevkit", + "streetsidesoftware.code-spell-checker", + "yzhang.markdown-all-in-one" ] } } diff --git a/PhotoCleaner/CommandLine.cs b/PhotoCleaner/CommandLine.cs index 76c730c..135a994 100644 --- a/PhotoCleaner/CommandLine.cs +++ b/PhotoCleaner/CommandLine.cs @@ -45,7 +45,7 @@ internal CommandLine(string[] args) internal RootCommand CreateRootCommand() { RootCommand command = new( - "PhotoCleaner - An application that prepares photos and videos for import into photo managers." + "PhotoCleaner - Utility to prepare photos and videos for import into photo managers." ) { _logLevelOption, @@ -57,6 +57,25 @@ internal RootCommand CreateRootCommand() command.Subcommands.Add(CreateImportCommand()); command.Subcommands.Add(CreateIndexCommand()); command.Subcommands.Add(CreateTrashCommand()); + command.Subcommands.Add(CreateVerifyCommand()); + + return command; + } + + private Command CreateVerifyCommand() + { + Command command = new("verify", "Verify that media files can be rendered by Immich") + { + _pathOption, + _threadsOption, + _dbFileOption, + _rehashOption, + _reprocessOption, + }; + command.SetAction( + (parseResult, cancellationToken) => + new VerifyCommand(CreateOptions(parseResult), cancellationToken).ExecuteAsync() + ); return command; } @@ -299,7 +318,7 @@ internal Options CreateOptions( private static Option CreatePathOption() => new Option("--path") { - Description = "The directory path to process", + Description = "The media directory path", Required = true, }.AcceptExistingOnly(); @@ -389,7 +408,7 @@ private static Option CreateRehashOption() => { Option option = new("--skipdb") { - Description = "SQLite database with indexed files to be skipped", + Description = "SQLite database with indexed files to be skipped (read-only)", }; option.Validators.Add(result => { @@ -403,10 +422,7 @@ private static Option CreateRehashOption() => } private static Option CreateReprocessOption() => - new("--reprocess") - { - Description = "Re-process files even if already marked as processed in the database", - }; + new("--reprocess") { Description = "Re-run every file even if the database marks it done" }; private static Option CreateMarkProcessedOption() => new("--processed") diff --git a/PhotoCleaner/CommandRunner.cs b/PhotoCleaner/CommandRunner.cs index 3fc248b..24ff2ee 100644 --- a/PhotoCleaner/CommandRunner.cs +++ b/PhotoCleaner/CommandRunner.cs @@ -2,23 +2,43 @@ namespace PhotoCleaner; internal static class CommandRunner { - internal static async Task RunAsync(string commandName, Func work) + internal static async Task RunAsync(string commandName, Func> work) { Log.Information("{CommandName} started", commandName); + int exitCode; try { - await work().ConfigureAwait(false); + exitCode = await work().ConfigureAwait(false); } catch (OperationCanceledException) { Log.Information("{CommandName} cancelled", commandName); - return 1; + return ExitCode.Error; } catch (Exception ex) when (Log.Logger.LogAndHandle(ex)) { - return 1; + return ExitCode.Error; } - Log.Information("{CommandName} complete", commandName); - return 0; + + // Each outcome is named rather than reading everything that is not Failed as success. + // Nothing returns Error from the work itself today, and the catch blocks return it directly. + // This branch is what keeps a future one from reporting completion. + if (exitCode == ExitCode.Success) + { + Log.Information("{CommandName} complete", commandName); + } + else if (exitCode == ExitCode.Failed) + { + Log.Warning("{CommandName} complete with failures", commandName); + } + else + { + Log.Error( + "{CommandName} did not complete, exit code {ExitCode}", + commandName, + exitCode + ); + } + return exitCode; } } diff --git a/PhotoCleaner/ExifToolJson.cs b/PhotoCleaner/ExifToolJson.cs index f3731b5..27d5a44 100644 --- a/PhotoCleaner/ExifToolJson.cs +++ b/PhotoCleaner/ExifToolJson.cs @@ -1,4 +1,5 @@ using System.Diagnostics.CodeAnalysis; +using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; @@ -153,6 +154,12 @@ public sealed class ExifToolJson [JsonPropertyName("Matroska:DateTimeOriginal")] public string? MatroskaDateTimeOriginal { get; set; } + [JsonPropertyName("ExifTool:Validate")] + public string? Validate { get; set; } + + [JsonPropertyName("ExifTool:Error")] + public string? ExifToolError { get; set; } + [JsonPropertyName("QuickTime:ContentIdentifier")] public string? QuickTimeContentIdentifier { get; set; } @@ -232,6 +239,56 @@ out DateTime date return !string.IsNullOrEmpty(RIFFDateTimeOriginal) ? RIFFDateTimeOriginal : null; } + // Verdicts read "OK", "4 Warnings (all minor)", or "2 Errors, 3 Warnings". + // Only the leading count of each clause matters, since the minor breakdown qualifies warnings. + internal static (int Errors, int Warnings) ParseValidate(string? validate) + { + if (string.IsNullOrWhiteSpace(validate)) + { + return (0, 0); + } + + int errors = 0, + warnings = 0; + foreach (Range clauseRange in validate.AsSpan().Split(',')) + { + ReadOnlySpan clause = validate.AsSpan()[clauseRange].Trim(); + + // Strip any minor qualifier in parentheses before looking for the noun. + int parenIndex = clause.IndexOf('('); + if (parenIndex >= 0) + { + clause = clause[..parenIndex].Trim(); + } + + int spaceIndex = clause.IndexOf(' '); + if ( + spaceIndex <= 0 + || !int.TryParse( + clause[..spaceIndex], + NumberStyles.None, + CultureInfo.InvariantCulture, + out int count + ) + ) + { + continue; + } + + ReadOnlySpan noun = clause[(spaceIndex + 1)..].Trim(); + if (noun.StartsWith("Error", StringComparison.OrdinalIgnoreCase)) + { + errors += count; + } + else if (noun.StartsWith("Warning", StringComparison.OrdinalIgnoreCase)) + { + warnings += count; + } + } + + return (errors, warnings); + } + internal static bool IsDngVersionNewer(string? versionString) { if (string.IsNullOrEmpty(versionString)) diff --git a/PhotoCleaner/ExitCode.cs b/PhotoCleaner/ExitCode.cs new file mode 100644 index 0000000..ce2e399 --- /dev/null +++ b/PhotoCleaner/ExitCode.cs @@ -0,0 +1,13 @@ +namespace PhotoCleaner; + +// Callers script against these values, so each code's meaning is part of the contract. +internal static class ExitCode +{ + internal const int Success = 0; + + // The command could not complete, so it reports nothing about the files. + internal const int Error = 1; + + // The command completed, but at least one file failed. + internal const int Failed = 2; +} diff --git a/PhotoCleaner/ImmichVerifyScript.cs b/PhotoCleaner/ImmichVerifyScript.cs new file mode 100644 index 0000000..29ff665 --- /dev/null +++ b/PhotoCleaner/ImmichVerifyScript.cs @@ -0,0 +1,109 @@ +namespace PhotoCleaner; + +// Calls Immich's own compiled modules rather than reimplementing the preview pipeline. +// Behavior therefore tracks Immich across releases. +// The coupling surface is three module paths and their exported names, which the preflight checks. +internal static class ImmichVerifyScript +{ + // Required for module resolution inside the image. + internal const string WorkingDirectory = "/usr/src/app/server"; + + // Loads the same modules the verify script needs, so a moved module fails before any file is judged. + internal const string Preflight = """ + const ROOT = "/usr/src/app/server/dist"; + const { MediaRepository } = require(ROOT + "/repositories/media.repository.js"); + const { defaults } = require(ROOT + "/config.js"); + const { ThumbnailConfig } = require(ROOT + "/utils/media.js"); + if (!MediaRepository || !defaults || !defaults.image || !ThumbnailConfig) { + throw new Error("Immich modules loaded but expected exports are missing"); + } + process.stdout.write("PHOTOCLEANER_PREFLIGHT_OK\n"); + """; + + internal const string PreflightSentinel = "PHOTOCLEANER_PREFLIGHT_OK"; + + // Reads absolute paths on stdin and writes one JSON object per line: { path, ok, via, error }. + internal const string Verify = """ + const path = require("node:path"); + const readline = require("node:readline"); + const ROOT = "/usr/src/app/server/dist"; + const { MediaRepository } = require(ROOT + "/repositories/media.repository.js"); + const { defaults } = require(ROOT + "/config.js"); + const { ThumbnailConfig } = require(ROOT + "/utils/media.js"); + + // MediaRepository calls whatever the logger interface exposes, and that set has grown before. + // Swallowing every call through a Proxy keeps this working across Immich updates. + const logger = new Proxy( + {}, + { get: (_target, prop) => (prop === "isLevelEnabled" ? () => false : () => undefined) }, + ); + const repo = new MediaRepository(logger); + + const RAW = new Set([".arw", ".cr2", ".dng", ".nef", ".orf", ".rw2"]); + const VIDEO = new Set([ + ".3gp", ".asf", ".avi", ".m2t", ".m2ts", ".mkv", ".mov", ".mp4", ".mts", ".wmv", + ]); + const OUT = "/tmp/photocleaner-verify.jpg"; + + async function verifyVideo(file) { + const info = await repo.probe(file); + const videoStream = info.videoStreams[0]; + if (!videoStream) { + throw new Error("No video stream found"); + } + const config = ThumbnailConfig.create({ + ...defaults.ffmpeg, + targetResolution: defaults.image.preview.size.toString(), + }); + const options = config.getCommand(0, videoStream, undefined, info.format); + await repo.transcode(file, OUT, options); + return "transcode"; + } + + async function verifyImage(file, ext) { + let via = "decode"; + if (RAW.has(ext)) { + try { + if (await repo.extract(file)) { + via = "decode+extract"; + } + } catch { + // Immich treats extraction as best effort too; only the decode must succeed + } + } + await repo.generateThumbnail( + file, + { ...defaults.image.preview, colorspace: "srgb", processInvalidImages: false }, + OUT, + ); + return via; + } + + const rl = readline.createInterface({ input: process.stdin, crlfDelay: Infinity }); + (async () => { + for await (const line of rl) { + const file = line.trim(); + if (!file) { + continue; + } + const ext = path.extname(file).toLowerCase(); + try { + const via = VIDEO.has(ext) + ? await verifyVideo(file) + : await verifyImage(file, ext); + process.stdout.write(JSON.stringify({ path: file, ok: true, via }) + "\n"); + } catch (e) { + const error = String((e && e.message) || e) + .replaceAll(/\s+/g, " ") + .slice(0, 300); + process.stdout.write( + JSON.stringify({ path: file, ok: false, via: "", error }) + "\n", + ); + } + } + })().catch((e) => { + process.stderr.write("fatal: " + String((e && e.message) || e) + "\n"); + process.exit(1); + }); + """; +} diff --git a/PhotoCleaner/ImportCommand.cs b/PhotoCleaner/ImportCommand.cs index 01ff1dd..b084368 100644 --- a/PhotoCleaner/ImportCommand.cs +++ b/PhotoCleaner/ImportCommand.cs @@ -25,6 +25,7 @@ await CommandRunner int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await TrashDatabaseScope @@ -81,7 +82,10 @@ await DatabaseScope trashSkipped ); Log.Information("Deleted {DeletedCount} empty directories", deletedDirs); + Log.Information("Invalid {InvalidCount} files", invalid); Log.Information("Failed {FailedCount} files", failed); + + return failed > 0 || invalid > 0 ? ExitCode.Failed : ExitCode.Success; } ) .ConfigureAwait(false); diff --git a/PhotoCleaner/ImportTask.cs b/PhotoCleaner/ImportTask.cs index 4e2d863..ffda7cc 100644 --- a/PhotoCleaner/ImportTask.cs +++ b/PhotoCleaner/ImportTask.cs @@ -40,6 +40,7 @@ internal enum ImportResult Skipped, SkippedBySkipDb, TrashedInImmich, + Invalid, Failed, } @@ -54,6 +55,7 @@ internal enum ImportResult int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs )> ExecuteAsync( @@ -67,6 +69,7 @@ int deletedDirs skipped = 0, skipDbSkipped = 0, trashSkipped = 0, + invalid = 0, failed = 0; Log.Information("Importing {FileCount} files", allFiles.Count); await Parallel @@ -90,6 +93,7 @@ await Parallel ref skipDbSkipped ), ImportResult.TrashedInImmich => Interlocked.Increment(ref trashSkipped), + ImportResult.Invalid => Interlocked.Increment(ref invalid), ImportResult.Failed => Interlocked.Increment(ref failed), _ => throw new NotImplementedException(), }; @@ -100,6 +104,8 @@ ref skipDbSkipped } catch (Exception ex) { + // Every failure counts, including a file missing since it was indexed. + // Import never renames its source, so a vanished file means the tree changed. Log.Error(ex, "Failed to import '{FilePath}'", file); _ = Interlocked.Increment(ref failed); } @@ -110,7 +116,16 @@ ref skipDbSkipped int deletedDirs = options.DeleteEmpty ? DirectoryCleaner.DeleteEmptyDirectories(options.OutPath!, options.DryRun) : 0; - return (imported, ignored, skipped, skipDbSkipped, trashSkipped, failed, deletedDirs); + return ( + imported, + ignored, + skipped, + skipDbSkipped, + trashSkipped, + invalid, + failed, + deletedDirs + ); } private async Task ImportFileAsync( @@ -133,9 +148,8 @@ private async Task ImportFileAsync( string? sha1 = null; if (database is not null || skipDatabase is not null || trashDatabase is not null) { - // Source-side hash caching: when the source file is already recorded in the import DB - // and its size/mtime match disk, ResolveHashesAsync returns the cached hashes without - // rehashing. The cache is keyed by source path because import inserts source paths. + // ResolveHashesAsync returns cached hashes when size and mtime still match disk. + // The cache is keyed by source path because import inserts source paths. FileRecord? cached = database is null ? null : await database.GetByPathAsync(file, cancellationToken).ConfigureAwait(false); @@ -147,9 +161,8 @@ private async Task ImportFileAsync( sha1 = resolvedSha1; Log.Debug("File '{FilePath}' has SHA-256 '{Sha256}'", file, sha256); - // Trash check is mandatory and runs FIRST. The Trash.db is the durable record of - // "user threw this away in Immich" beyond Immich's 30-day trash retention. Skipping - // here is the only thing preventing re-import after Immich purges its own trash. + // Trash.db outlives Immich's 30-day trash retention. + // Skipping here is the only thing preventing re-import after Immich purges its trash. if (trashDatabase is not null) { bool trashed = await trashDatabase @@ -203,6 +216,20 @@ private async Task ImportFileAsync( ExifToolJson? meta = await GetFileMetaAsync(file, cancellationToken).ConfigureAwait(false); + // Warnings are ignored on purpose, since most healthy files carry them. + (int validateErrors, _) = ExifToolJson.ParseValidate(meta?.Validate); + if (validateErrors > 0) + { + Log.Error( + "Skipping import of '{FilePath}': exiftool validation reported {ErrorCount} error(s): {Validate} {Error}", + file, + validateErrors, + meta!.Validate, + meta.ExifToolError + ); + return ImportResult.Invalid; + } + string inferredDateStr = string.Empty; DateTime? inferredDate = null; if (options.DatePath && !(meta?.IsDateSet() ?? false)) @@ -310,8 +337,8 @@ await MediaUtilities // Restore source mtime last - after any exiftool writes File.SetLastWriteTimeUtc(finalDest, sourceInfo.LastWriteTimeUtc); - // Record the SOURCE in the import DB. The row identifies the source file we - // imported, not the destination. Lookups are by source content hash (sha256). + // The row identifies the source file, not the destination. + // Lookups are by source content hash. if (database is not null && sha256 is not null && sha1 is not null) { Log.Debug("Inserting source '{SourcePath}' with SHA-256 '{Sha256}'", file, sha256); @@ -334,9 +361,8 @@ await database } catch (OperationCanceledException) { - // Clean up partial destination file on cancelled copy (not move - moves are atomic - // on the same filesystem; for cross-device moves .NET leaves the source intact on - // failure, so no orphan risk) + // Only a cancelled copy can leave a partial destination. + // A move is atomic on one filesystem, and cross-device failure leaves the source intact. if (!options.Move && File.Exists(finalDest)) { try @@ -414,9 +440,8 @@ internal static string[] ComputePathTags(string sourceFile, DirectoryInfo source ); } - // Adds each tag to XMP:Subject on destFile without creating duplicates. - // Uses remove-then-add (-= then +=) per value so existing copies are replaced - // rather than doubled. -overwrite_original skips exiftool's _original backup. + // Remove-then-add per value replaces an existing copy instead of doubling it. + // -overwrite_original skips exiftool's _original backup. private static async Task ApplyTagsAsync( string[] tags, string destFile, diff --git a/PhotoCleaner/IndexCommand.cs b/PhotoCleaner/IndexCommand.cs index bef475d..a56ee9c 100644 --- a/PhotoCleaner/IndexCommand.cs +++ b/PhotoCleaner/IndexCommand.cs @@ -38,6 +38,8 @@ await DatabaseScope Log.Information("Ignored {IgnoredCount} non-media files", ignored); _skippedExtensions.LogWarnings(); Log.Information("Failed {FailedCount} files", failed); + + return failed > 0 ? ExitCode.Failed : ExitCode.Success; } ) .ConfigureAwait(false); diff --git a/PhotoCleaner/MediaUtilities.cs b/PhotoCleaner/MediaUtilities.cs index 555d4e7..c85937c 100644 --- a/PhotoCleaner/MediaUtilities.cs +++ b/PhotoCleaner/MediaUtilities.cs @@ -73,17 +73,46 @@ internal static string GetUniqueFileName(string filePath) ) { Log.Debug("exiftool: Getting metadata for '{FilePath}'", filePath); + + // Separates a file this tool cannot open from one it reads but exiftool cannot parse. + // Both come back from exiftool as an error alongside well formed JSON. + // Without this they are indistinguishable, so a permission problem reads as damaged media. + EnsureReadable(filePath); + + // -all is required because -validate alone narrows the output to just that one tag. + // Measured at no cost over the plain call, so validation rides along instead of a second run. + // A file whose verdict reports an error also makes exiftool exit non-zero, JSON and all. + // The parsed output decides, not the exit code. BufferedCommandResult result = await Cli.Wrap("exiftool") - .WithArguments(["-groupNames", "-json", filePath]) + .WithArguments(["-groupNames", "-json", "-validate", "-all", filePath]) + .WithValidation(CommandResultValidation.None) .ExecuteBufferedAsync(cancellationToken); + + ReadOnlySpan json = result.StandardOutput.AsSpan().Trim([' ', '\n', '\r', '[', ']']); + if (json.IsEmpty) + { + throw new InvalidOperationException( + $"exiftool returned no metadata for '{filePath}' (exit {result.ExitCode}): " + + result.StandardError.Trim() + ); + } + ExifToolJson? exifToolJson = JsonSerializer.Deserialize( - result.StandardOutput.AsSpan().Trim([' ', '\n', '\r', '[', ']']), + json, ExifToolJsonContext.Default.ExifToolJson ); ArgumentNullException.ThrowIfNull(exifToolJson); return exifToolJson; } + // Reading attributes needs no read permission on the content. + // Nothing about a file's metadata therefore proves its bytes can be reached. + // A command that hands the file to another process sees its own lack of access nowhere else. + internal static void EnsureReadable(string filePath) + { + using FileStream probe = File.OpenRead(filePath); + } + internal static async Task SetCreateDateAsync( string createdDate, string outputFile, diff --git a/PhotoCleaner/ProcessCommand.cs b/PhotoCleaner/ProcessCommand.cs index ff4a234..5acca0a 100644 --- a/PhotoCleaner/ProcessCommand.cs +++ b/PhotoCleaner/ProcessCommand.cs @@ -10,6 +10,7 @@ CancellationToken cancellationToken private ConcurrentBag _fileNames = []; private readonly SkippedExtensionTracker _skippedExtensions = new(); private int _failedCount; + private int _invalidCount; private int _deletedCount; private int _modifiedCount; private int _skippedCount; @@ -78,7 +79,12 @@ await ExecuteProcessAsync(database, trashDatabase) ); Log.Information("Modified {ModifiedCount} files", _modifiedCount); Log.Information("Deleted {DeletedCount} files", _deletedCount); + Log.Information("Invalid {InvalidCount} files", _invalidCount); Log.Information("Failed {FailedCount} files", _failedCount); + + return _failedCount > 0 || _invalidCount > 0 + ? ExitCode.Failed + : ExitCode.Success; } ) .ConfigureAwait(false); @@ -90,9 +96,8 @@ await ExecuteProcessAsync(database, trashDatabase) )] private async Task ExecuteProcessAsync(Database? database, TrashDatabase? trashDatabase) { - // Separate files that share a stem (different extensions) so they - // are never processed in parallel - prevents one thread from - // deleting/renaming a file that another thread is reading. + // Files sharing a stem are never processed in parallel. + // Otherwise one thread could delete or rename a file another thread is reading. ConcurrentBag deferred = FixExtensionConflicts(); ConcurrentBag reProcessNames = []; @@ -127,6 +132,9 @@ await ProcessTask case ProcessTask.ProcessResult.Failure: _ = Interlocked.Increment(ref _failedCount); break; + case ProcessTask.ProcessResult.Invalid: + _ = Interlocked.Increment(ref _invalidCount); + break; case ProcessTask.ProcessResult.Deleted: _ = Interlocked.Increment(ref _deletedCount); break; @@ -144,10 +152,18 @@ await ProcessTask } } catch (Exception ex) - when (ex is FileNotFoundException || !File.Exists(fileName)) + when (ex is FileNotFoundException or DirectoryNotFoundException) { + // Process is the only command that rewrites the tree it walks as it goes. + // A missing name is usually its own earlier rename rather than a fault. + // It could equally be an external deletion, and the two are not cheaply told apart. + // So this neither fails the run nor claims to know which one happened. + // Import and verify both count a missing file as failed instead. + // Neither modifies its input, so there a vanished file can only be external. + // Matching the exception alone keeps a permission error out of this branch. + // Information rather than debug, so the run records why the file is gone. Log.Information( - "File no longer exists during processing (concurrent rename): '{FilePath}'", + "File no longer exists during processing: '{FilePath}'", fileName ); } @@ -223,12 +239,8 @@ internal static ( ConcurrentBag deferred ) SplitExtensionConflicts(ConcurrentBag fileNames) { - // Group media files by directory + stem (filename without extension). - // When multiple media files share the same stem (e.g. IMG.DNG + IMG.jpg), - // keep only one per group and defer the rest so they are never - // processed in parallel. Non-media files (e.g. PhotoCleaner.Process.db, - // PhotoCleaner.Process.log) bypass conflict detection - they are skipped - // by ProcessTask and cannot race with media-file pipeline steps. + // Media files are grouped by directory and stem, keeping one per group and deferring the rest. + // Non-media files bypass conflict detection, since ProcessTask skips them and they cannot race. ConcurrentBag filtered = []; ConcurrentBag deferred = []; Dictionary> stemMap = new( diff --git a/PhotoCleaner/ProcessTask.cs b/PhotoCleaner/ProcessTask.cs index 234d0c9..edbd8d6 100644 --- a/PhotoCleaner/ProcessTask.cs +++ b/PhotoCleaner/ProcessTask.cs @@ -23,6 +23,7 @@ public enum ProcessResult Modified, UnknownExtension, Skipped, + Invalid, } private ExifToolJson? _exifToolJson; @@ -114,10 +115,9 @@ private async Task ExecuteAsync() preProcessHash ); - // Trash check: delete files whose SHA-1 matches an Immich-trashed asset. - // The user explicitly threw these away in Immich; uploading them again on the - // next immich-cli run would be wasted work, so prune them here while we are - // already touching every file. Persist deletion in Process.db so re-runs are no-ops. + // Deletes files whose SHA-1 matches an asset already trashed in Immich. + // Pruning here costs nothing, since every file is being touched anyway. + // The deletion is persisted so re-runs are no-ops. if ( trashDatabase is not null && await TryDeleteIfTrashedAsync(sha1).ConfigureAwait(false) @@ -142,6 +142,11 @@ trashDatabase is not null .GetExifToolJsonAsync(fileInfo.FullName, cancellationToken) .ConfigureAwait(false); + if (!CheckExifToolValidation()) + { + return ProcessResult.Invalid; + } + // Process files ProcessResult result = !RenameMismatchedMimeExtensions() @@ -635,6 +640,34 @@ await MediaUtilities } } + // Roughly three quarters of healthy files carry validation warnings, so only errors fail a file. + private bool CheckExifToolValidation() + { + (int errors, int warnings) = ExifToolJson.ParseValidate(_exifToolJson!.Validate); + if (errors == 0) + { + if (warnings > 0) + { + Log.Debug( + "exiftool validation reported {WarningCount} warning(s) for '{FilePath}': {Validate}", + warnings, + fileInfo.FullName, + _exifToolJson.Validate + ); + } + return true; + } + + Log.Error( + "exiftool validation reported {ErrorCount} error(s) for '{FilePath}': {Validate} {Error}", + errors, + fileInfo.FullName, + _exifToolJson.Validate, + _exifToolJson.ExifToolError + ); + return false; + } + private bool WarnDngVersion() { if (!fileInfo.Extension.Equals(".dng", StringComparison.OrdinalIgnoreCase)) diff --git a/PhotoCleaner/SkippedExtensionTracker.cs b/PhotoCleaner/SkippedExtensionTracker.cs index 9e6f9fd..e189c33 100644 --- a/PhotoCleaner/SkippedExtensionTracker.cs +++ b/PhotoCleaner/SkippedExtensionTracker.cs @@ -23,6 +23,14 @@ internal void LogWarnings() sorted.Sort(StringComparer.OrdinalIgnoreCase); foreach (string extension in sorted) { + // A file with no extension tracks as an empty string, which reads as a quoted nothing. + // Naming the case keeps the count honest without printing a blank. + if (extension.Length == 0) + { + Log.Warning("Skipped files carrying no extension"); + continue; + } + Log.Warning("Unknown file extension: '{Extension}'", extension); } } diff --git a/PhotoCleaner/TrashCommand.cs b/PhotoCleaner/TrashCommand.cs index deef835..ffe8c99 100644 --- a/PhotoCleaner/TrashCommand.cs +++ b/PhotoCleaner/TrashCommand.cs @@ -14,6 +14,7 @@ await CommandRunner "Trash", async () => { + bool partialSync = false; await TrashDatabaseScope .RunAsync( options.TrashDbFile, @@ -120,11 +121,12 @@ await trashDatabase } else { - Log.Warning( + Log.Error( "Stopping pagination: invalid NextPage value '{NextPage}' after page {Page}", result.NextPage, page ); + partialSync = true; hasMore = false; } } @@ -152,6 +154,13 @@ await trashDatabase cancellationToken: cancellationToken ) .ConfigureAwait(false); + + // Pagination stopping early leaves the database short of the server. + // Callers use it to skip files, so a short one re-imports trashed assets. + // That is a completed run carrying a failure rather than a success. + // A page that throws leaves it short as well, keeping the pages before it. + // That exits Error from the handler above rather than reaching here. + return partialSync ? ExitCode.Failed : ExitCode.Success; } ) .ConfigureAwait(false); diff --git a/PhotoCleaner/UndoCommand.cs b/PhotoCleaner/UndoCommand.cs index b46b224..7554b77 100644 --- a/PhotoCleaner/UndoCommand.cs +++ b/PhotoCleaner/UndoCommand.cs @@ -25,7 +25,7 @@ await CommandRunner Log.Information("Deleted {DeletedCount} files", deleted); Log.Information("Failed {FailedCount} files", failed); - return Task.CompletedTask; + return Task.FromResult(failed > 0 ? ExitCode.Failed : ExitCode.Success); } ) .ConfigureAwait(false); diff --git a/PhotoCleaner/UndoTask.cs b/PhotoCleaner/UndoTask.cs index 1a03d66..95fe1fa 100644 --- a/PhotoCleaner/UndoTask.cs +++ b/PhotoCleaner/UndoTask.cs @@ -59,9 +59,8 @@ internal static int GetBackupSortKey(string path) // Pass 1 - identify derived base paths HashSet derivedBases = new(StringComparer.OrdinalIgnoreCase); - // Among groups that share the same stem+dir but different - // extensions, the one whose base file currently exists on disk is the derived conversion - // output - the original was backed up then removed; the output is still present. + // Among same-stem groups, the one whose base file still exists is the conversion output. + // The original was backed up then removed, so only the output remains on disk. foreach (KeyValuePair> kvp in groups) { string basePath = kvp.Key; @@ -191,9 +190,8 @@ internal static int GetBackupSortKey(string path) restored++; - // Delete any extra (numbered) backups for this group - the original is now - // restored and intermediate-state backups (.bak1, .bak2, ...) serve no purpose. - // The primaryBackup was already consumed (moved) by TryRestore, so we skip it. + // Once the original is restored the intermediate-state backups serve no purpose. + // TryRestore already consumed primaryBackup, so it is skipped. foreach ( string backup in backups.Where(b => !string.Equals(b, primaryBackup, StringComparison.OrdinalIgnoreCase) @@ -211,10 +209,8 @@ string backup in backups.Where(b => } } - // Locate derived conversion output. - // Prefer the explicit .bak.out companion written by ProcessTask (reliable even - // when GetUniqueFileName appended a counter suffix like img_1.mp4). - // Fall back to the stem-based heuristic for backups created before this feature. + // The .bak.out companion stays reliable when the output name gained a counter suffix. + // The stem-based heuristic is the fallback for backups predating that companion. string companionPath = primaryBackup + ".out"; if (File.Exists(companionPath)) { @@ -253,10 +249,9 @@ string backup in backups.Where(b => ) ) { - // Legacy fallback: look for same-stem .mp4 with no backup of its own. - // Note: a format-agnostic scan (any same-stem different-ext file) is unsafe - // because companion images (e.g. img.heic alongside img.mov) would be falsely - // deleted. We therefore target only the known video output extension. + // Legacy fallback looking for a same-stem .mp4 with no backup of its own. + // Only the known video output extension is targeted. + // A format-agnostic scan would falsely delete companion images. string stem = Path.GetFileNameWithoutExtension(basePath); string? dir = Path.GetDirectoryName(basePath); string derivedOutput = Path.Combine( diff --git a/PhotoCleaner/VerifyCommand.cs b/PhotoCleaner/VerifyCommand.cs new file mode 100644 index 0000000..d8d5eaa --- /dev/null +++ b/PhotoCleaner/VerifyCommand.cs @@ -0,0 +1,52 @@ +namespace PhotoCleaner; + +internal sealed class VerifyCommand( + CommandLine.Options options, + CancellationToken cancellationToken +) +{ + private readonly SkippedExtensionTracker _skippedExtensions = new(); + + internal async Task ExecuteAsync() => + await CommandRunner + .RunAsync( + "Verify", + async () => + { + (IReadOnlyList files, int totalCount) = FileEnumerator.Enumerate( + options.Path, + options.Threads, + cancellationToken + ); + + VerifyTask.Counts counts = await DatabaseScope + .RunAsync( + options.DbFile, + async database => + { + VerifyTask task = new(options, database, _skippedExtensions); + return await task.ExecuteAsync(files, cancellationToken) + .ConfigureAwait(false); + }, + cancellationToken: cancellationToken + ) + .ConfigureAwait(false); + + Log.Information("Total {TotalCount} files", totalCount); + Log.Information("Verified {VerifiedCount} files", counts.Verified); + Log.Information( + "Skipped {SkippedCount} already verified files", + counts.Skipped + ); + Log.Information("Ignored {IgnoredCount} non-media files", counts.Ignored); + _skippedExtensions.LogWarnings(); + Log.Information("Invalid {InvalidCount} files", counts.Invalid); + Log.Information("Failed {FailedCount} files", counts.Failed); + + return counts.Invalid > 0 || counts.Failed > 0 + ? ExitCode.Failed + : ExitCode.Success; + } + ) + .ConfigureAwait(false); +} diff --git a/PhotoCleaner/VerifyResult.cs b/PhotoCleaner/VerifyResult.cs new file mode 100644 index 0000000..f517dc4 --- /dev/null +++ b/PhotoCleaner/VerifyResult.cs @@ -0,0 +1,22 @@ +using System.Text.Json.Serialization; + +namespace PhotoCleaner; + +// One line of the JSON protocol emitted by the in-container verify script. +internal sealed class ImmichVerifyLine +{ + [JsonPropertyName("path")] + public string? Path { get; set; } + + [JsonPropertyName("ok")] + public bool Ok { get; set; } + + [JsonPropertyName("via")] + public string? Via { get; set; } + + [JsonPropertyName("error")] + public string? Error { get; set; } +} + +[JsonSerializable(typeof(ImmichVerifyLine))] +internal partial class ImmichVerifyJsonContext : JsonSerializerContext; diff --git a/PhotoCleaner/VerifyTask.cs b/PhotoCleaner/VerifyTask.cs new file mode 100644 index 0000000..5ec7e75 --- /dev/null +++ b/PhotoCleaner/VerifyTask.cs @@ -0,0 +1,449 @@ +using System.Text.Json; +using CliWrap; +using CliWrap.Buffered; + +namespace PhotoCleaner; + +// Only running Immich's own decoder can catch files that are intact but undecodable. +internal sealed class VerifyTask( + CommandLine.Options options, + Database? database, + SkippedExtensionTracker skippedExtensions +) +{ + // Immich publishes no :latest tag, so :release is the rolling equivalent. + internal const string ImmichImage = "ghcr.io/immich-app/immich-server:release"; + + // Namespaced to this application and purpose so it cannot collide with any other container. + internal const string ContainerLabel = "photocleaner-verify"; + + // Stateless and shared across the parallel loop, rather than rebuilt for every file. + private readonly IndexTask? _indexTask = database is null + ? null + : new IndexTask(options, database, skippedExtensions); + + // A host path is not a valid container path on every platform. + // The tree is mounted here and paths are translated across the boundary. + private const string ContainerMount = "/photocleaner"; + + // Files per container invocation, amortizing a measured four seconds of fixed startup. + // Most of that is Immich's own module graph loading, which no faithful invocation can avoid. + // Per-file cost spans fiftyfold between a thumbnail and a raw frame. + // The size is set for the cheap end, where it still outruns startup sevenfold. + // Going higher risks fewer batches than threads on a small tree, which costs more than it saves. + private const int BatchSize = 1024; + + internal sealed record Counts(int Verified, int Invalid, int Skipped, int Ignored, int Failed); + + private sealed class Tally + { + private int _verified; + private int _invalid; + private int _skipped; + private int _ignored; + private int _failed; + + internal void Verified() => Interlocked.Increment(ref _verified); + + internal void Invalid() => Interlocked.Increment(ref _invalid); + + internal void Skipped() => Interlocked.Increment(ref _skipped); + + internal void Ignored() => Interlocked.Increment(ref _ignored); + + internal void Failed() => Interlocked.Increment(ref _failed); + + internal Counts ToCounts() => new(_verified, _invalid, _skipped, _ignored, _failed); + } + + internal async Task ExecuteAsync( + IReadOnlyCollection files, + CancellationToken cancellationToken = default + ) + { + Tally tally = new(); + await RunAsync(files, tally, cancellationToken).ConfigureAwait(false); + return tally.ToCounts(); + } + + // Batched so one container serves many files, since startup dominates the per-file cost. + // Batches run in parallel, each recording its own results, so state lands throughout the run. + [System.Diagnostics.CodeAnalysis.SuppressMessage( + "Design", + "CA1031:Do not catch general exception types", + Justification = "Per-file catch-all logs the file path and continues verifying remaining files" + )] + private async Task RunAsync( + IReadOnlyCollection files, + Tally tally, + CancellationToken cancellationToken + ) + { + // Nothing to decode means nothing needs Docker. + // The preflight therefore waits for a file that actually reaches the decoder. + // A tree of non-media files needs none. + // Neither does a database-backed run whose files are all already verified. + // The first batch to find a candidate pays for it, bounding a Docker failure to one batch. + Lazy preflight = new( + () => PreflightAsync(cancellationToken), + LazyThreadSafetyMode.ExecutionAndPublication + ); + + string mount = options.Path.FullName; + List batches = [.. files.Chunk(BatchSize)]; + Log.Information( + "Verifying {FileCount} files in {BatchCount} batches", + files.Count, + batches.Count + ); + + await Parallel + .ForEachAsync( + batches, + CreateParallelOptions(cancellationToken), + async (batch, token) => + { + List candidates = new(batch.Length); + foreach (string file in batch) + { + try + { + if (!await ShouldVerifyAsync(file, tally, token).ConfigureAwait(false)) + { + continue; + } + candidates.Add(file); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + // One unreadable file must not abort the batch, nor the run. + Log.Error(ex, "Failed to verify '{FilePath}'", file); + tally.Failed(); + } + } + + if (candidates.Count > 0) + { + await preflight.Value.ConfigureAwait(false); + await DecodeBatchAsync(candidates, mount, tally, token) + .ConfigureAwait(false); + } + } + ) + .ConfigureAwait(false); + } + + // Extension filter and database skip check, which is what hashes the file. + private async Task ShouldVerifyAsync( + string file, + Tally tally, + CancellationToken cancellationToken + ) + { + string extension = Path.GetExtension(file); + if (!MediaUtilities.SupportedExtensions.Contains(extension)) + { + Log.Debug("Skipping non-media file: '{FilePath}'", file); + skippedExtensions.Track(extension); + tally.Ignored(); + return false; + } + + // Verify only reads, so a file gone since enumeration means the tree changed mid-run. + // The run no longer covers what it was asked to, which is a failure rather than damage. + // Import reaches the same verdict through the exception its file access throws. + // Verify needs the check stated, because without a database it never opens the file. + // The path would otherwise reach Immich and come back as unrenderable instead. + if (!File.Exists(file)) + { + Log.Error("Failed to verify '{FilePath}': the file no longer exists", file); + tally.Failed(); + return false; + } + + if (_indexTask is not null) + { + (IndexStatus status, _, _, bool wasVerified) = await _indexTask + .IndexFileAsync(file, cancellationToken) + .ConfigureAwait(false); + if (status == IndexStatus.Unchanged && wasVerified && !options.Reprocess) + { + Log.Debug("Skipping already verified '{FilePath}'", file); + tally.Skipped(); + return false; + } + } + + // Everything reaching here goes to the decoder, so this is where readability has to hold. + // With no database nothing opens the file at all. + // With one the size and mtime cache returns hashes without reading it. + // The per-file guard turns the throw into a failed count. + MediaUtilities.EnsureReadable(file); + return true; + } + + private async Task DecodeBatchAsync( + List batch, + string mount, + Tally tally, + CancellationToken cancellationToken + ) + { + List<(string Host, string Container)> mapped = new(batch.Count); + foreach (string file in batch) + { + // Paths reach the container one per line, so a line break cannot be expressed here. + // Without this the name splits and the container decodes fragments of it. + // The counts land the same either way, because a split path matches no returned verdict. + // So this buys an accurate reason rather than a different tally. + if ( + file.Contains('\n', StringComparison.Ordinal) + || file.Contains('\r', StringComparison.Ordinal) + ) + { + Log.Error("Failed to verify '{FilePath}': the path contains a line break", file); + tally.Failed(); + continue; + } + + if (!TryToContainerPath(mount, file, out string container)) + { + // The container path would leave the mount, so Immich would judge a different file. + Log.Error( + "File '{FilePath}' is not under the verified directory '{MountRoot}'", + file, + mount + ); + tally.Failed(); + continue; + } + mapped.Add((file, container)); + } + + if (mapped.Count == 0) + { + return; + } + + BufferedCommandResult result; + try + { + result = await RunImmichAsync( + ImmichVerifyScript.Verify, + mount, + string.Join('\n', mapped.Select(entry => entry.Container)), + $"decode of {mapped.Count} files", + cancellationToken + ) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // The batch never ran, so none of its files were judged. + // Counted over mapped rather than batch, since anything dropped in mapping already counted. + Log.Error(ex, "Immich decode batch of {FileCount} files failed to run", mapped.Count); + for (int i = 0; i < mapped.Count; i++) + { + tally.Failed(); + } + return; + } + + if (result.ExitCode != 0) + { + Log.Error( + "Immich decode batch exited {ExitCode}: {Error}", + result.ExitCode, + result.StandardError.Trim() + ); + } + + Dictionary results = new(StringComparer.Ordinal); + foreach (string line in result.StandardOutput.Split('\n')) + { + ImmichVerifyLine? parsed = ParseLine(line); + if (parsed?.Path is not null) + { + results[parsed.Path] = parsed; + } + } + + foreach ((string host, string container) in mapped) + { + if (!results.TryGetValue(container, out ImmichVerifyLine? line)) + { + // A missing verdict is a tooling gap, so it must never be reported as corruption. + Log.Error("No verification result returned for '{FilePath}'", host); + tally.Failed(); + continue; + } + + if (!line.Ok) + { + // A file removed after the batch was assembled makes the decoder fail too. + // Reporting that as unrenderable would call a vanished file damaged media. + // Deliberately untested, because reproducing it needs a race. + // The file has to vanish between assembling a batch and reading its results. + if (!File.Exists(host)) + { + Log.Error("Failed to verify '{FilePath}': the file no longer exists", host); + tally.Failed(); + continue; + } + + Log.Error("Immich cannot render '{FilePath}': {Error}", host, line.Error); + tally.Invalid(); + continue; + } + + Log.Debug("Verified '{FilePath}' via {Via}", host, line.Via); + tally.Verified(); + if (database is not null) + { + await database.SetProcessedAsync(host, cancellationToken).ConfigureAwait(false); + } + } + } + + internal static ImmichVerifyLine? ParseLine(string line) + { + ReadOnlySpan trimmed = line.AsSpan().Trim(); + if (trimmed.IsEmpty || trimmed[0] != '{') + { + return null; + } + + try + { + return JsonSerializer.Deserialize( + trimmed, + ImmichVerifyJsonContext.Default.ImmichVerifyLine + ); + } + catch (JsonException) + { + Log.Warning("Ignoring unparsable verification output: '{Line}'", line); + return null; + } + } + + // Throws rather than returning, so an unreachable Docker exits Error instead of condemning every file. + private static async Task PreflightAsync(CancellationToken cancellationToken) + { + Log.Information("Verifying Immich image '{Image}' is usable", ImmichImage); + + BufferedCommandResult result; + try + { + result = await RunImmichAsync( + ImmichVerifyScript.Preflight, + mount: null, + standardInput: null, + "preflight", + cancellationToken + ) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw new InvalidOperationException( + "Verification requires the 'docker' command and the Immich image, and has no " + + "offline mode, because Immich's decoder is the whole check. Run verify " + + "somewhere Docker is available.", + ex + ); + } + + if ( + result.ExitCode != 0 + || !result.StandardOutput.Contains( + ImmichVerifyScript.PreflightSentinel, + StringComparison.Ordinal + ) + ) + { + throw new InvalidOperationException( + $"Immich image '{ImmichImage}' could not be prepared for verification " + + $"(exit {result.ExitCode}): {result.StandardError.Trim()}" + ); + } + } + + private static async Task RunImmichAsync( + string script, + string? mount, + string? standardInput, + string label, + CancellationToken cancellationToken + ) + { + List arguments = + [ + "run", + "--rm", + "-i", + // The image's healthcheck cannot pass when the entrypoint is node. + // Without this every container reports unhealthy and alerts whatever watches Docker. + "--no-healthcheck", + // Marks the containers this tool starts so they can be found without matching on image. + // Other containers may run the same image, so an image or ancestor filter is not unique. + "--label", + $"{ContainerLabel}=1", + "--entrypoint", + "node", + "-w", + ImmichVerifyScript.WorkingDirectory, + ]; + if (mount is not null) + { + // Not -v, whose spec is colon delimited, so a colon in the host path is rejected. + // --mount is comma delimited and would reject a comma instead. + // A comma is the more common of the two in a photo directory name. + // Quoting the whole source field as CSV covers both, with any quote doubled. + string source = mount.Replace("\"", "\"\"", StringComparison.Ordinal); + arguments.Add("--mount"); + arguments.Add($"type=bind,\"source={source}\",target={ContainerMount},readonly"); + } + arguments.Add(ImmichImage); + arguments.Add("-e"); + arguments.Add(script); + + // Paired start and stop lines, so the log timestamps carry the duration. + Log.Debug("docker: Starting Immich {Label}", label); + BufferedCommandResult result = await Cli.Wrap("docker") + .WithArguments(arguments) + .WithStandardInputPipe( + standardInput is null ? PipeSource.Null : PipeSource.FromString(standardInput) + ) + .WithValidation(CommandResultValidation.None) + .ExecuteBufferedAsync(cancellationToken); + Log.Debug("docker: Stopped Immich {Label}", label); + return result; + } + + // Fails rather than mapping a file outside the mount. + // Such a path resolves to a container path that leaves the mount, naming a different file. + internal static bool TryToContainerPath(string mountRoot, string file, out string containerPath) + { + string relative = Path.GetRelativePath(mountRoot, file); + if ( + Path.IsPathRooted(relative) + || relative.Equals("..", StringComparison.Ordinal) + || relative.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) + ) + { + containerPath = string.Empty; + return false; + } + + containerPath = ContainerMount + "/" + relative.Replace(Path.DirectorySeparatorChar, '/'); + return true; + } + + private ParallelOptions CreateParallelOptions(CancellationToken cancellationToken) => + new() { MaxDegreeOfParallelism = options.Threads, CancellationToken = cancellationToken }; +} diff --git a/PhotoCleanerTests/CommandLineTests.cs b/PhotoCleanerTests/CommandLineTests.cs index c94bd18..e4f1d4f 100644 --- a/PhotoCleanerTests/CommandLineTests.cs +++ b/PhotoCleanerTests/CommandLineTests.cs @@ -6,6 +6,63 @@ public sealed class CommandLineTests { private static string ExistingDir => Directory.GetCurrentDirectory(); + // -- Verify command ------------------------------------------------------- + + [Fact] + public void VerifyCommand_PathOnly_ParsesWithoutErrors() + { + CommandLine cli = new(["verify", "--path", ExistingDir]); + + cli.Result.Errors.Should().BeEmpty(); + } + + // An option the command does not define is an error, never something silently ignored. + [Fact] + public void VerifyCommand_UnknownOption_IsRejected() + { + CommandLine cli = new(["verify", "--path", ExistingDir, "--nosuchoption"]); + + cli.Result.Errors.Should().NotBeEmpty(); + } + + [Fact] + public void VerifyCommand_WithDbAndThreads_ParsesOptions() + { + CommandLine cli = new([ + "verify", + "--path", + ExistingDir, + "--db", + "Verify.db", + "--threads", + "2", + "--reprocess", + ]); + + CommandLine.Options options = cli.CreateOptions(cli.Result); + + cli.Result.Errors.Should().BeEmpty(); + options.DbFile!.Name.Should().Be("Verify.db"); + options.Threads.Should().Be(2); + options.Reprocess.Should().BeTrue(); + } + + [Fact] + public void VerifyCommand_MissingPath_ReportsError() + { + CommandLine cli = new(["verify"]); + + cli.Result.Errors.Should().NotBeEmpty(); + } + + [Fact] + public void VerifyCommand_IsRegisteredAsSubcommand() + { + CommandLine cli = new(["--help"]); + + cli.Root.Subcommands.Select(command => command.Name).Should().Contain("verify"); + } + // -- SkipBackup option ---------------------------------------------------- [Fact] diff --git a/PhotoCleanerTests/DirectoryCleanerTests.cs b/PhotoCleanerTests/DirectoryCleanerTests.cs index 1e45ad8..7759476 100644 --- a/PhotoCleanerTests/DirectoryCleanerTests.cs +++ b/PhotoCleanerTests/DirectoryCleanerTests.cs @@ -156,9 +156,8 @@ public void DeleteEmptyDirectories_DryRun_DoesNotDeleteFilesystem() dryRun: true ); - // Dry-run counts each directory that is empty at scan time but does not delete. - // Cascading deletes (parent that would become empty after child removal) are not - // simulated, so this only reports the leaves. + // Dry-run counts each directory empty at scan time but deletes nothing. + // Cascading deletes are not simulated, so only the leaves are reported. deleted.Should().Be(2); Directory.Exists(emptyA).Should().BeTrue(); Directory.Exists(emptyB).Should().BeTrue(); diff --git a/PhotoCleanerTests/ExifToolJsonTests.cs b/PhotoCleanerTests/ExifToolJsonTests.cs index a99361f..7aa52be 100644 --- a/PhotoCleanerTests/ExifToolJsonTests.cs +++ b/PhotoCleanerTests/ExifToolJsonTests.cs @@ -433,4 +433,48 @@ public void IsDngVersionNewer_WhenMalformed_ReturnsFalse(string version) // Assert result.Should().BeFalse(); } + + // -- Validate verdict parsing --------------------------------------------- + // The sample strings below are verbatim from a sweep of a real collection. + // Roughly three quarters of healthy files carry warnings, so only the error count is acted on. + + [Theory] + [InlineData("OK", 0, 0)] + [InlineData("1 Warning", 0, 1)] + [InlineData("1 Warning (minor)", 0, 1)] + [InlineData("2 Warnings", 0, 2)] + [InlineData("4 Warnings (all minor)", 0, 4)] + [InlineData("5 Warnings (1 minor)", 0, 5)] + [InlineData("12 Warnings (9 minor)", 0, 12)] + [InlineData("1 Error", 1, 0)] + [InlineData("2 Errors, 3 Warnings", 2, 3)] + [InlineData("1 Error, 11 Warnings (3 minor)", 1, 11)] + public void ParseValidate_KnownVerdicts_ReturnsCounts( + string validate, + int expectedErrors, + int expectedWarnings + ) + { + // Act + (int errors, int warnings) = ExifToolJson.ParseValidate(validate); + + // Assert + errors.Should().Be(expectedErrors); + warnings.Should().Be(expectedWarnings); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("something unexpected")] + public void ParseValidate_AbsentOrUnrecognized_ReturnsZeroCounts(string? validate) + { + // An unreadable verdict must not count as an error. + // An exiftool output change would otherwise start failing every file in the collection. + (int errors, int warnings) = ExifToolJson.ParseValidate(validate); + + errors.Should().Be(0); + warnings.Should().Be(0); + } } diff --git a/PhotoCleanerTests/ImportTaskTests.cs b/PhotoCleanerTests/ImportTaskTests.cs index a7d8ddf..f88efd8 100644 --- a/PhotoCleanerTests/ImportTaskTests.cs +++ b/PhotoCleanerTests/ImportTaskTests.cs @@ -79,6 +79,64 @@ await Cli.Wrap("exiftool") .WithValidation(CommandResultValidation.None) .ExecuteAsync(); + // -- Invalid: exiftool reports an error, file is skipped ----------------- + + [Fact] + public async Task ExecuteAsync_ExifToolErrorFile_IsInvalidAndNotImported() + { + string srcDir = TempDir(); + string outDir = TempDir(); + try + { + // A file whose bytes are not a real image makes exiftool report a format error. + string bad = Path.Combine(srcDir, "garbage.jpg"); + await File.WriteAllBytesAsync( + bad, + [.. Enumerable.Range(0, 500).Select(i => (byte)(i % 251))], + TestContext.Current.CancellationToken + ); + + ImportTask task = new( + CreateOptions(outDir), + database: null, + skipDatabase: null, + trashDatabase: null, + new() + ); + ( + int organized, + int ignored, + int skipped, + int skipDbSkipped, + int trashSkipped, + int invalid, + int failed, + int deletedDirs + ) = await task.ExecuteAsync( + [bad], + new DirectoryInfo(srcDir), + TestContext.Current.CancellationToken + ); + + // Invalid is its own outcome, distinct from failed, and nothing is copied out. + invalid.Should().Be(1); + failed.Should().Be(0); + organized.Should().Be(0); + ignored.Should().Be(0); + skipped.Should().Be(0); + skipDbSkipped.Should().Be(0); + trashSkipped.Should().Be(0); + deletedDirs.Should().Be(0); + Directory.GetFiles(outDir, "*", SearchOption.AllDirectories).Should().BeEmpty(); + File.Exists(bad).Should().BeTrue(); + } + finally + { + Directory.Delete(srcDir, true); + Directory.Delete(outDir, true); + } + } + // -- UnsupportedFile: ignored, count incremented ------------------------- [Fact] @@ -104,6 +162,7 @@ public async Task ExecuteAsync_UnsupportedFile_Ignored() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -152,6 +211,7 @@ public async Task ExecuteAsync_DryRun_ReportsCountButLeavesFiles() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -201,6 +261,7 @@ public async Task ExecuteAsync_SupportedFile_CopiedToDateSubdir() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -250,6 +311,7 @@ public async Task ExecuteAsync_NoExifDate_FallsBackToMinValue() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -305,6 +367,7 @@ public async Task ExecuteAsync_SameNameFilesInSameMonth_SecondGetsUniqueName() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -359,6 +422,7 @@ public async Task ExecuteAsync_OrganizedFile_PreservesLastWriteTime() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -407,6 +471,7 @@ public async Task ExecuteAsync_MoveFlag_SourceFileRemoved() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -455,6 +520,7 @@ public async Task ExecuteAsync_CopyDefault_SourceFileRetained() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -505,6 +571,7 @@ public async Task ExecuteAsync_DeleteEmpty_RemovesEmptyTargetSubdirectories() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -575,6 +642,7 @@ await db.InsertAsync( int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -623,6 +691,7 @@ public async Task ExecuteAsync_SubdirectoryFormat_CreatesNestedDirs() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -678,6 +747,7 @@ public async Task ExecuteAsync_WithDatabase_NewFile_RecordedInDb() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -697,9 +767,8 @@ int deletedDirs ); recorded.Should().BeTrue(); - // The DB row identifies the SOURCE file we imported (not the destination), - // so the dedup lookup keeps working even if the destination is later mutated - // (e.g. by tag injection or process re-hash). + // The row identifies the source file, not the destination. + // Dedup therefore survives later mutation of the destination. FileRecord? row = await db.GetByPathAsync( jpg, cancellationToken: TestContext.Current.CancellationToken @@ -727,12 +796,11 @@ int deletedDirs [Fact] public async Task ExecuteAsync_TagPathMutatesDest_SourceRowIsUntouched() { - // Models the central design property: import inserts a row keyed by SOURCE path, - // even though the destination file gets mutated by tag injection. Because no - // command writes to the source path's row through normal use, dedup remains stable. + // Import keys its row by source path even though tag injection mutates the destination. + // No command writes that row in normal use, so dedup stays stable. string srcDir = TempDir(); - // Put the file in a sub-directory so --tagpath actually has tokens to apply - // (ComputePathTags returns [] for files at the root of --path). + // The file needs a sub-directory for --tagpath to have any tokens to apply. + // ComputePathTags returns nothing for files at the root of --path. string srcSubDir = Path.Combine(srcDir, "vacation"); Directory.CreateDirectory(srcSubDir); string outDir = TempDir(); @@ -757,15 +825,14 @@ public async Task ExecuteAsync_TagPathMutatesDest_SourceRowIsUntouched() trashDatabase: null, new() ); - (int organized, _, _, _, _, _, _) = await task.ExecuteAsync( + (int organized, _, _, _, _, _, _, _) = await task.ExecuteAsync( [jpg], new DirectoryInfo(srcDir), TestContext.Current.CancellationToken ); organized.Should().Be(1); - // The dest file on disk has had XMP tags written and now hashes to something - // different from the source. + // Writing XMP tags changed the destination, so it no longer hashes to the source. string dest = Path.Combine(outDir, "2024-06", "photo.jpg"); File.Exists(dest).Should().BeTrue(); (string destSha256, _) = await Database.ComputeHashesAsync( @@ -782,8 +849,8 @@ public async Task ExecuteAsync_TagPathMutatesDest_SourceRowIsUntouched() srcRow.Should().NotBeNull(); srcRow.Sha256.Should().Be(srcSha256); - // No row was inserted for the dest path. Process operating on /Processed with a - // separate Process.db is what tracks the dest state; import never writes there. + // No row is inserted for the destination path. + // A separate Process.db tracks destination state, and import never writes there. FileRecord? destRow = await db.GetByPathAsync( dest, cancellationToken: TestContext.Current.CancellationToken @@ -822,7 +889,7 @@ public async Task ExecuteAsync_ImportTwice_SecondRunSkipsAndCreatesNoDuplicate() trashDatabase: null, new() ); - (int organized1, _, int skipped1, _, _, int failed1, _) = await first.ExecuteAsync( + (int organized1, _, int skipped1, _, _, _, int failed1, _) = await first.ExecuteAsync( [jpg], new DirectoryInfo(srcDir), TestContext.Current.CancellationToken @@ -838,7 +905,7 @@ public async Task ExecuteAsync_ImportTwice_SecondRunSkipsAndCreatesNoDuplicate() trashDatabase: null, new() ); - (int organized2, _, int skipped2, _, _, int failed2, _) = await second.ExecuteAsync( + (int organized2, _, int skipped2, _, _, _, int failed2, _) = await second.ExecuteAsync( [jpg], new DirectoryInfo(srcDir), TestContext.Current.CancellationToken @@ -1095,6 +1162,7 @@ public async Task ExecuteAsync_TagPath_DryRun_NoTagsApplied() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1140,6 +1208,7 @@ public async Task ExecuteAsync_DatePath_DateInferredFromPath_SetsExifDateAndOrga int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1264,6 +1333,7 @@ public async Task ExecuteAsync_DatePath_DryRun_NoDateWritten() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1310,6 +1380,7 @@ public async Task ExecuteAsync_NonMediaFile_TracksExtension() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1370,6 +1441,7 @@ await trashDb.InsertHashAsync( int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1427,6 +1499,7 @@ await trashDb.InsertHashAsync( int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1494,6 +1567,7 @@ await skipDb.InsertAsync( int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1559,6 +1633,7 @@ await skipDb.InsertAsync( int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( diff --git a/PhotoCleanerTests/IndexTaskTests.cs b/PhotoCleanerTests/IndexTaskTests.cs index ee9996b..7f4817c 100644 --- a/PhotoCleanerTests/IndexTaskTests.cs +++ b/PhotoCleanerTests/IndexTaskTests.cs @@ -229,8 +229,7 @@ public async Task IndexFileAsync_Rehash_AlwaysRecomputesHash() cancellationToken: TestContext.Current.CancellationToken ); - // Rehash task forces recomputation - result should be same hash (file unchanged) - // but the code path goes through ComputeHashesAsync + // The file is unchanged so the hash matches, but the path runs through ComputeHashesAsync. IndexTask rehashTask = new( CreateOptions(rehash: true), db, @@ -322,9 +321,8 @@ public async Task IndexFileAsync_MarkProcessed_InsertsWithIsProcessedTrue() [Fact] public async Task IndexFileAsync_MarkProcessed_PreservesExistingFlagOnUpdate() { - // --processed only affects rows being INSERTED. Existing rows keep their flag - // (they get cleared by UpdateHashesAsync on a hash change, which is the existing - // behavior, but the --processed flag itself never alters rows during update). + // --processed affects only rows being inserted, and existing rows keep their flag. + // UpdateHashesAsync clears that flag on a hash change, but --processed never alters it. string dbPath = TempDb(); string filePath = TempFile("original"); try @@ -339,8 +337,8 @@ await noFlag.IndexFileAsync( cancellationToken: TestContext.Current.CancellationToken ); - // Re-run WITH --processed; the file is unchanged so this should be Unchanged - // and the existing row's is_processed should remain 0 (not flipped to 1). + // The file is unchanged, so the re-run reports Unchanged. + // The existing row keeps is_processed at 0. IndexTask withFlag = new( CreateOptions(markProcessed: true), db, diff --git a/PhotoCleanerTests/ProcessTaskTests.cs b/PhotoCleanerTests/ProcessTaskTests.cs index 288b910..fe277b4 100644 --- a/PhotoCleanerTests/ProcessTaskTests.cs +++ b/PhotoCleanerTests/ProcessTaskTests.cs @@ -33,6 +33,66 @@ private sealed class InMemorySink : ILogEventSink // -- Extension / MIME handling -------------------------------------------- + // A file exiftool cannot open looks like one it cannot parse: an error plus valid JSON. + // The shared metadata call separates them by opening the file first. + // A permission problem is therefore never recorded as damaged media, in any command. + [Fact] + public async Task GetExifToolJsonAsync_UnreadableFile_Throws() + { + if (!OperatingSystem.IsLinux()) + { + Assert.Skip("File mode permissions are only enforced on Linux"); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + string locked = Path.Combine(workDir, "locked.jpg"); + try + { + // Arrange + File.Copy(fixture.SourceFile(TempDirectoryFixture.SmallJpegFile), locked); + File.SetUnixFileMode(locked, UnixFileMode.None); + using (FileStream? readable = TryOpen(locked)) + { + if (readable is not null) + { + Assert.Skip("Running with permission to read anything"); + return; + } + } + + // Act + Func act = async () => + await MediaUtilities.GetExifToolJsonAsync( + locked, + TestContext.Current.CancellationToken + ); + + // Assert + await act.Should().ThrowAsync(); + } + finally + { + if (File.Exists(locked)) + { + File.SetUnixFileMode(locked, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + private static FileStream? TryOpen(string path) + { + try + { + return File.OpenRead(path); + } + catch (UnauthorizedAccessException) + { + return null; + } + } + [Fact] public async Task ExecuteAsync_UnknownExtension_ReturnsUnknownExtension() { @@ -62,9 +122,8 @@ public async Task ExecuteAsync_MixedCaseExtension_RenamesFile() string workDir = TempDirectoryFixture.CreateWorkDir(); try { - // On case-insensitive filesystems (Windows NTFS, macOS HFS+) "photo.Jpg" and - // "photo.jpg" are the same file, so the rename is meaningless and File.Exists() - // cannot distinguish them. Skip on those platforms; the test runs fully on Linux. + // On a case-insensitive filesystem the two names are the same file. + // The rename is then meaningless and File.Exists cannot distinguish them. if (!TempDirectoryFixture.IsFileSystemCaseSensitive(workDir)) { Assert.Skip( @@ -365,8 +424,8 @@ public async Task ExecuteAsync_MtsFile_RemuxesToMp4() string filePath = Path.Combine(workDir, TempDirectoryFixture.MtsFile); File.Copy(fixture.SourceFile(TempDirectoryFixture.MtsFile), filePath); - // Act - first pass: exiftool returns "m2t" for MPEG-TS, so .mts is renamed to .m2t - // and queued for reprocess; remux to .mp4 happens on the second pass + // For MPEG-TS, exiftool returns "m2t", so the first pass renames and re-queues. + // The remux to .mp4 happens on the second pass. ProcessTask.ProcessResult result = await CreateContext(filePath); // Assert @@ -390,8 +449,8 @@ public async Task ExecuteAsync_M2tsFile_RemuxesToMp4() string filePath = Path.Combine(workDir, TempDirectoryFixture.M2tsFile); File.Copy(fixture.SourceFile(TempDirectoryFixture.M2tsFile), filePath); - // Act - first pass: exiftool returns "m2t" for MPEG-TS, so .m2ts is renamed to .m2t - // and queued for reprocess; remux to .mp4 happens on the second pass + // For MPEG-TS, exiftool returns "m2t", so the first pass renames and re-queues. + // The remux to .mp4 happens on the second pass. ProcessTask.ProcessResult result = await CreateContext(filePath); // Assert @@ -614,8 +673,8 @@ public async Task ExecuteAsync_MovWithPcmAudio_ReencodesAudio() // Act ProcessTask.ProcessResult result = await CreateContext(filePath); - // Assert - exiftool returns "mov" for QuickTime, which matches the extension, so no - // rename occurs; PCM audio is detected and re-encoded directly in the first pass + // For QuickTime, exiftool returns "mov", which matches the extension, so no rename occurs. + // PCM audio is detected and re-encoded directly in the first pass. result.Should().Be(ProcessTask.ProcessResult.Reprocess); File.Exists(filePath).Should().BeFalse(); File.Exists(filePath + ".bak").Should().BeTrue(); @@ -640,8 +699,8 @@ public async Task ExecuteAsync_MovWithAacAudio_NoConversion() // Act ProcessTask.ProcessResult result = await CreateContext(filePath); - // Assert - exiftool returns "mov" for QuickTime, which matches the extension, so no - // rename occurs; AAC audio requires no conversion, file is left as-is + // For QuickTime, exiftool returns "mov", which matches the extension, so no rename occurs. + // AAC audio needs no conversion, so the file is left as-is. result.Should().Be(ProcessTask.ProcessResult.Success); File.Exists(filePath).Should().BeTrue(); File.Exists(Path.ChangeExtension(filePath, ".mp4")).Should().BeFalse(); @@ -704,8 +763,8 @@ public async Task ExecuteAsync_Mp4WithAacAudio_ReturnsSuccess() [Fact] public async Task ExecuteAsync_PcmMp4WithContentIdentifier_PreservesContentIdentifierAfterConversion() { - // Arrange - live photo MP4 with PCM audio: first pass re-encodes audio (no companion yet), - // then companion is added and second pass deletes the video as a live photo + // The first pass re-encodes the audio, since no companion image exists yet. + // The companion is then added and the second pass deletes the video as a live photo. string workDir = TempDirectoryFixture.CreateWorkDir(); try { @@ -741,6 +800,93 @@ public async Task ExecuteAsync_PcmMp4WithContentIdentifier_PreservesContentIdent } } + // -- exiftool validation --------------------------------------------------- + + [Fact] + public async Task ExecuteAsync_FileWithValidationWarnings_IsNotTreatedAsInvalid() + { + // A truncated JPEG makes exiftool report "1 Warning: JPEG format error". + // Warnings must stay advisory, since roughly three quarters of healthy files carry one. + string workDir = TempDirectoryFixture.CreateWorkDir(); + try + { + // Arrange + string filePath = Path.Combine(workDir, "truncated.jpg"); + byte[] source = await File.ReadAllBytesAsync( + fixture.SourceFile(TempDirectoryFixture.SmallJpegFile), + TestContext.Current.CancellationToken + ); + await File.WriteAllBytesAsync( + filePath, + source[..(source.Length / 2)], + TestContext.Current.CancellationToken + ); + + // Act + ProcessTask.ProcessResult result = await CreateContext(filePath); + + // Assert + result.Should().NotBe(ProcessTask.ProcessResult.Invalid); + File.Exists(filePath).Should().BeTrue(); + } + finally + { + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + [Fact] + public async Task ExecuteAsync_FileExifToolErrorsOn_ReturnsInvalid() + { + // On exactly these files, exiftool exits non-zero while still emitting the JSON verdict. + // Without validation disabled on that call this path throws and is counted as a failure. + string workDir = TempDirectoryFixture.CreateWorkDir(); + try + { + // Arrange + string filePath = Path.Combine(workDir, "garbage.jpg"); + await File.WriteAllBytesAsync( + filePath, + [.. Enumerable.Range(0, 500).Select(i => (byte)(i % 251))], + TestContext.Current.CancellationToken + ); + + // Act + ProcessTask.ProcessResult result = await CreateContext(filePath); + + // Assert + result.Should().Be(ProcessTask.ProcessResult.Invalid); + File.Exists(filePath).Should().BeTrue(); + } + finally + { + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + [Fact] + public async Task ExecuteAsync_HealthyFile_ReturnsSuccessWithValidationEnabled() + { + // Guards the always-on -validate addition: it must not perturb the existing pipeline. + string workDir = TempDirectoryFixture.CreateWorkDir(); + try + { + // Arrange + string filePath = Path.Combine(workDir, "photo.jpg"); + File.Copy(fixture.SourceFile(TempDirectoryFixture.SmallJpegFile), filePath); + + // Act + ProcessTask.ProcessResult result = await CreateContext(filePath); + + // Assert + result.Should().Be(ProcessTask.ProcessResult.Success); + } + finally + { + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + // -- DNG version warning --------------------------------------------------- [Fact] diff --git a/PhotoCleanerTests/TempDirectoryFixture.cs b/PhotoCleanerTests/TempDirectoryFixture.cs index 465e3f6..603f382 100644 --- a/PhotoCleanerTests/TempDirectoryFixture.cs +++ b/PhotoCleanerTests/TempDirectoryFixture.cs @@ -91,10 +91,7 @@ internal static void DeleteWorkDir(string workDir) } } - /// - /// Returns true when the filesystem hosting is case-sensitive. - /// Probes at runtime so it works correctly on Linux (true), Windows NTFS (false), and macOS HFS+ (false). - /// + // Probed at runtime because Linux is case-sensitive while Windows NTFS and macOS HFS+ are not. internal static bool IsFileSystemCaseSensitive(string directory) { string upper = Path.Combine(directory, "_CASE_PROBE_"); @@ -370,8 +367,8 @@ private static async Task RunFfmpegAsync(string[] arguments) => await Cli.Wrap("ffmpeg").WithArguments(["-nostdin", "-y", .. arguments]).ExecuteAsync(); private static async Task RunExiftoolAsync(string[] arguments) => - // exiftool exits with code 1 for warnings (e.g. writing DNG tags to TIFF); - // the write still succeeds, so we suppress exit code validation here. + // Warnings such as writing DNG tags to TIFF make exiftool exit 1. + // The write still succeeds, so exit code validation is suppressed. await Cli.Wrap("exiftool") .WithArguments(arguments) .WithValidation(CommandResultValidation.None) diff --git a/PhotoCleanerTests/TrashCommandTests.cs b/PhotoCleanerTests/TrashCommandTests.cs index a2fab4b..2d0ea37 100644 --- a/PhotoCleanerTests/TrashCommandTests.cs +++ b/PhotoCleanerTests/TrashCommandTests.cs @@ -111,6 +111,48 @@ public void ConvertChecksum_EmptyString_ReturnsNull() => public void ConvertChecksum_InvalidBase64_ReturnsNull() => TrashCommand.ConvertChecksum("not-valid-base64!!!").Should().BeNull(); + // Pagination stopping early leaves the database short of the server. + // Callers use it to skip files, so a short one silently re-imports trashed assets. + [Fact] + public async Task ExecuteAsync_InvalidNextPage_ExitsFailed() + { + string dbPath = TempDb(); + try + { + // Arrange - a page that hands back a NextPage the loop cannot use + byte[] sha1 = new byte[20]; + sha1[0] = 0x01; + using MockImmichHandler handler = new(); + handler.EnqueueResponse( + MakeResponse([Convert.ToBase64String(sha1)], nextPage: "not-a-page") + ); + + using HttpClient client = new(handler, disposeHandler: false) + { + BaseAddress = new Uri("http://localhost:9999"), + }; + TrashCommand command = new( + CreateOptions(dbPath), + TestContext.Current.CancellationToken, + client + ); + + // Act + int exitCode = await command.ExecuteAsync(); + + // Assert - the page it did read is kept, and the run reports the shortfall + exitCode.Should().Be(2); + await using TrashDatabase db = new(dbPath); + await db.InitializeAsync(TestContext.Current.CancellationToken); + long count = await db.GetCountAsync(TestContext.Current.CancellationToken); + count.Should().Be(1); + } + finally + { + File.Delete(dbPath); + } + } + [Fact] public async Task ExecuteAsync_SinglePage_InsertsHashes() { @@ -273,9 +315,6 @@ public async Task ExecuteAsync_DuplicateHashes_Idempotent() } } - /// - /// Mock HTTP handler that returns queued ImmichSearchResponse payloads. - /// internal sealed class MockImmichHandler : DelegatingHandler { private readonly Queue _responses = new(); diff --git a/PhotoCleanerTests/UndoTaskTests.cs b/PhotoCleanerTests/UndoTaskTests.cs index 515857c..0a76831 100644 --- a/PhotoCleanerTests/UndoTaskTests.cs +++ b/PhotoCleanerTests/UndoTaskTests.cs @@ -114,7 +114,7 @@ public void Execute_SingleBackupNoCurrentFile_RestoresOriginal() string dir = TempDir(); try { - // img.mp4 was deleted; img.mp4.bak is the only artefact + // The only remaining artifact is img.mp4.bak, because img.mp4 was deleted string bakPath = Path.Combine(dir, "img.mp4.bak"); WriteContent(bakPath); diff --git a/PhotoCleanerTests/VerifyTaskTests.cs b/PhotoCleanerTests/VerifyTaskTests.cs new file mode 100644 index 0000000..9730da2 --- /dev/null +++ b/PhotoCleanerTests/VerifyTaskTests.cs @@ -0,0 +1,560 @@ +using PhotoCleaner; +using Serilog.Events; + +namespace PhotoCleanerTests; + +// A misread line becomes a false accusation against a real photo, so non-verdict output is ignored. +public sealed class VerifyTaskTests(TempDirectoryFixture fixture) + : IClassFixture +{ + [Fact] + public void ParseLine_SuccessfulVerdict_ParsesPathAndVia() + { + // Act + ImmichVerifyLine? line = VerifyTask.ParseLine( + """{"path":"/photos/IMG_0969.HEIC","ok":true,"via":"decode"}""" + ); + + // Assert + line.Should().NotBeNull(); + line!.Path.Should().Be("/photos/IMG_0969.HEIC"); + line.Ok.Should().BeTrue(); + line.Via.Should().Be("decode"); + } + + [Fact] + public void ParseLine_FailureVerdict_ParsesError() + { + // Arrange: the error Immich actually emits for the reported defect + const string json = """ + {"path":"/photos/bad.heic","ok":false,"via":"","error":"Input file has corrupt header: bad seek to 104941"} + """; + + // Act + ImmichVerifyLine? line = VerifyTask.ParseLine(json); + + // Assert + line.Should().NotBeNull(); + line!.Ok.Should().BeFalse(); + line.Error.Should().Contain("corrupt header"); + } + + [Fact] + public void ParseLine_RawExtractVerdict_ParsesVia() + { + // Act + ImmichVerifyLine? line = VerifyTask.ParseLine( + """{"path":"/photos/IMG_1603.dng","ok":true,"via":"decode+extract"}""" + ); + + // Assert + line!.Via.Should().Be("decode+extract"); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("(node:1) Warning: some deprecation notice")] + [InlineData("npm notice")] + public void ParseLine_NonJsonOutput_ReturnsNull(string line) + { + // Act + ImmichVerifyLine? result = VerifyTask.ParseLine(line); + + // Assert + result.Should().BeNull(); + } + + [Fact] + public void ParseLine_TruncatedJson_ReturnsNullWithoutThrowing() + { + // Arrange: what a killed container leaves behind mid-write + const string partial = """{"path":"/photos/IMG_0001.HEIC","ok":tr"""; + + // Act + Func act = () => VerifyTask.ParseLine(partial); + + // Assert + act.Should().NotThrow(); + act().Should().BeNull(); + } + + [Fact] + public void ParseLine_UnknownFields_AreIgnored() + { + // Arrange: a future script revision adding a field must not break an older binary + const string json = """ + {"path":"/photos/a.jpg","ok":true,"via":"decode","durationMs":42,"newField":{"x":1}} + """; + + // Act + ImmichVerifyLine? line = VerifyTask.ParseLine(json); + + // Assert + line.Should().NotBeNull(); + line!.Path.Should().Be("/photos/a.jpg"); + line.Ok.Should().BeTrue(); + } + + // A host path is not a valid container path on Windows, so paths are translated at the boundary. + + [Fact] + public void TryToContainerPath_MapsUnderTheFixedMount() + { + string mount = Path.Combine(Path.DirectorySeparatorChar.ToString(), "photos"); + string file = Path.Combine(mount, "2024", "IMG_0001.HEIC"); + + VerifyTask.TryToContainerPath(mount, file, out string container).Should().BeTrue(); + container.Should().Be("/photocleaner/2024/IMG_0001.HEIC"); + } + + [Fact] + public void TryToContainerPath_FileAtMountRoot_HasNoExtraSeparator() + { + string mount = Path.Combine(Path.DirectorySeparatorChar.ToString(), "photos"); + string file = Path.Combine(mount, "IMG_0001.HEIC"); + + VerifyTask.TryToContainerPath(mount, file, out string container).Should().BeTrue(); + container.Should().Be("/photocleaner/IMG_0001.HEIC"); + } + + [Fact] + public void TryToContainerPath_AlwaysUsesForwardSlashes() + { + string mount = Path.Combine(Path.DirectorySeparatorChar.ToString(), "photos"); + string file = Path.Combine(mount, "a", "b", "c.jpg"); + + VerifyTask.TryToContainerPath(mount, file, out string container).Should().BeTrue(); + + container.Should().StartWith("/photocleaner/"); + container.Should().NotContain("\\"); + } + + // A path outside the mount maps to a container path that leaves it, naming a different file. + [Theory] + [InlineData("elsewhere", "secret.jpg")] + [InlineData("photos-other", "IMG_0001.HEIC")] + public void TryToContainerPath_FileOutsideTheMount_Fails(string sibling, string name) + { + string root = Path.DirectorySeparatorChar.ToString(); + string mount = Path.Combine(root, "photos"); + string file = Path.Combine(root, sibling, name); + + VerifyTask.TryToContainerPath(mount, file, out string container).Should().BeFalse(); + container.Should().BeEmpty(); + } + + // A name merely starting with two dots is an ordinary file, not an escape. + [Fact] + public void TryToContainerPath_NameStartingWithDots_IsMapped() + { + string mount = Path.Combine(Path.DirectorySeparatorChar.ToString(), "photos"); + string file = Path.Combine(mount, "..hidden.jpg"); + + VerifyTask.TryToContainerPath(mount, file, out string container).Should().BeTrue(); + container.Should().Be("/photocleaner/..hidden.jpg"); + } + + // A file that cannot be read is a tooling gap, never damaged media. + // The verdict must not depend on whether a database happens to be configured. + [Fact] + public async Task ExecuteAsync_UnreadableFileWithoutDatabase_CountsFailedNotInvalid() + { + if (!OperatingSystem.IsLinux()) + { + Assert.Skip("File mode permissions are only enforced on Linux"); + return; + } + + if (!ImmichImageAvailable()) + { + Assert.Skip(ImageSkipReason); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + string locked = Path.Combine(workDir, "unreadable.jpg"); + try + { + // Arrange - a decodable file the process cannot open + File.Copy(fixture.SourceFile(TempDirectoryFixture.SmallJpegFile), locked); + File.SetUnixFileMode(locked, UnixFileMode.None); + if (CanRead(locked)) + { + Assert.Skip("Running with permission to read anything, so the file stays readable"); + return; + } + + VerifyTask task = new(CreateOptions(workDir), null, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [locked], + TestContext.Current.CancellationToken + ); + + // Assert + counts.Failed.Should().Be(1); + counts.Invalid.Should().Be(0); + counts.Verified.Should().Be(0); + } + finally + { + if (File.Exists(locked)) + { + File.SetUnixFileMode(locked, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + // Nothing to decode means nothing needs Docker. + // Deliberately ungated on the image, since a host without one is the case this protects. + // An unconditional preflight throws here instead of counting. + [Fact] + public async Task ExecuteAsync_OnlyNonMediaFiles_NeedsNoDocker() + { + string workDir = TempDirectoryFixture.CreateWorkDir(); + try + { + // Arrange - a tree holding nothing this command can verify + string notes = Path.Combine(workDir, "notes.txt"); + string readme = Path.Combine(workDir, "readme.md"); + await File.WriteAllTextAsync(notes, "text", TestContext.Current.CancellationToken); + await File.WriteAllTextAsync(readme, "text", TestContext.Current.CancellationToken); + + VerifyTask task = new(CreateOptions(workDir), null, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [notes, readme], + TestContext.Current.CancellationToken + ); + + // Assert + counts.Ignored.Should().Be(2); + counts.Verified.Should().Be(0); + counts.Invalid.Should().Be(0); + counts.Failed.Should().Be(0); + } + finally + { + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + // A database row matching on size and mtime returns cached hashes without opening the file. + // The hash read therefore cannot be relied on to notice that the file is unreadable. + [Fact] + public async Task ExecuteAsync_UnreadableFileWithCachedHashes_CountsFailedNotInvalid() + { + if (!OperatingSystem.IsLinux()) + { + Assert.Skip("File mode permissions are only enforced on Linux"); + return; + } + + if (!ImmichImageAvailable()) + { + Assert.Skip(ImageSkipReason); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + string target = Path.Combine(workDir, "cached.jpg"); + string dbPath = Path.Combine(workDir, "Verify.db"); + try + { + // Arrange - index the file while it is readable, so the row caches its hashes + File.Copy(fixture.SourceFile(TempDirectoryFixture.SmallJpegFile), target); + await using Database database = new(dbPath); + await database.InitializeAsync(TestContext.Current.CancellationToken); + IndexTask index = new(CreateOptions(workDir), database, new SkippedExtensionTracker()); + _ = await index.IndexFileAsync(target, TestContext.Current.CancellationToken); + + // The row is present and unverified, so the run reaches the decoder without rehashing + File.SetUnixFileMode(target, UnixFileMode.None); + if (CanRead(target)) + { + Assert.Skip("Running with permission to read anything, so the file stays readable"); + return; + } + + VerifyTask task = new(CreateOptions(workDir), database, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [target], + TestContext.Current.CancellationToken + ); + + // Assert + counts.Failed.Should().Be(1); + counts.Invalid.Should().Be(0); + counts.Verified.Should().Be(0); + } + finally + { + if (File.Exists(target)) + { + File.SetUnixFileMode(target, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + [Fact] + public async Task ExecuteAsync_UnreadableDirectory_CountsFailedRatherThanSkipping() + { + if (!OperatingSystem.IsLinux()) + { + Assert.Skip("File mode permissions are only enforced on Linux"); + return; + } + + if (!ImmichImageAvailable()) + { + Assert.Skip(ImageSkipReason); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + string lockedDir = Path.Combine(workDir, "locked"); + string filePath = Path.Combine(lockedDir, "photo.heic"); + try + { + // Arrange + Directory.CreateDirectory(lockedDir); + await File.WriteAllBytesAsync( + filePath, + [.. Enumerable.Range(0, 64).Select(i => (byte)i)], + TestContext.Current.CancellationToken + ); + File.SetUnixFileMode(lockedDir, UnixFileMode.None); + if (File.Exists(filePath)) + { + Assert.Skip("Running with permission to read anything, so the path stays visible"); + return; + } + + VerifyTask task = new(CreateOptions(workDir), null, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [filePath], + TestContext.Current.CancellationToken + ); + + // Assert + counts.Failed.Should().Be(1); + counts.Verified.Should().Be(0); + } + finally + { + if (Directory.Exists(lockedDir)) + { + File.SetUnixFileMode( + lockedDir, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + ); + } + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + // Verify only reads, so a file that vanished mid-run means the tree changed underneath it. + [Fact] + public async Task ExecuteAsync_FileMissingSinceIndexing_CountsFailed() + { + if (!ImmichImageAvailable()) + { + Assert.Skip(ImageSkipReason); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + try + { + // Arrange: a path that was enumerated but is gone by the time it is verified + string missing = Path.Combine(workDir, "gone.heic"); + + VerifyTask task = new(CreateOptions(workDir), null, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [missing], + TestContext.Current.CancellationToken + ); + + // Assert + counts.Failed.Should().Be(1); + counts.Verified.Should().Be(0); + counts.Invalid.Should().Be(0); + } + finally + { + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + // Hashing runs before the structural check when a database is in use. + // An exception there must not abort a run that may span a whole library. + [Fact] + public async Task ExecuteAsync_UnreadableFileWithDatabase_DoesNotAbortTheRun() + { + if (!OperatingSystem.IsLinux()) + { + Assert.Skip("File mode permissions are only enforced on Linux"); + return; + } + + if (!ImmichImageAvailable()) + { + Assert.Skip(ImageSkipReason); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + string locked = Path.Combine(workDir, "unreadable.jpg"); + string dbPath = Path.Combine(workDir, "Verify.db"); + try + { + // Arrange - two decodable files either side of one that cannot be opened + string source = fixture.SourceFile(TempDirectoryFixture.SmallJpegFile); + string first = Path.Combine(workDir, "a.jpg"); + string last = Path.Combine(workDir, "b.jpg"); + File.Copy(source, first); + File.Copy(source, last); + File.Copy(source, locked); + File.SetUnixFileMode(locked, UnixFileMode.None); + if (CanRead(locked)) + { + Assert.Skip("Running with permission to read anything, so the file stays readable"); + return; + } + + await using Database database = new(dbPath); + await database.InitializeAsync(TestContext.Current.CancellationToken); + VerifyTask task = new(CreateOptions(workDir), database, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [first, locked, last], + TestContext.Current.CancellationToken + ); + + // Assert - the unreadable file is counted, and the others are still verified + counts.Failed.Should().Be(1); + counts.Verified.Should().Be(2); + } + finally + { + if (File.Exists(locked)) + { + File.SetUnixFileMode(locked, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + private static bool CanRead(string filePath) + { + try + { + using FileStream stream = File.OpenRead(filePath); + return true; + } + catch (UnauthorizedAccessException) + { + return false; + } + } + + private const string ImageSkipReason = + "Docker or the Immich image is unavailable, and verify has no offline mode"; + + // Verify runs Immich's decoder for every file, so these cases cannot run without the image. + private static bool ImmichImageAvailable() + { + try + { + using System.Diagnostics.Process? probe = System.Diagnostics.Process.Start( + new System.Diagnostics.ProcessStartInfo("docker") + { + // Only the exit code is wanted, so render nothing rather than the manifest. + ArgumentList = { "image", "inspect", "--format", "", VerifyTask.ImmichImage }, + RedirectStandardOutput = true, + RedirectStandardError = true, + } + ); + if (probe is null) + { + return false; + } + + // Redirected pipes are drained before waiting. + // A child that fills one blocks on the write while the wait blocks on the child. + Task output = probe.StandardOutput.ReadToEndAsync(); + Task error = probe.StandardError.ReadToEndAsync(); + if (!probe.WaitForExit(60_000)) + { + return false; + } + + Task.WaitAll(output, error); + return probe.ExitCode == 0; + } + catch (System.ComponentModel.Win32Exception) + { + return false; + } + } + + private static CommandLine.Options CreateOptions(string path) => + new() + { + Path = new DirectoryInfo(path), + Threads = 1, + DryRun = false, + DatePath = false, + SkipBackup = false, + OutPath = null, + Format = "yyyy/MM/dd", + DeleteEmpty = false, + Move = false, + TagPath = false, + Tags = null, + DbFile = null, + Rehash = false, + ShortVideoDuration = MediaUtilities.ShortVideoDuration, + Reprocess = false, + MarkProcessed = false, + ImmichUrl = null, + ImmichApiKey = null, + TrashDbFile = null, + SkipDbFile = null, + LogOptions = new LoggerFactory.Options + { + Level = LogEventLevel.Information, + File = null, + FileClear = false, + }, + }; + + [Fact] + public void ImmichImage_UsesReleaseTag() + { + // Immich publishes no :latest tag, so a wrong tag here fails preflight on every run. + VerifyTask.ImmichImage.Should().Be("ghcr.io/immich-app/immich-server:release"); + } + + [Fact] + public void VerifyScript_ReferencesImmichOwnModules() + { + // Replacing these requires with a reimplementation would silently lose the fidelity. + ImmichVerifyScript.Verify.Should().Contain("media.repository.js"); + ImmichVerifyScript.Verify.Should().Contain("generateThumbnail"); + ImmichVerifyScript.Verify.Should().Contain("defaults.image.preview"); + ImmichVerifyScript.Preflight.Should().Contain(ImmichVerifyScript.PreflightSentinel); + } +} diff --git a/README.md b/README.md index 81b9575..a9c5722 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,51 @@ # PhotoCleaner -An application that prepares photos and videos for import into photo managers. +Utility to prepare photos and videos for import into photo managers. ## Build and Distribution -- **Source Code**: [GitHub][photocleaner-link], holding the source, the issues, and the CI/CD pipelines. -- **Versioned Releases**: [GitHub Releases][releases-link], attaching the Linux and Windows executables as a 7z archive. -- **Docker Images**: [Docker Hub][docker-link], multi-arch `linux/amd64` and `linux/arm64`. +- **Source Code**: [GitHub][github-link] for source, issues, and the CI/CD pipelines. +- **Versioned Releases**: [GitHub Releases][releases-link] for pre-compiled executables for Windows, Linux, and macOS. +- **Docker Images**: [Docker Hub][docker-link] for container images with all tools pre-installed. ### Build Status -[![Docker Image Size][docker-size-shield]][docker-link]\ -[![License][license-shield]][license-link] +[![Release Status][release-status-shield]][actions-link]\ +[![Docker Status][docker-status-shield]][actions-link]\ +[![Last Commit][last-commit-shield]][commit-link] ### Releases -[![Docker Latest][docker-latest-shield]][docker-link] +[![GitHub Release][release-version-shield]][releases-link]\ +[![GitHub Pre-Release][pre-release-version-shield]][releases-link]\ +[![Docker Latest][docker-latest-version-shield]][docker-link]\ +[![Docker Develop][docker-develop-version-shield]][docker-link] ### Release Notes -**Version: 1.0**: +**Version 1.1**: **Summary**: -- First published release, carrying the multi-arch Docker image and the GitHub release with the Linux and Windows executables attached. +- Added `verify` command to detect possibly corrupt images, e.g. Immich fails to generate a preview image. +- Added `-validate` to exiftool to report metadata warnings and errors. -See [Release History][history-link] for the full history. +> **Breaking**: commands now exit `2` when they complete with per-file failures. + +See [Release History][history-link] for complete release notes and older versions. ## Table of Contents - [Overview](#overview) - [Usage](#usage) - [Command Line Syntax](#command-line-syntax) + - [Exit Codes](#exit-codes) - [Examples](#examples) - [Processing Flow](#processing-flow) - [Undo Flow](#undo-flow) - [Import Flow](#import-flow) - [Trash Flow](#trash-flow) +- [Verify Flow](#verify-flow) - [Supported File Types](#supported-file-types) - [Docker](#docker) - [Workflow Example](#workflow-example) @@ -44,7 +53,6 @@ See [Release History][history-link] for the full history. - [Development Environment Setup](#development-environment-setup) - [Install](#install) - [Update](#update) -- [License](#license) ## Overview @@ -55,12 +63,12 @@ PhotoCleaner analyzes and transforms media files through a validation pipeline t compound extensions (e.g. `photo.heic.jpg` -> `photo.jpg`). - **Renames mixed-case extensions**: Converts uppercase or mixed-case extensions to lowercase (e.g. `.JPG` -> `.jpg`). -- **Handles Live Photos**: Removes Apple Live Photo video components - videos <= 1s are always - removed; videos <= 4s with a candidate companion image (same basename, or basename with `_hevc` - suffix stripped) are removed when both files share the same `ContentIdentifier` EXIF tag; - longer videos with a matching image trigger a warning but are kept. -- **Converts video formats**: Remuxes MTS, M2TS, and MKV to MP4; re-encodes WMV, AVI, 3GP, and - GIF to MP4 (H.264/AAC); re-encodes PCM audio to AAC in MOV and MP4 files while preserving +- **Handles Live Photos**: Removes Apple Live Photo video components. Videos <= 1s are always + removed. Videos <= 4s with a candidate companion image (same basename, or basename with `_hevc` + suffix stripped) are removed when both files share the same `ContentIdentifier` EXIF tag. + Longer videos with a matching image trigger a warning but are kept. +- **Converts video formats**: Remuxes MTS, M2TS, and MKV to MP4, re-encodes WMV, AVI, 3GP, and + GIF to MP4 (H.264/AAC), and re-encodes PCM audio to AAC in MOV and MP4 files while preserving the video stream. All source metadata (including `ContentIdentifier` and other QuickTime tags) is copied to the converted file using `exiftool -TagsFromFile`. - **Imports into date folders** (via `import` command): Copies (default) or moves supported @@ -79,6 +87,12 @@ PhotoCleaner analyzes and transforms media files through a validation pipeline t database. This trash DB can then be used with `import --trashdb` to skip files that were already imported and trashed in Immich, and with `process --trashdb` to delete files trashed in Immich after upload, preventing re-import of known duplicates. +- **Verifies files render in Immich** (via `verify` command): Answers whether a file will survive + Immich's preview generation, which metadata checks cannot predict. Immich's own thumbnail + pipeline is run inside the `immich-server` image, so the verdict is the one Immich will reach + rather than an approximation of it. That decoder is the sole authority: PhotoCleaner does not + parse container formats itself, because a format it did not recognize would be indistinguishable + from a damaged one. Docker is required. Nothing is modified, and `verify` only ever reads. - **Warns on DNG version**: Flags DNG files with a format version newer than v1.4 that may not render correctly in older applications. @@ -94,7 +108,7 @@ detailed logging of all operations. ```text $> PhotoCleaner --help Description: - PhotoCleaner - An application that prepares photos and videos for import into photo managers. + PhotoCleaner - Utility to prepare photos and videos for import into photo managers. Usage: PhotoCleaner [command] [options] @@ -105,6 +119,7 @@ Commands: import Import media files into date-based subdirectories index Index files into the database for deduplication tracking trash Sync trashed asset hashes from Immich + verify Verify that media files can be rendered by Immich Options: --loglevel Set the log level [default: Information] @@ -120,7 +135,7 @@ Description: Process media files Options: - --path (REQUIRED) The directory path to process + --path (REQUIRED) The media directory path --dryrun Perform a dry run without making changes --threads Number of parallel threads [default: 4] --skipbackup Skip creating backup files (disables undo) @@ -128,8 +143,8 @@ Options: --db SQLite database file for file state tracking --rehash Force rehashing of all files, ignoring size/mtime cache --duration Maximum duration in seconds below which a video is considered a short clip and deleted [default: 1] - --reprocess Re-process files even if already marked as processed in the database - --trashdb SQLite database with Immich trash hashes (read-only). Files matching are deleted from disk and the DB + --reprocess Re-run every file even if the database marks it done + --trashdb SQLite database with Immich trash hashes (read-only) ``` ```text @@ -138,7 +153,7 @@ Description: Undo media file processing Options: - --path (REQUIRED) The directory path to process + --path (REQUIRED) The media directory path --dryrun Perform a dry run without making changes ``` @@ -148,11 +163,11 @@ Description: Import media files into date-based subdirectories Options: - --path (REQUIRED) The directory path to process + --path (REQUIRED) The media directory path --dryrun Perform a dry run without making changes --threads Number of parallel threads [default: 4] --outpath (REQUIRED) Output directory for organized files - --format Date format for output subdirectory names [default: yyyy/MM/dd] + --format Date format for output subdirectory names; use '/' to create nested subdirectories (e.g. yyyy/MM/dd) [default: yyyy/MM/dd] --deleteempty Delete empty subdirectories under the target directory after the command completes --move Move files instead of copying (default: copy) --tagpath Apply path sub-directory components as XMP Subject tags to the organized file @@ -170,11 +185,11 @@ Description: Index files into the database for deduplication tracking Options: - --path (REQUIRED) The directory path to index + --path (REQUIRED) The media directory path --threads Number of parallel threads [default: 4] --db (REQUIRED) SQLite database file for file state tracking --rehash Force rehashing of all files, ignoring size/mtime cache - --processed Mark newly inserted rows as already processed (use when seeding a Process.db) + --processed Mark newly inserted rows as already processed (use when seeding a Process.db from existing files) ``` ```text @@ -189,41 +204,54 @@ Options: --trashdb (REQUIRED) SQLite database to store Immich trash hashes ``` +```text +$> PhotoCleaner verify --help +Description: + Verify that media files can be rendered by Immich + +Options: + --path (REQUIRED) The media directory path + --threads Number of parallel threads [default: 4] + --db SQLite database file for file state tracking + --rehash Force rehashing of all files, ignoring size/mtime cache + --reprocess Re-run every file even if the database marks it done +``` + **Option notes:** -- `--path` - must point to an existing directory; accepts exactly one directory per command +- `--path`: must point to an existing directory. Accepts exactly one directory per command invocation. -- `--threads` - defaults to `min(CPU count, 4)`; must be `> 0` and `<= CPU count`. -- `--skipbackup` - opt-in (`process` only); skips all `.bak` file creation. The `undo` +- `--threads`: defaults to `min(CPU count, 4)`. Must be `> 0` and `<= CPU count`. +- `--skipbackup`: opt-in (`process` only). Skips all `.bak` file creation. The `undo` command cannot reverse a run made with this flag. -- `--outpath` - required for `import`; target directory (created on demand). -- `--format` - optional (`import` only); a C# date format string used to name date - subdirectories (default `"yyyy/MM/dd"`). Must be date-only - time components are rejected. +- `--outpath`: required for `import`. Target directory (created on demand). +- `--format`: optional (`import` only). A C# date format string used to name date + subdirectories (default `"yyyy/MM/dd"`). Must be date-only, so time components are rejected. Files with no EXIF date land in a `"0001/01/01"` fallback bucket. -- `--deleteempty` - optional (`import`, `process`); after the command completes, deletes +- `--deleteempty`: optional (`import`, `process`). After the command completes, deletes empty child subdirectories from the target directory (deepest first). For `import` the - target is `--outpath`; for `process` it is `--path` (which is operated on in-place). The + target is `--outpath`, and for `process` it is `--path` (which is operated on in-place). The target root itself is never deleted. Useful for cleaning up directory trees left behind after `process` deletes files (live photos, originals when `--skipbackup`) or after pruning organized output. -- `--move` - optional (`import` only); moves files instead of copying. Default behavior is +- `--move`: optional (`import` only). Moves files instead of copying. Default behavior is to copy, which preserves the source files. Use `--move` when the source directory is temporary. -- `--tagpath` - optional (`import` only); splits the source sub-directory path relative to +- `--tagpath`: optional (`import` only). Splits the source sub-directory path relative to `--path` into tokens and writes each token as an `XMP:Subject` tag on the destination file using exiftool. Files at the root of `--path` receive no tags. Tags are applied with a remove-then-add pattern (`-XMP:Subject-= / -XMP:Subject+=`) so existing tags are preserved and duplicates are not created. Only file types that support XMP writes are tagged. -- `--tags` - optional (`import` only); a comma-separated list of `XMP:Subject` tags applied to +- `--tags`: optional (`import` only). A comma-separated list of `XMP:Subject` tags applied to every organized file (e.g. `--tags "vacation,family trip,2018"`). Tags are applied using the - same remove-then-add pattern as `--tagpath`. Can be combined with `--tagpath` - both sets of + same remove-then-add pattern as `--tagpath`. Can be combined with `--tagpath`, and both sets of tags are merged. Only file types that support XMP writes are tagged. -- `--datepath` - optional (`import` only); when a file has no embedded creation date, infers +- `--datepath`: optional (`import` only). When a file has no embedded creation date, infers one from the filename or directory path structure (via `DateFromPath`) and writes it to the destination file before restoring mtime. Opt-in because writing to files is destructive and the source path context is only available during `import` (before files move to date-based directories). -- `--db ` - optional for `import` and `process`, **required** for `index`; path to a +- `--db `: optional for `import` and `process`, **required** for `index`. Path to a SQLite database file. Uses a single `files` table (`path` PRIMARY KEY, `sha256`, `sha1`, `file_size`, `mtime_ticks`, `is_processed`). The schema is the same for every command, but the **meaning of the `path` column depends on which command writes the row**, so each @@ -237,14 +265,14 @@ Options: The DB file is created automatically on first use. **The two stages MUST use separate DB files**: if `import` and `process` shared one DB, `process` would overwrite the source-content hashes that `import` wrote (because `import` mutates the dest file via XMP tag injection, so the dest - hash diverges from the source hash; `process` then re-hashes the dest and clobbers the row). - This was a real bug; the per-stage layout is the fix. -- `--trashdb ` - **required** for `trash`, optional for `import` and `process`; path + hash diverges from the source hash, and `process` then re-hashes the dest and clobbers the row). + This was a real bug, and the per-stage layout is the fix. +- `--trashdb `: **required** for `trash`, optional for `import` and `process`. Path to a SQLite database with Immich trash hashes. - For `trash`: hashes are fetched from the Immich API and written to the database. - For `import`: files matching a trash hash are skipped (read-only). This is the durable "do not re-import" record beyond Immich's own ~30-day trash retention. Without it, a file - the user trashed > 30 days ago can come back the next time icloudpd re-downloads it, because + the user trashed > 30 days ago can come back the next time the downloader re-fetches it, because Immich no longer has the hash to deduplicate against on re-upload. Limitation: `import` compares the **source-file** SHA-1 against the trash DB. When `import` rewrites the destination via `--tags`, `--tagpath`, or `--datepath`, the dest file's @@ -256,30 +284,53 @@ Options: cleans up files trashed in Immich after upload, before the next `immich-cli` upload would re-upload them. Also acts as the safety net for files that `import --trashdb` could not match because of the source-vs-dest SHA-1 drift described above. -- `--skipdb ` - optional for `import`; path to a SQLite database with indexed files +- `--skipdb `: optional for `import`. Path to a SQLite database with indexed files to be skipped (read-only). Files matching a record in this DB are skipped without being recorded. Use to skip files already present in another collection. -- `--url ` - **required** for `trash`; the Immich server URL (e.g. `http://immich:2283`). -- `--apikey ` / `--apikey-file ` - the Immich API key for `trash`; supply it via +- `--url `: **required** for `trash`. The Immich server URL (e.g. `http://immich:2283`). +- `--apikey ` / `--apikey-file `: the Immich API key for `trash`. Supply it via exactly one of these two mutually exclusive options (one is required). Create the key in Immich - under Account Settings > API Keys. `--apikey` passes the key inline; `--apikey-file` points to a + under Account Settings > API Keys. `--apikey` passes the key inline, while `--apikey-file` points to a file whose trimmed contents are the key, keeping the secret out of shell history and process arguments. The file must exist and be non-empty. -- `--rehash` - optional (`process`, `import`, `index`); forces SHA-256 recomputation for +- `--rehash`: optional (`process`, `import`, `index`). Forces SHA-256 recomputation for every file, bypassing the size/mtime cache. SHA-1 is also recomputed when `--trashdb` is in use. Use when file content may have changed without the modification timestamp being updated. -- `--duration` - optional (`process` only); overrides the short-video deletion threshold +- `--duration`: optional (`process` only). Overrides the short-video deletion threshold (default `1.0` seconds). Videos in a live-photo-compatible format whose duration is <= this value are always deleted. Must be `> 0`. -- `--reprocess` - optional (`process` only); when set, ignores the `is_processed` flag in +- `--reprocess`: optional (`process`, `verify`). When set, ignores the `is_processed` flag in the database and processes every file regardless of prior run history. Useful after changing pipeline settings (e.g. `--duration`) without wiping the database. -- `--processed` - optional (`index` only); marks newly inserted rows with `is_processed = 1`. +- `--processed`: optional (`index` only). Marks newly inserted rows with `is_processed = 1`. Use this when seeding a Process.db from existing files so a subsequent `process` run treats them as already processed and only touches new arrivals. The flag does not flip the bit on rows that already exist in the DB. +### Exit Codes + +Every command uses the same three codes, so a pipeline can branch on the result without parsing +logs: + +| Code | Meaning | +| ---- | ------- | +| `0` | Success. The command completed and every file succeeded. | +| `1` | Error. The command could not complete: unhandled exception, fatal configuration error, cancellation, or a failed `verify` preflight. | +| `2` | Completed with failures. The command ran to completion, but one or more files failed or failed verification, or `trash` synced only part of the server. | + +The distinction between `1` and `2` matters most for `verify`. A `1` means the check itself could +not run at all, because Docker was unreachable or the Immich image could not be prepared, and it +says nothing about any file. A `2` means the run completed and one or more files were invalid **or could +not be verified**, the latter covering a file that could not be read or that no verdict came back +for. A script must never treat an infrastructure failure as a verdict on the collection, and should +read the `Invalid` and `Failed` counts to tell a bad file from a gap in coverage. + +`trash` exits `2` when pagination stops early, which leaves the trash database holding fewer +hashes than the server. That database is used by `import --trashdb` and `process --trashdb` to skip +files, so a short one silently re-imports assets that were trashed, and a pipeline gating on the +exit code should not go on to upload against it. + ### Examples ```bash @@ -346,21 +397,32 @@ PhotoCleaner import --path /home/user/Photos --outpath /home/user/Organized --db # Import and skip files already in another collection (read-only reference) PhotoCleaner import --path /home/user/Photos --outpath /home/user/Organized --skipdb /data/existing-collection.db +# Verify that Immich can render every file, before uploading +PhotoCleaner verify --path /home/user/Intermediate --db /data/verify.db + # Full workflow with Immich trash integration PhotoCleaner trash --url http://immich:2283 --apikey $IMMICH_KEY --trashdb /data/trash.db PhotoCleaner import --path /home/user/iCloud --outpath /home/user/Intermediate --db /data/photos.db --trashdb /data/trash.db PhotoCleaner process --path /home/user/Intermediate --db /data/process.db +PhotoCleaner verify --path /home/user/Intermediate --db /data/verify.db ``` ## Processing Flow 1. **File enumeration**: Recursively scans all specified directories. 2. **Case conflict detection**: Identifies files with the same name but different casing that - would collide on case-insensitive file systems; renames conflicting files before processing. + would collide on case-insensitive file systems, then renames them before processing. 3. **Per-file validation pipeline** (runs in parallel, stops on first action per file): - 1. Rename to canonical MIME extension - corrects mismatches and strips compound extensions. + 0. Act on the exiftool `-validate` verdict, which rides along with the metadata read that + `process` already performs and so costs nothing extra. A file exiftool reports **errors** + on is marked invalid and no further step touches it. **Warnings are logged at debug level + and nothing more**: measured across a real collection, roughly three quarters of perfectly + healthy files carry at least one (odd IFD offsets, non-standard maker note tags, short + IPTC fields), so failing on warnings would condemn most of a library. This is a cheap net + for a rare case, not a substitute for the `verify` command. + 1. Rename to canonical MIME extension, correcting mismatches and stripping compound extensions. 2. Rename mixed-case extension to lowercase. - 3. Delete short or Live Photo video clips: videos <= 1s are always deleted; videos <= 4s + 3. Delete short or Live Photo video clips. Videos <= 1s are always deleted, and videos <= 4s with a candidate companion image (direct name match or `_hevc`-suffix match) are deleted when both files share a matching `ContentIdentifier` tag. 4. Convert legacy or incompatible video formats to MP4: @@ -370,13 +432,16 @@ PhotoCleaner process --path /home/user/Intermediate --db /data/process.db - After every conversion: all source metadata copied to output via `exiftool -TagsFromFile` 5. Warn on DNG version > v1.4. 4. **Reprocess loop**: Any file that was renamed or converted is re-queued until stable. -5. **Results summary**: Reports counts of failed, modified, and successfully processed files; - lists any unrecognized file extensions. +5. **Results summary**: Reports counts of failed, invalid, modified, and successfully processed + files, and lists any unrecognized file extensions. Exits `2` if anything failed or was invalid. + +The `import` command applies the same exiftool validation, skipping any file that reports errors +rather than pulling it into the collection. ## Undo Flow Every file modification or deletion made by `process` creates a `.bak` backup alongside the -original: the first backup is `X.bak`; if that already exists (from a prior run) the next is +original: the first backup is `X.bak`, and if that already exists (from a prior run) the next is `X.bak1`, then `X.bak2`, etc. The `undo` command reverses all processing by scanning the given directories for backup files and applying a two-pass algorithm: @@ -389,7 +454,7 @@ directories for backup files and applying a two-pass algorithm: - *Non-derived* base: delete the current file if it exists (overwritten in-place), rename `X.bak` -> `X` to restore the original. The converted output is located via the `X.bak.out` companion file written at conversion time (handles uniquified names like - `stem_1.mp4`); if no companion exists, falls back to deleting `stem.mp4` when present + `stem_1.mp4`). If no companion exists, falls back to deleting `stem.mp4` when present and untracked (legacy single-run heuristic). **Known limitation**: extension renames that target a filename that did not previously exist @@ -412,7 +477,7 @@ directories to `outpath/date/filename`: - `--trashdb`: if the file matches a hash in the Immich trash DB, the file is skipped (counted as "trashed in Immich"). - `--skipdb`: if the file matches a record in the reference DB, the file is skipped - (counted as "skipped by reference"). This is a read-only check - no records are written. + (counted as "skipped by reference"). This is a read-only check, so no records are written. - `--db` (Import.db): if the source SHA-256 is already present (from a previous import run), the file is skipped (counted as "skipped"). Otherwise, the file is copied/moved and a record is inserted **keyed by the SOURCE path** (not the dest path) with the source @@ -421,15 +486,15 @@ directories to `outpath/date/filename`: Process.db cannot clobber the dedup key. The DB file is created automatically on first use. 3. **Date resolution**: reads EXIF metadata via `exiftool`. Uses `EXIF:DateTimeOriginal` or `QuickTime:CreateDate` (whichever is set). Falls back to `DateTime.MinValue` when no date - is found - those files land in a `"0001/01/01"` bucket (with the default `yyyy/MM/dd` format), + is found. Those files land in a `"0001/01/01"` bucket (with the default `yyyy/MM/dd` format), making undated files easy to locate and handle manually. 4. **Subdirectory naming**: the date is formatted using `--format` (default `"yyyy/MM/dd"`). - The format is validated at startup - time components are rejected. + The format is validated at startup, and time components are rejected. 5. **Copy or move**: by default files are copied and the source is preserved. Pass `--move` to remove the source file after a successful copy. 6. **Tagging**: `--tagpath` splits the source sub-directory path relative to `--path` into tokens and writes each as an `XMP:Subject` tag. `--tags` applies explicit comma-separated - tags to every file. Both can be combined - tags are merged and deduplicated. Applied after + tags to every file. Both can be combined, and tags are merged and deduplicated. Applied after copy/move, before mtime restore. Files at the root of `--path` receive no path tags. 7. **Collision handling**: if a file with the same name already exists in the destination, `_1`, `_2`, ... suffixes are appended (e.g. `photo_1.jpg`). A warning is logged. @@ -459,6 +524,50 @@ were already imported and trashed in Immich. The trash database is append-only. If an asset is restored (un-trashed) in Immich, its hash remains in the database. Delete the database file and re-run `trash` to rebuild from scratch. +## Verify Flow + +The `verify` command answers one question: will Immich be able to generate a preview for this +file? It exists because a file can be byte-complete, pass every other check, upload successfully, +and then fail thumbnail generation forever. Metadata inspection cannot see this, because the file +reports as a perfectly clean HEIC or DNG, so the only reliable answer comes from running the decoder +Immich runs. + +`verify` is a standalone pipeline step rather than an option on `process`, so the calling script +chooses where to run it and whether a failure should stop the pipeline or merely be recorded. + +1. **Partition**: non-media files are ignored. With `--db`, files already verified and unchanged + since are skipped, so a repeat run over a large collection is cheap. + + **Give `verify` its own database file.** It records the verified state in the same + `is_processed` column that `process` writes, so pointing `--db` at a `Process.db` makes + `verify` skip every file as "already verified" when they were only processed. Nothing detects + this, so use a separate `Verify.db`, as the examples below do. +2. **Decode pass** (requires Docker). Every file is handed to + Immich's own `MediaRepository` running inside the `immich-server` image, using the same + `generateThumbnail`, the same libvips build, the same libheif and libraw versions, and for RAW + the same embedded-preview extraction. Paths are streamed in batches over stdin so container + startup is paid once per batch rather than once per file. The media directory is mounted + read-only at the fixed container path `/photocleaner`, and every path is translated onto it + before being sent in, so a host path that is not a valid container path still works. + + The decoder is the only judge of a file's health. PhotoCleaner deliberately carries no + container parser of its own, because such a parser condemns whatever it fails to understand, + and an unfamiliar but valid format is indistinguishable from a damaged one from the inside. + A file that cannot be read, or that vanishes mid-run, counts as failed rather than invalid, + since neither is evidence of damage. +3. **Report**: logs each rejection with the decoder's own message, then a summary. Exits `2` if + any file is invalid or any file failed. + +Nothing is modified, moved, or deleted. `verify` only ever reads. + +Because the decode pass calls Immich's own compiled code rather than reimplementing its pipeline, +it tracks Immich's behavior across releases automatically. It runs `docker` directly, so it must +be run somewhere Docker is available, and is not supported from inside PhotoCleaner's own +container. There is no offline mode, because the decoder is the whole check. + +Before any file is judged, the command runs a preflight against the image. If Docker is +unreachable or the image cannot be prepared, it exits `1` without marking a single file invalid. + ## Supported File Types - **Images**: ARW, CR2, DNG, HEIC, HEIF, JPEG, JPG, NEF, ORF, PNG, PSD, RW2, TIF, TIFF @@ -525,29 +634,89 @@ docker run --rm \ ## Workflow Example -**Run [icloudpd][icloudpd-link] to download photos from iCloud**: +**Run [kei][kei-link] to download photos from iCloud**: + +kei keeps its settings in a TOML file rather than on the command line, so the same configuration +serves the one-shot and the long-running forms. A minimal `config.toml`: + +```toml +[auth] +username = "your@icloud.email" + +[download] +directory = "/photos" +folder_structure = "%Y/%m/%Y-%m-%d" + +[filters] +media = ["photos", "videos", "live-photos"] + +[photos] +raw_policy = "prefer-raw" + +[watch] +interval = 86400 +``` + +Authenticate once, interactively, so the session is stored in the data directory: ```shell #!/bin/bash set -Eeuo pipefail -docker run -it --rm --name icloudpd \ - -v $(pwd)/.icloudpd:/cookies \ - -v /data/media/Test:/data \ - -e TZ=America/Los_Angeles \ - docker.io/icloudpd/icloudpd:latest \ - icloudpd \ - --cookie-directory /cookies \ - --username your@icloud.email \ - --directory /data \ - --set-exif-datetime \ - --folder-structure "{:%Y/%m}" \ - --recent 1000 - # --skip-created-before 2025-01-01 +docker run -it --rm --name kei \ + -v /data/appdata/kei/config:/config \ + -v /data/media/icloud:/photos \ + -e KEI_DATA_DIR=/config \ + ghcr.io/rhoopr/kei:latest \ + kei login ``` -**Run [PhotoCleaner][photocleaner-link] to sync Immich trash hashes** (optional, prevents re-importing trashed files): +Then sync on demand: + +```shell +#!/bin/bash + +set -Eeuo pipefail + +docker run -it --rm --name kei \ + -v /data/appdata/kei/config:/config \ + -v /data/media/icloud:/photos \ + -e KEI_DATA_DIR=/config \ + ghcr.io/rhoopr/kei:latest \ + kei sync \ + --config /config/config.toml \ + --recent 30d + # --dry-run to preview without writing +``` + +Or run it as a service that keeps mirroring on the `[watch]` interval: + +```yaml +services: + kei: + image: ghcr.io/rhoopr/kei:latest + container_name: kei + restart: unless-stopped + environment: + - TZ=America/Los_Angeles + - KEI_DATA_DIR=/config + volumes: + - /data/media/icloud:/photos + - /data/appdata/kei/config:/config + secrets: + - icloud_password + command: + - kei + - service + - run + - --config + - /config/config.toml + - --password-file + - /run/secrets/icloud_password +``` + +**Run PhotoCleaner to sync [Immich][immich-link] trash hashes** (optional, prevents re-importing trashed files): ```shell #!/bin/bash @@ -563,10 +732,10 @@ docker run --rm \ --trashdb /db/trash.db ``` -**Run [PhotoCleaner][photocleaner-link] to import new photos**: +**Run PhotoCleaner to import new photos**: -Copy only new files (not already in the DB and not trashed in Immich) from the icloudpd -directory to an intermediate directory, without touching the icloudpd originals: +Copy only new files (not already in the DB and not trashed in Immich) from the download +directory to an intermediate directory, without touching the downloaded originals: ```shell #!/bin/bash @@ -586,7 +755,7 @@ docker run --rm \ --threads 4 ``` -**Run [PhotoCleaner][photocleaner-link] to process the intermediate photos**: +**Run PhotoCleaner to process the intermediate photos**: ```shell #!/bin/bash @@ -601,6 +770,31 @@ docker run --rm \ --threads 4 ``` +**Run PhotoCleaner to verify Immich can render the photos**: + +Run this on the host rather than inside the PhotoCleaner container: the decode pass invokes +`docker` to run Immich's own decoder, and Docker-in-Docker is not supported. Capture the exit +code instead of letting `set -e` abort, so the upload can be skipped while the run still reports +cleanly: + +```shell +#!/bin/bash + +set -Eeuo pipefail + +rc=0 +PhotoCleaner verify \ + --path /data/media/intermediate \ + --db /data/media/verify.db \ + --threads 4 || rc=$? + +case $rc in + 0) echo "All files verified" ;; + 2) echo "Invalid files found - skipping upload, review the log" >&2; exit 2 ;; + *) echo "Verification could not run (exit $rc) - this says nothing about the files" >&2; exit "$rc" ;; +esac +``` + **Run [Immich CLI][immich-cli-link] to import photos into Immich**: ```shell @@ -649,7 +843,7 @@ immich-go upload from-folder --server=https://your.immich.server \ ## Questions or Issues -Report a bug, request a feature, or ask a question in [GitHub Issues][issues-link]. +Ask questions in the [Discussions][discussions-link] forum and report bugs in [GitHub Issues][issues-link]. ## Development Environment Setup @@ -706,28 +900,40 @@ dotnet tool update --all dotnet outdated --upgrade:prompt ``` -## License +## License Licensed under the [MIT License][license-link]\ ![GitHub License][license-shield] -[docker-latest-shield]: https://img.shields.io/docker/v/ptr727/photocleaner/latest?logo=docker&label=Docker%20Latest -[docker-size-shield]: https://img.shields.io/docker/image-size/ptr727/photocleaner/latest?logo=docker&label=Image%20Size +[docker-develop-version-shield]: https://img.shields.io/docker/v/ptr727/photocleaner/develop?label=Docker%20Develop&logo=docker&color=orange +[docker-latest-version-shield]: https://img.shields.io/docker/v/ptr727/photocleaner/latest?label=Docker%20Latest&logo=docker +[docker-status-shield]: https://img.shields.io/github/actions/workflow/status/ptr727/PhotoCleaner/publish-release.yml?event=schedule&logo=github&label=Docker%20Build +[last-commit-shield]: https://img.shields.io/github/last-commit/ptr727/PhotoCleaner?logo=github&label=Last%20Commit [license-shield]: https://img.shields.io/github/license/ptr727/PhotoCleaner?label=License +[pre-release-version-shield]: https://img.shields.io/github/v/release/ptr727/PhotoCleaner?include_prereleases&label=GitHub%20Pre-Release&logo=github +[release-status-shield]: https://img.shields.io/github/actions/workflow/status/ptr727/PhotoCleaner/publish-release.yml?event=schedule&logo=github&label=Releases%20Build +[release-version-shield]: https://img.shields.io/github/v/release/ptr727/PhotoCleaner?logo=github&label=GitHub%20Release [history-link]: ./HISTORY.md [license-link]: ./LICENSE - + +[actions-link]: https://github.com/ptr727/PhotoCleaner/actions +[commit-link]: https://github.com/ptr727/PhotoCleaner/commits/main +[discussions-link]: https://github.com/ptr727/PhotoCleaner/discussions [docker-link]: https://hub.docker.com/r/ptr727/photocleaner -[icloudpd-link]: https://icloud-photos-downloader.github.io -[immich-cli-link]: https://docs.immich.app/features/command-line-interface/ -[immich-go-link]: https://github.com/simulot/immich-go +[github-link]: https://github.com/ptr727/PhotoCleaner [issues-link]: https://github.com/ptr727/PhotoCleaner/issues -[photocleaner-link]: https://github.com/ptr727/PhotoCleaner [releases-link]: https://github.com/ptr727/PhotoCleaner/releases + + + +[kei-link]: https://github.com/rhoopr/kei +[immich-link]: https://immich.app +[immich-cli-link]: https://docs.immich.app/features/command-line-interface +[immich-go-link]: https://github.com/simulot/immich-go diff --git a/WORKFLOW.md b/WORKFLOW.md index d1d9889..aeeeeb0 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -2,13 +2,13 @@ The guide for CI/CD **workflows** (GitHub Actions): a deliberate mixture of code style, architecture, a **behavioral contract** (expected inputs and outputs), and a **test methodology**. Code style lives in [`CODESTYLE.md`][codestyle]. This file is its sibling for everything under [`.github/workflows/`][workflows]. -Its defining principle: **it describes required outcomes, not a required implementation.** Two repos may implement the same guarantee with different YAML. A workflow is correct when it **satisfies the contract** in section 4 and is **defect-free against the expected inputs and outputs** - not when it matches a reference implementation byte for byte. The conventions in section 2 keep workflows legible. The contract in section 4 is what they must *do*. +Its defining principle: **it describes required outcomes, not a required implementation.** Two repos may implement the same guarantee with different YAML. A workflow is correct when it **satisfies the contract** in section 4 and is **defect-free against the expected inputs and outputs**, not when it matches a reference implementation byte for byte. The conventions in section 2 keep workflows legible. The contract in section 4 is what they must *do*. Given this document, an agent must be able to do three things to any project: 1. **Audit** - statically check the workflows against the conventions (section 2) and the structural facts each guarantee implies (section 5A). 2. **Test** - trace the expected inputs/outputs (section 5B) and, where warranted, drive a live probe (section 5C). -3. **Assess** - render a verdict: **operational** (every *applicable* guarantee holds and every *applicable* scenario's observed output equals the expected) or **not operational** (any mismatch - which is a *defect*, not a style nit). +3. **Assess** - render a verdict: **operational** (every *applicable* guarantee holds and every *applicable* scenario's observed output equals the expected) or **not operational** (any mismatch, which is a *defect*, not a style nit). > **Canonical scope.** This document is authoritative for the workflow contract and test methodology (sections 3 to 6). The conventions in section 2 and the release policy also live in `GOVERNANCE.md` ("Workflow YAML Conventions" and "Release Model"), which is authoritative where the two overlap. Section 2 restates them so this file reads on its own. On any conflict in that overlap, `GOVERNANCE.md` wins. @@ -17,7 +17,7 @@ The guarantees are distilled from failures observed in practice and stated as th ## 1. Purpose and How to Use This Document - **Contract, not implementation.** Conform to the *outcomes* in section 4. Shape, job names, and file layout may differ between repos, but the input/output behavior may not. -- **Applicability.** A guarantee (or a 5A check, or a 5B scenario) is **applicable** only if the repo contains the construct it governs - a given target, a transfer artifact, a registry push, a wrapper-version source. An item that governs an absent construct is **N/A**: record it as N/A and **exclude it from the verdict**. N/A is never a defect. Section 6 names which items go N/A per project type. A near-empty pipeline (source-only) is mostly N/A and that is fine. +- **Applicability.** A guarantee (or a 5A check, or a 5B scenario) is **applicable** only if the repo contains the construct it governs: a given target, a transfer artifact, a registry push, a wrapper-version source. An item that governs an absent construct is **N/A**: record it as N/A and **exclude it from the verdict**. N/A is never a defect. Section 6 names which items go N/A per project type. A near-empty pipeline (source-only) is mostly N/A and that is fine. - **Operational is binary.** A workflow is operational only if every *applicable* guarantee holds. A single applicable input/output mismatch is a defect and makes the workflow non-operational, regardless of how clean the YAML looks. - **Default branch.** Guarantees say "default branch" portably. It is implemented as the literal `main` in several places (the validate gate, the `prerelease` expression, and `version.json`'s `publicReleaseRefSpec`). These MUST all reference the repo's *actual* default branch. A divergence is a defect (section 5A). - **Two layers when auditing.** The pipeline splits into an **orchestrator** layer (the PR entry workflow, the publisher, and the version/release/badge jobs) and a **build-leaf** layer (`build--task.yml`). Inputs like `github`/`nuget`/`dockerhub`/`expect_release_assets` live on the orchestrator. A leaf only ever receives `ref`/`branch`/`smoke` (and a derived `push`). When a check names an input, assert it in the layer that declares it. @@ -30,9 +30,9 @@ Prescriptive style/legibility rules. Cheap to check, necessary but not sufficien - **Action pinning.** Pin **every** action to a commit SHA with a trailing `# vX.Y.Z` comment. Use `# vX` only when the upstream floating major tag has no specific patch SHA. The single documented no-pin exception is a tool whose tag stream lags `master` such that tag-tracking would downgrade (here, `dotnet/nbgv@master`). Invent no others. - **Filename.** Reusable workflows (`on: workflow_call`) end in `-task.yml`. Entry-point workflows do not (`-pull-request.yml`, `-release.yml`). Lowercase, hyphen-separated. - **Workflow `name:`.** Reusable names end in **"task"**. Entry-point names end in **"action"**. -- **Job and step `name:`.** Every job ends in **"job"**, every step in **"step"** - including a ruleset-bound required-check job, whose `name:` and the ruleset `context:` are one string renamed together (never independently). +- **Job and step `name:`.** Every job ends in **"job"**, every step in **"step"**, including a ruleset-bound required-check job, whose `name:` and the ruleset `context:` are one string renamed together (never independently). - **Concurrency.** Top-level workflows declare `concurrency: { group: '${{ github.workflow }}-${{ github.ref }}', cancel-in-progress: true }`. Document exceptions inline (D7). -- **Shells.** Every multi-line bash `run:` - and every committed `.sh` script - starts `set -Eeuo pipefail`. +- **Shells.** Every multi-line bash `run:` (and every committed `.sh` script) starts `set -Eeuo pipefail`. - **Conditionals.** Multi-line `if:` uses the folded scalar `if: >-`. - **Boolean inputs.** A boolean used by both `workflow_call` and `workflow_dispatch` is declared in **both** trigger blocks, and `workflow_dispatch` delivers the **string** `"true"`/`"false"`, so any `if:` compares both forms: `${{ inputs.foo == true || inputs.foo == 'true' }}`. - **Reusable-workflow permissions.** Job-level `permissions:` are validated **before** `if:`, so even a skipped job needs valid permissions. Grant least privilege. A reusable callee's extra scope (e.g. `actions: write` for cleanup) is granted by the **caller**. @@ -53,7 +53,7 @@ flowchart LR main -.->|no back-merge| develop ``` -`operational` repos (live-service config; `workflowModel: operational`) commit directly to `develop` and promote a known-good snapshot to `main` via an occasional PR: +`operational` repos (live-service config, `workflowModel: operational`) commit directly to `develop` and promote a known-good snapshot to `main` via an occasional PR: ```mermaid flowchart LR @@ -61,13 +61,13 @@ flowchart LR develop -->|merge commit, enforced lint CI| main ``` -Their CI is lint/validation only (editorconfig/EOL plus domain linters - Home Assistant or ESPHome config validation, a firmware build - **no unit tests**), so the D-guarantees below that assume a build/test pipeline are **N/A** exactly as for `source-only` (Section 6). What binds: the promotion gate - the `develop -> main` PR must pass the required `Check pull request workflow status job` - and the source-only release on manual dispatch (`releaseTrigger: dispatch-only`; tag + source zip). Branch-model rulesets are specified in [GOVERNANCE.md "Branching Model"][governance-branching-model] and [repo-config/README.md][repo-config-readme], not here. +Their CI is lint/validation only (editorconfig/EOL plus domain linters such as Home Assistant or ESPHome config validation or a firmware build, but **no unit tests**), so the D-guarantees below that assume a build/test pipeline are **N/A** exactly as for `source-only` (Section 6). What binds: the promotion gate, where the `develop -> main` PR must pass the required `Check pull request workflow status job`, and the source-only release on manual dispatch (`releaseTrigger: dispatch-only`; tag + source zip). Branch-model rulesets are specified in [GOVERNANCE.md "Branching Model"][governance-branching-model] and [repo-config/README.md][repo-config-readme], not here. ### Two Layers: Orchestration vs Build - **Orchestration** is generic and forms the standardization baseline **at the job level**: the single-branch publisher, the `get-version`, `validate-release`, and `github-release` jobs, the date-badge job, and the `changes -> smoke-build -> aggregator` shape of the PR workflow. These job *bodies* should not need per-repo edits. - **Build** is repo-owned: the `build--task.yml` leaf tasks. -- **What the repo curates** (by design, not a leak): the *list* of targets. This is **not** a byte-for-byte file carry. Adding or dropping a target edits the orchestrator's surface - the `enable_` inputs and the `build-` job + its `github-release` `needs:` entry in the release task, **and** the `changes` paths-filter entry + output + the `smoke-build` enable-forward in the PR workflow. "Verbatim" applies to the `github-release` job and the version/publish-plan logic, not to the release task's job list or the paths-filter. Subsetting is symmetric: the same surface you trim to drop a target you extend to add a new one (e.g. a `release-asset--library` producer needs a new `enable_library` input, a `build-library` job, a `needs:` entry, and a `library` paths-filter). +- **What the repo curates** (by design, not a leak): the *list* of targets. This is **not** a byte-for-byte file carry. Adding or dropping a target edits the orchestrator's surface: the `enable_` inputs and the `build-` job + its `github-release` `needs:` entry in the release task, **and** the `changes` paths-filter entry + output + the `smoke-build` enable-forward in the PR workflow. "Verbatim" applies to the `github-release` job and the version/publish-plan logic, not to the release task's job list or the paths-filter. Subsetting is symmetric: the same surface you trim to drop a target you extend to add a new one (e.g. a `release-asset--library` producer needs a new `enable_library` input, a `build-library` job, a `needs:` entry, and a `library` paths-filter). ### The Seam Contract @@ -87,7 +87,7 @@ Every leaf and the release task take `ref`, `branch` (the **logical** branch tha ### Versioning -NBGV versions the branch being published. Each run builds a single branch (the trigger ref), so `GITHUB_REF` already names it and NBGV classifies it directly - no `IGNORE_GITHUB_REF` override is required. The default branch is the public-release ref, so it builds clean `X.Y.Z`. Every other branch builds a prerelease `X.Y.Z-g`. `version.json`'s `version` is the major.minor floor. NBGV appends the git height as the patch. **NBGV and `version.json` are retained even by a repo with no compiled code** - they are the source of the release tag (`SemVer2`) and `target_commitish` (`GitCommitId`) and the prerelease classification. The .NET SDK is pulled in only as the versioning toolchain. A package build derives its registry version from the same NBGV outputs, but **not always from `SemVer2`**: the PyPI version is built from `AssemblyFileVersion` (four-part `M.N.P.B`) with a PEP 440 `.dev0` appended on the `develop` branch. A wrapper repo may drive its build/image version from an external committed `name -> version` state file while NBGV still tags the release. +NBGV versions the branch being published. Each run builds a single branch (the trigger ref), so `GITHUB_REF` already names it and NBGV classifies it directly, and no `IGNORE_GITHUB_REF` override is required. The default branch is the public-release ref, so it builds clean `X.Y.Z`. Every other branch builds a prerelease `X.Y.Z-g`. `version.json`'s `version` is the major.minor floor. NBGV appends the git height as the patch. **NBGV and `version.json` are retained even by a repo with no compiled code**, since they are the source of the release tag (`SemVer2`) and `target_commitish` (`GitCommitId`) and the prerelease classification. The .NET SDK is pulled in only as the versioning toolchain. A package build derives its registry version from the same NBGV outputs, but **not always from `SemVer2`**: the PyPI version is built from `AssemblyFileVersion` (four-part `M.N.P.B`) with a PEP 440 `.dev0` appended on the `develop` branch. A wrapper repo may drive its build/image version from an external committed `name -> version` state file while NBGV still tags the release. ### Validate-at-Entry @@ -95,7 +95,7 @@ When a workflow's inputs carry a cross-input or input-versus-derived-state invar ### Resource Lifecycle -Workflow artifacts are an **intra-run handoff** only. Durable copies live on the release/registry. The rule: a transfer artifact handed **between jobs** is deleted by exact name/pattern **at its point of consumption**, the delete is **gated to the same condition as the consumer**, and it is **best-effort**. **Every** `upload-artifact` sets `retention-days: 1` as the universal failure-path backstop, so no terminal blanket-delete job is needed - and an intermediate consumed only within the same run (e.g. an executable's per-runtime outputs feeding an aggregation step) may rely on the retention backstop alone. The run is **never** blanket-deleted (`.artifacts[].id`). See D5. +Workflow artifacts are an **intra-run handoff** only. Durable copies live on the release/registry. The rule: a transfer artifact handed **between jobs** is deleted by exact name/pattern **at its point of consumption**, the delete is **gated to the same condition as the consumer**, and it is **best-effort**. **Every** `upload-artifact` sets `retention-days: 1` as the universal failure-path backstop, so no terminal blanket-delete job is needed, and an intermediate consumed only within the same run (e.g. an executable's per-runtime outputs feeding an aggregation step) may rely on the retention backstop alone. The run is **never** blanket-deleted (`.artifacts[].id`). See D5. ### Fast PR Feedback @@ -114,7 +114,7 @@ flowchart TD ### Release Model -Each publish builds a **single branch** - the trigger ref (`main` a release, `develop` a prerelease) - so there is no branch matrix and `github.ref` always names the built branch. A **human merge never auto-publishes**: a first `plan` job (`publish-plan-task.yml`) decides once and every job gates on it. A run publishes on a **code-affecting bot push to `main`** (the App merges every Dependabot/codegen PR, so `github.actor` gates it, and a shared paths filter also drops a non-substantive change like an Actions bump), a **manual dispatch** of `main`/`develop`, or a **main-only weekly schedule** (Docker, to refresh the base image). The `push` is main-only, so a develop bot merge publishes nothing (its prerelease comes via dispatch). A **source-only** repo publishes on **dispatch only**. Every release is a tag on the built commit plus a source archive, README, and LICENSE. Targets amend it with `release-asset-*` files or push to their own registry. An unchanged version re-pushes nothing (no-op republish). Docker re-pushes by design. +Each publish builds a **single branch**, the trigger ref (`main` a release, `develop` a prerelease), so there is no branch matrix and `github.ref` always names the built branch. A **human merge never auto-publishes**: a first `plan` job (`publish-plan-task.yml`) decides once and every job gates on it. A run publishes on a **code-affecting bot push to `main`** (the App merges every Dependabot/codegen PR, so `github.actor` gates it, and a shared paths filter also drops a non-substantive change like an Actions bump), a **manual dispatch** of `main`/`develop`, or a **main-only weekly schedule** (Docker, to refresh the base image). The `push` is main-only, so a develop bot merge publishes nothing (its prerelease comes via dispatch). A **source-only** repo publishes on **dispatch only**. Every release is a tag on the built commit plus a source archive, README, and LICENSE. Targets amend it with `release-asset-*` files or push to their own registry. An unchanged version re-pushes nothing (no-op republish). Docker re-pushes by design. ```mermaid flowchart TD @@ -130,22 +130,22 @@ flowchart TD Pick each output's path by **where the artifact goes**: - **File on the GitHub release** (zip, binary, packaged library): one leaf per output uploading `release-asset--`. The repo keeps `expect_release_assets: true` (its default). -- **Package-registry push** (NuGet, PyPI): the leaf builds and publishes to its registry. NuGet pushes from the leaf *and* uploads a `release-asset-*`. PyPI is **split** - the leaf only builds + uploads its build artifact, a separate publish job does the OIDC upload (so `id-token: write` is granted at one entry point, behind an environment gate) and contributes **no** `release-asset-*`. -- **Image-registry push** (Docker): the leaf pushes the default branch multi-arch (amd64+arm64) and any other branch `amd64`-only (arm64 emulation is reserved for the released image) - contributes no `release-asset-*`. -- **No file target via the release task** (Docker-only, PyPI-only): the release is tag + source zip + README + LICENSE. The repo's **caller MUST pass `expect_release_assets: false`** to the release task (the input is never set by a publisher that ships file targets, which keeps the default `true`). This is the one case where the otherwise-verbatim publisher is edited. With the default `true` and no assets, the release-create step fails on `fail_on_unmatched_files`. A **source-only** repo has no release task at all - its standalone `publish-release.yml` inlines `action-gh-release`, so `expect_release_assets` does not apply (see Section 6). +- **Package-registry push** (NuGet, PyPI): the leaf builds and publishes to its registry. NuGet pushes from the leaf *and* uploads a `release-asset-*`. PyPI is **split**: the leaf only builds + uploads its build artifact, a separate publish job does the OIDC upload (so `id-token: write` is granted at one entry point, behind an environment gate) and contributes **no** `release-asset-*`. +- **Image-registry push** (Docker): the leaf pushes the default branch multi-arch (amd64+arm64) and any other branch `amd64`-only (arm64 emulation is reserved for the released image), and contributes no `release-asset-*`. +- **No file target via the release task** (Docker-only, PyPI-only): the release is tag + source zip + README + LICENSE. The repo's **caller MUST pass `expect_release_assets: false`** to the release task (the input is never set by a publisher that ships file targets, which keeps the default `true`). This is the one case where the otherwise-verbatim publisher is edited. With the default `true` and no assets, the release-create step fails on `fail_on_unmatched_files`. A **source-only** repo has no release task at all. Its standalone `publish-release.yml` inlines `action-gh-release`, so `expect_release_assets` does not apply (see Section 6). -## 4. Behavioral Contract - Expected Outcomes +## 4. Behavioral Contract: Expected Outcomes The required behaviors, organized by domain. Each is a **MUST**, stated as input -> output plus the failure-mode it prevents. A workflow that violates any *applicable* guarantee is **not operational**. ### D1 - PR Fast-Feedback (Smoke) - **D1.1 Only changed targets build.** Input: a PR touching some targets. Output: the paths-filter marks exactly those targets and only their smoke builds run. Unchanged targets skip. A repo's own targets MUST each have a filter entry (so a touched target is never silently skipped). *Prevents: rebuilding everything, and a changed target slipping through unbuilt.* -- **D1.2 A validation job always runs.** Input: any PR. Output: a type-appropriate validation job runs unconditionally and the aggregator `needs:` it. In a .NET repo this is the `unit-test` job (format/style/test). A non-.NET repo **replaces** it (not deletes) with its own validator (lint, schema-check) and re-points **every** `needs:` on it - both the aggregator and `smoke-build` (which `needs:` the validation job by name) - to the replacement. *Prevents: a PR merging with no validation, or a dangling `needs:` that fails the whole workflow to load.* +- **D1.2 A validation job always runs.** Input: any PR. Output: a type-appropriate validation job runs unconditionally and the aggregator `needs:` it. In a .NET repo this is the `unit-test` job (format/style/test). A non-.NET repo **replaces** it (not deletes) with its own validator (lint, schema-check) and re-points **every** `needs:` on it (both the aggregator and `smoke-build`, which `needs:` the validation job by name) to the replacement. *Prevents: a PR merging with no validation, or a dangling `needs:` that fails the whole workflow to load.* - **D1.3 Smoke never publishes and never uploads.** Input: `smoke: true`. Output: full compile/lint/test, but no registry/image push, no release, and **no** artifact uploads (every `upload-artifact`, including any aggregation job, is gated `!smoke`). *Prevents: a PR publishing; orphaned artifacts churning the storage quota.* - **D1.4 Workflow-file changes are not smoke-built.** Input: a PR changing only `.github/workflows/**`. Output: the paths-filter excludes workflow files, so smoke-build skips. *Implication: a workflow-only change is not smoke-built, but actionlint still validates it in CI.* - **D1.5 One required aggregator gates merge.** Input: any PR. Output: a single aggregator job must **succeed**, `needs:` the changes job and the validation job, treat a **skipped** smoke build as pass, and **block** on `failure`/`cancelled`. Its name is ruleset-bound: the job `name:` and the ruleset `context:` are the same string and MUST be renamed together, never independently. *Prevents: a paths-filter error letting a target-changing PR merge unbuilt.* -- **D1.6 Coverage is reported to Codecov (C# and Python).** Input: a C# or Python repo's validation/test job. Output: tests run with coverage collection (`dotnet test --collect:"XPlat Code Coverage"` or `pytest --cov-report=xml`) and a `codecov/codecov-action` step uploads it, **best-effort** (`continue-on-error` and/or `fail_ci_if_error: false`, so a Codecov outage or an absent token never reds the gate). `CODECOV_TOKEN` lives in the repo's **actions** secret store and reaches the reusable validator via `secrets: inherit`. Required for **every** C# and Python repo that has tests (see `spec/secrets.json` `typeMechanisms`). The repo also ships a **`codecov.yml`** that sets the project and patch statuses to **`informational: true`** so a coverage delta never gates a PR - a distinct knob from `fail_ci_if_error` (which only guards the upload step) - and excludes intentionally-untested, non-shipped code (an example/demo or benchmark project) from the coverage denominator via `ignore`. A repo may override this to enforce a coverage threshold where its quality bar requires it. Coverage output is a build artifact - `.gitignore` excludes it (e.g. `coverage/`, `*.cobertura.xml`; `.gitignore` is the full source of truth) so a blanket `git add -A` won't stage the untracked output. *Prevents: coverage silently going unreported; a stale, unused token; a coverage regression blocking an unrelated PR; a coverage artifact committed by a blanket add.* +- **D1.6 Coverage is reported to Codecov (C# and Python).** Input: a C# or Python repo's validation/test job. Output: tests run with coverage collection (`dotnet test --collect:"XPlat Code Coverage"` or `pytest --cov-report=xml`) and a `codecov/codecov-action` step uploads it, **best-effort** (`continue-on-error` and/or `fail_ci_if_error: false`, so a Codecov outage or an absent token never reds the gate). `CODECOV_TOKEN` lives in the repo's **actions** secret store and reaches the reusable validator via `secrets: inherit`. Required for **every** C# and Python repo that has tests (see `spec/secrets.json` `typeMechanisms`). The repo also ships a **`codecov.yml`** that sets the project and patch statuses to **`informational: true`** so a coverage delta never gates a PR (a distinct knob from `fail_ci_if_error`, which only guards the upload step) and excludes intentionally-untested, non-shipped code (an example/demo or benchmark project) from the coverage denominator via `ignore`. A repo may override this to enforce a coverage threshold where its quality bar requires it. Coverage output is a build artifact, so `.gitignore` excludes it (e.g. `coverage/`, `*.cobertura.xml`; `.gitignore` is the full source of truth) so a blanket `git add -A` won't stage the untracked output. *Prevents: coverage silently going unreported; a stale, unused token; a coverage regression blocking an unrelated PR; a coverage artifact committed by a blanket add.* ### D2 - Input/State Validation at Entry @@ -160,19 +160,19 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as input - **D3.2 Default = public, others = prerelease.** Output: default branch -> `X.Y.Z`; any other -> `X.Y.Z-g`. The default-branch literal in the gate, the `prerelease` expression, and `version.json` MUST all name the repo's real default branch. - **D3.3 Version floor + git height.** Output: `version.json` sets the major.minor floor. NBGV appends the git height as the patch, bumped only for a functional change by the maintainer. NBGV and `version.json` are retained even by a no-compiler repo (they own the tag). - **D3.4 Registry versions follow the classification, per registry.** Output: NuGet default = stable, others = prerelease (derived by NuGet.org from the SemVer2 `-g` suffix on `PackageVersion`, not a flag the workflow sets). PyPI builds from `AssemblyFileVersion` (`M.N.P.B`) and appends `.dev0` on the `develop` branch only (a two-branch literal, not a generic N-branch rule). The develop `.dev0` build must remain `pip install --pre`-selectable and sort above the default release (NBGV git height in the release segment keeps develop ahead). *Prevents: a non-default leg published as a release; a renamed/extra branch silently getting a plain version.* -- **D3.5 Wrapper repos may use an external version.** Output: a repo wrapping an upstream release drives its build/image version from a committed `name -> version` state file, while NBGV still tags the release. *Note: the tracker (the writer) ships without consumer wiring - a wrapper must wire the leaf to read the state file (e.g. `jq` into the image tag) instead of `SemVer2`. If the leaf still tags off NBGV, the wrapper is not actually pinned to upstream.* +- **D3.5 Wrapper repos may use an external version.** Output: a repo wrapping an upstream release drives its build/image version from a committed `name -> version` state file, while NBGV still tags the release. *Note: the tracker (the writer) ships without consumer wiring, so a wrapper must wire the leaf to read the state file (e.g. `jq` into the image tag) instead of `SemVer2`. If the leaf still tags off NBGV, the wrapper is not actually pinned to upstream.* ### D4 - Release / Publish - **D4.1 Gated single-branch publish.** Output: PRs smoke-test and publish nothing. A **human merge never auto-publishes**. A first `plan` job (`publish-plan-task.yml`) decides once and every job gates on it: publish on a **code-affecting bot push to `main`** (gated to the codegen App / Dependabot `github.actor`; an Actions-only bump matches no release path and publishes nothing), a **dispatch** of `main`/`develop`, or a **main-only weekly schedule** (Docker). A source-only repo publishes on dispatch only. Each run builds one branch. - **D4.2 Tag the built commit.** Output: the release `target_commitish` is the built commit's SHA (NBGV's commit id), never `github.sha` or a moving branch ref. *Prevents: the tag landing on the default branch instead of the built tree.* - **D4.3 Release contents.** Output: every release is a tag on the built commit plus the auto source zip, README, and LICENSE; file-producing targets attach `release-asset-*`; `prerelease` equals `branch != default`. A no-file-target repo that uses the release task (Docker-only, PyPI-only) reaches the tag-only shape **only** with `expect_release_assets: false` set by the caller (which relaxes `fail_on_unmatched_files` and skips the asset download). With the default `true` and no assets the release-create step fails. A source-only repo reaches the same shape through its inlined `action-gh-release` instead, with no release task or `expect_release_assets`. -- **D4.4 No-op republish.** Input: a re-run whose version is unchanged. Output: nothing is re-pushed - the release-create step is skipped when the tag exists (refreshed only on `workflow_dispatch`), and the paired asset-delete is skipped with it. Registry pushes are no-ops. The NuGet/PyPI publish steps are **not** statically gated on existence - they run and the **server** dedupes (`dotnet nuget push --skip-duplicate` turns a 409 into success; PyPI `skip-existing: true`). **Docker always re-pushes** the image (base-image refresh), independently of the release-create skip, within the same run. *Prevents: duplicate releases and wasted pushes.* +- **D4.4 No-op republish.** Input: a re-run whose version is unchanged. Output: nothing is re-pushed, because the release-create step is skipped when the tag exists (refreshed only on `workflow_dispatch`), and the paired asset-delete is skipped with it. Registry pushes are no-ops. The NuGet/PyPI publish steps are **not** statically gated on existence. They run and the **server** dedupes (`dotnet nuget push --skip-duplicate` turns a 409 into success; PyPI `skip-existing: true`). **Docker always re-pushes** the image (base-image refresh), independently of the release-create skip, within the same run. *Prevents: duplicate releases and wasted pushes.* ### D5 - Resource Cleanup - **D5.1 Delete at the point of consumption.** Output: the job that downloads a **cross-job** transfer artifact deletes it (by exact name/pattern) right after consuming it. An intermediate consumed only within the same run (e.g. an executable's per-runtime outputs feeding an in-run aggregation) MAY instead rely on the `retention-days: 1` backstop. *Prevents: transfer artifacts accumulating against the storage quota.* -- **D5.2 Gate the delete to the consumer's condition.** Output: the delete runs under the **same** condition as its consuming step. Where the consumer is conditional (the GitHub release create), the delete is conditional too. Where the consumer always runs when its job runs (the PyPI publish step), the delete always runs - so on a no-op re-run the `release-asset-*` delete is **skipped** while the PyPI build-artifact delete still **runs** (its publish ran). *Prevents: deleting freshly built assets on a no-op re-run.* +- **D5.2 Gate the delete to the consumer's condition.** Output: the delete runs under the **same** condition as its consuming step. Where the consumer is conditional (the GitHub release create), the delete is conditional too. Where the consumer always runs when its job runs (the PyPI publish step), the delete always runs, so on a no-op re-run the `release-asset-*` delete is **skipped** while the PyPI build-artifact delete still **runs** (its publish ran). *Prevents: deleting freshly built assets on a no-op re-run.* - **D5.3 Best-effort.** Output: cleanup is `continue-on-error`, tolerates a failed listing, and deletes **all** matching ids. *Prevents: a cleanup hiccup reddening a job whose publish succeeded.* - **D5.4 Retention backstop.** Output: **every** `upload-artifact` sets `retention-days: 1`. - **D5.5 Never blanket-delete.** Output: cleanup MUST NOT enumerate and delete the run's whole artifact set. *Prevents: destroying diagnostic/log artifacts and auto-emitted build-records.* @@ -195,8 +195,8 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as input - **D8.1 Merge-bot.** Output: enables auto-merge on `opened`/`reopened` for **every** Dependabot tier including semver-major (the required checks are the gate, not the bump magnitude); dispatches `--squash`/`--merge` by the PR's base ref; disables on a maintainer-pushed `synchronize`; concurrency keyed on the **PR number**, not `github.ref`. *Prevents: two PRs colliding in auto-merge.* - **D8.2 CodeGen and Dependabot.** Output: codegen runs as a matrix over both branches and is deterministic from an external source. Dependabot targets both branches, security PRs to default. -- **D8.3 Upstream-version tracker.** Output: a scheduled resolver prints a JSON `name -> version` object to a committed state file, opens a rolling per-branch bump PR naming only the moved keys, the merge-bot auto-merges it. The `main` pin push publishes via the release gate, while a `develop` pin does not auto-publish - it ships via a `develop` dispatch (prerelease) or the next promotion to `main`. The tracker's `bump-branch-prefix` + `branches` MUST match the merge-bot's hard-coded `-` head/base pairs, or auto-merge silently never fires. -- **D8.4 An identity allowlist used as a gate fails loud.** Where a gate compares `github.actor` (or a PR author) against hard-coded bot identities, the non-matching branch on an otherwise-legitimate trigger **emits a `::warning::`** rather than falling through silently. Output: a run that declines to act on an unrecognized identity is visibly annotated. *Prevents: the App being renamed, replaced, or reinstalled under a new slug, after which the comparison quietly evaluates false and the gate stops firing - a green, silent run that looks identical to a healthy one.* The masking matters most where a second path hides the loss: a weekly schedule keeps publishing, so the only symptom is release *timeliness*, easily missed for months. Where the failure is self-announcing instead (the merge-bot simply stops merging, so bot PRs visibly pile up) an annotation is optional. Resolving the identity at run time (mint an App token, read `GET /app`) removes the hard-coded string entirely and is the escalation if an allowlist proves fragile in practice. +- **D8.3 Upstream-version tracker.** Output: a scheduled resolver prints a JSON `name -> version` object to a committed state file, opens a rolling per-branch bump PR naming only the moved keys, the merge-bot auto-merges it. The `main` pin push publishes via the release gate, while a `develop` pin does not auto-publish. It ships via a `develop` dispatch (prerelease) or the next promotion to `main`. The tracker's `bump-branch-prefix` + `branches` MUST match the merge-bot's hard-coded `-` head/base pairs, or auto-merge silently never fires. +- **D8.4 An identity allowlist used as a gate fails loud.** Where a gate compares `github.actor` (or a PR author) against hard-coded bot identities, the non-matching branch on an otherwise-legitimate trigger **emits a `::warning::`** rather than falling through silently. Output: a run that declines to act on an unrecognized identity is visibly annotated. *Prevents: the App being renamed, replaced, or reinstalled under a new slug, after which the comparison quietly evaluates false and the gate stops firing, a green and silent run that looks identical to a healthy one.* The masking matters most where a second path hides the loss: a weekly schedule keeps publishing, so the only symptom is release *timeliness*, easily missed for months. Where the failure is self-announcing instead (the merge-bot simply stops merging, so bot PRs visibly pile up) an annotation is optional. Resolving the identity at run time (mint an App token, read `GET /app`) removes the hard-coded string entirely and is the escalation if an allowlist proves fragile in practice. ### D9 - Style / Static (See Section 2) @@ -219,7 +219,7 @@ Read the workflow files plus `version.json` and assert the structural fact behin - **D1:** a `changes` paths-filter job exists, covers each of the repo's targets, and **excludes** `.github/workflows/**`; the PR entry workflow's smoke call sets `github/nuget/dockerhub: false` on the release task; the leaf receives `smoke: true` and a derived `push` (false on smoke); every build-task `upload-artifact` (and any aggregation job) is gated `!smoke`; the aggregator `needs:` the `changes` and validation jobs, blocks on `failure`/`cancelled`, passes on `skipped`; a validation job runs unconditionally. - **D2:** an entry validation job/step exists per complex-input workflow; the release gate checks both directions, strips `+buildmetadata`, and skips on smoke; the publisher rejects a dispatch from a ref other than `main` or `develop`. - **D3:** each run builds one branch, so NBGV classifies `github.ref` directly (no `IGNORE_GITHUB_REF`); the default-branch literal in the gate (`== 'main'`), the `prerelease` expression (`!= 'main'`), and `version.json`'s `publicReleaseRefSpec` all name the repo's actual default branch. -- **D4:** `target_commitish` is the NBGV commit id; `prerelease` equals `branch != default`; the release-create step is gated `exists == 'false' || github.event_name == 'workflow_dispatch'` (the step output is the string `'false'`, not a boolean); the asset-delete step is gated identically. A dispatch-only publisher (`releaseTrigger: dispatch-only`) may omit the gate and the exists-check entirely - every run is a dispatch, so the skip leg can never fire and create-or-refresh is unconditional. Record the gate N/A there, not missing. +- **D4:** `target_commitish` is the NBGV commit id; `prerelease` equals `branch != default`; the release-create step is gated `exists == 'false' || github.event_name == 'workflow_dispatch'` (the step output is the string `'false'`, not a boolean); the asset-delete step is gated identically. A dispatch-only publisher (`releaseTrigger: dispatch-only`) may omit the gate and the exists-check entirely: every run is a dispatch, so the skip leg can never fire and create-or-refresh is unconditional. Record the gate N/A there, not missing. - **D5:** each cross-job transfer artifact has a delete step at its consumer, gated to the consumer's condition, `continue-on-error: true`, looping all ids; **every** upload sets `retention-days: 1`; **no** `.artifacts[].id` blanket delete exists anywhere. - **D6:** the release download uses `pattern:`/`merge-multiple:` (no `artifact-ids:`). Branch-derived config reads `inputs.branch` (a `github.ref_name` in such config is a finding). Artifact names are branch-suffixed. The target set is consistent across the release task and the paths-filter. - **D7:** the publisher concurrency group is ref-independent with `cancel-in-progress: false`. Reusable jobs declare permissions. Boolean `if:` uses both forms. @@ -230,7 +230,7 @@ Read the workflow files plus `version.json` and assert the structural fact behin - **Console/executable:** the smoke runtime matrix is a strict non-empty subset of the full matrix. The per-runtime outputs (`publish--`) are aggregated by `pattern:` + `merge-multiple:` into one `release-asset--` and the aggregation job is gated `!smoke`. The per-runtime intermediates rely on the retention backstop (no explicit delete is required for an in-run intermediate). - **NuGet:** the publish step is gated `if: inputs.push` only (not on an existence check) and uses `--skip-duplicate`. `*.nupkg` push also carries the paired `.snupkg` to the symbol server where symbols are enabled. The `release-asset` zip carries the package(s). - **PyPI:** `publish-pypi` declares `environment: { name: pypi }`. `id-token: write` appears only on that job (absent from the build/PR path). `skip-existing: true` is set on the publish action. The build artifact is deleted after publish. The `pypi` environment has a deployment-branch rule. -- **Docker:** a Docker-only repo's caller passes `expect_release_assets: false`. The leaf reads the external state file for the tag instead of `SemVer2` (wrapper repos only - a plain Docker repo correctly tags off `SemVer2` and records this N/A). The readme/date-badge jobs are gated main-only. The docker-readme task validates `repositories` XOR `manifest`+`manifest-jq`. The buildcache follows D9.4. +- **Docker:** a Docker-only repo's caller passes `expect_release_assets: false`. The leaf reads the external state file for the tag instead of `SemVer2` (wrapper repos only, since a plain Docker repo correctly tags off `SemVer2` and records this N/A). The readme/date-badge jobs are gated main-only. The docker-readme task validates `repositories` XOR `manifest`+`manifest-jq`. The buildcache follows D9.4. ### 5B. End-to-End Trace Scenarios (No Execution, Deterministic from the YAML) @@ -239,23 +239,23 @@ For each *applicable* scenario, evaluate every job's `if:`/`needs:` against the | # | Input | Expected output | Exercises | | --- | --- | --- | --- | | S1 | PR touching a build target | `changes` flags it; validation runs; that target's smoke build runs; no push, **no uploads**; validate-release **skipped (smoke), succeeds**; release **skipped**; aggregator **success**; version = prerelease; no release; no dangling artifacts | D1, D2.2, D3 | -| S2 | PR changing only docs | smoke-build **skipped**; validation runs; aggregator **success** | D1.1, D1.5 | -| S3 | PR changing only `.github/workflows/**` | filter excludes -> smoke-build **skipped**; aggregator **success** | D1.4 | -| S4 | PR base = default branch, carrying a build target | smoke versions as prerelease; validate-release **skipped (smoke)** so the default-branch arm does **not** fire; aggregator **success**; promotion not blocked | D1.3, D2.2 | -| S5 | bot push to `main` not touching a release path (e.g. an Actions bump) | the paths filter excludes it; nothing publishes | D4.1 | -| S6 | code-affecting **bot** push to `main` (a human push/promotion, or any develop push, does not) | the `plan` job gates it to the App/Dependabot actor; `main` publishes a release | D3, D4 | +| S2 | PR changing only docs | smoke-build **skipped**, validation runs, aggregator **success** | D1.1, D1.5 | +| S3 | PR changing only `.github/workflows/**` | filter excludes -> smoke-build **skipped**, aggregator **success** | D1.4 | +| S4 | PR base = default branch, carrying a build target | smoke versions as prerelease, validate-release **skipped (smoke)** so the default-branch arm does **not** fire, aggregator **success**, promotion not blocked | D1.3, D2.2 | +| S5 | bot push to `main` not touching a release path (e.g. an Actions bump) | the paths filter excludes it, so nothing publishes | D4.1 | +| S6 | code-affecting **bot** push to `main` (a human push/promotion, or any develop push, does not) | the `plan` job gates it to the App/Dependabot actor, and `main` publishes a release | D3, D4 | | S7 | publish run (schedule, a bot push to main, or a dispatch) | builds the **one** trigger branch: `main` -> `X.Y.Z`, `prerelease=false`, registry stable, badge/readme run; `develop` -> `X.Y.Z-g`, `prerelease=true`, registry prerelease; `release-asset-*` consumed-then-deleted; PyPI build-artifact deleted after its publish; **no dangling artifacts** | D3, D4, D5, D6, D7 | | S8 | dispatch from a ref other than `main` or `develop` | **fails fast** | D2.3 | | S9 | re-run publish, version unchanged | release-create **skipped**, `release-asset-*` delete **skipped**; NuGet/PyPI pushes no-op (server dedupe); **PyPI build-artifact still deleted** (its publish ran); **Docker still re-pushes** the image; no duplicate release | D4.4, D5.2 | -| S10 | branch/version classification disagree | validate-release **fails loud**; build/publish skip | D2.2 | -| S11 | scheduled upstream-version bump (wrapper) | resolver detects a change -> commits the state file -> opens a `-` PR -> merge-bot auto-merges -> the `main` pin publishes via the gate (a develop pin does not auto-publish; it ships via a develop dispatch or promotion) | D8.3, D3.5 | +| S10 | branch/version classification disagree | validate-release **fails loud**, build/publish skip | D2.2 | +| S11 | scheduled upstream-version bump (wrapper) | resolver detects a change -> commits the state file -> opens a `-` PR -> merge-bot auto-merges -> the `main` pin publishes via the gate (a develop pin does not auto-publish, shipping instead via a develop dispatch or promotion) | D8.3, D3.5 | ### 5C. Live Probe (Where Warranted) - Open a trivial-change PR touching one target and confirm S1. -- Drive a `smoke: true` push-probe of the build task for **both** the default and a non-default branch and assert the version classification (clean vs prerelease) and that the gate passes - **without publishing**. *Caveat: the Docker leg logs in to the registry even on smoke and reads the buildcache, so it needs `DOCKER_HUB_*` secrets and cannot run on a fork PR (same-repo only).* +- Drive a `smoke: true` push-probe of the build task for **both** the default and a non-default branch and assert the version classification (clean vs prerelease) and that the gate passes, **without publishing**. *Caveat: the Docker leg logs in to the registry even on smoke and reads the buildcache, so it needs `DOCKER_HUB_*` secrets and cannot run on a fork PR (same-repo only).* - Per registry: after a real publish, query NuGet.org for the expected version + prerelease classification (and the `.snupkg` on the symbol server), and confirm a re-run added no duplicate. For PyPI inspect the `Compute PyPI version step` log and the built `dist/*` filenames for `.dev0` off `develop` vs a plain version on the default branch. -- Inspect the latest real publish's logs for `PublicRelease`/`SemVer2` per leg and confirm the artifact lifecycle (uploaded, consumed, deleted; none left behind). +- Inspect the latest real publish's logs for `PublicRelease`/`SemVer2` per leg and confirm the artifact lifecycle (uploaded, consumed, deleted, with none left behind). ### Assessment @@ -270,13 +270,13 @@ The workflow is **operational** iff every *applicable* 5A item passes and every Each type maps the *applicable* S-scenarios onto its targets. The differences are which leaf tasks exist and what each produces, which 5A addenda apply, and which scenarios are N/A. Walking these is the self-check that the contract holds for each shape. -- **Console / executable application.** Target produces `release-asset--executable` (a 7z archive, `Console.7z`) by building a per-runtime `dotnet publish` matrix, then an aggregation job downloads the per-runtime `publish--` intermediates (`pattern:` + `merge-multiple:`), zips them, and uploads the single asset. Smoke builds a strict subset of runtimes. The per-runtime upload **and** the aggregation job are both gated `!smoke`, so smoke uploads nothing. The per-runtime intermediates rely on `retention-days: 1` (no explicit delete). Test: S1 with a console change smoke-builds the subset and uploads nothing; S7 attaches the 7z, `prerelease=true` on the non-default leg and `prerelease=false` on the default leg (GitHub auto-marks the stable default release "Latest" - the workflow does not set it). -- **NuGet library.** The leaf both pushes (`dotnet nuget push *.nupkg --skip-duplicate`, gated `if: push` only) and uploads `release-asset--nugetlibrary`. Configuration is Release on the default branch, Debug otherwise. Where symbols are enabled (`snupkg`), the push auto-carries the paired `.snupkg` to NuGet.org's symbol server and the asset zip also contains it - a triple surface. NuGet.org derives `isPrerelease` from the SemVer2 `-g` suffix (the workflow sets no such flag). Test: S7 non-default leg publishes a prerelease package + asset, default a stable; S9 re-run is a server-side `--skip-duplicate` no-op. 5C: query NuGet.org for both versions and the symbol package. -- **PyPI library.** The leaf builds + uploads `pypilibrary-build-`. A **separate** `publish-pypi` job (with `environment: pypi`, `id-token: write`, `actions: write`) does the OIDC Trusted-Publishing upload with `skip-existing: true`, then **consume-then-deletes** the build artifact - **unconditionally on consume**, so on S9 it is deleted even though the `release-asset-*` delete is skipped. The version is `AssemblyFileVersion` with `.dev0` appended on `develop` only, and must stay `--pre`-selectable and sorted above the default release. PyPI contributes no `release-asset-*`. A PyPI-only repo sets `expect_release_assets: false` at the caller. Test: S7 default leg publishes a release, non-default a `.dev0`; S9 is a `skip-existing` no-op; 5C inspects the `dist/*` filenames and the compute-version log. +- **Console / executable application.** Target produces `release-asset--executable` (a 7z archive, `Console.7z`) by building a per-runtime `dotnet publish` matrix, then an aggregation job downloads the per-runtime `publish--` intermediates (`pattern:` + `merge-multiple:`), zips them, and uploads the single asset. Smoke builds a strict subset of runtimes. The per-runtime upload **and** the aggregation job are both gated `!smoke`, so smoke uploads nothing. The per-runtime intermediates rely on `retention-days: 1` (no explicit delete). Test: S1 with a console change smoke-builds the subset and uploads nothing; S7 attaches the 7z, `prerelease=true` on the non-default leg and `prerelease=false` on the default leg (GitHub auto-marks the stable default release "Latest", and the workflow does not set it). +- **NuGet library.** The leaf both pushes (`dotnet nuget push *.nupkg --skip-duplicate`, gated `if: push` only) and uploads `release-asset--nugetlibrary`. Configuration is Release on the default branch, Debug otherwise. Where symbols are enabled (`snupkg`), the push auto-carries the paired `.snupkg` to NuGet.org's symbol server and the asset zip also contains it, a triple surface. NuGet.org derives `isPrerelease` from the SemVer2 `-g` suffix (the workflow sets no such flag). Test: S7 non-default leg publishes a prerelease package + asset, default a stable; S9 re-run is a server-side `--skip-duplicate` no-op. 5C: query NuGet.org for both versions and the symbol package. +- **PyPI library.** The leaf builds + uploads `pypilibrary-build-`. A **separate** `publish-pypi` job (with `environment: pypi`, `id-token: write`, `actions: write`) does the OIDC Trusted-Publishing upload with `skip-existing: true`, then **consume-then-deletes** the build artifact, **unconditionally on consume**, so on S9 it is deleted even though the `release-asset-*` delete is skipped. The version is `AssemblyFileVersion` with `.dev0` appended on `develop` only, and must stay `--pre`-selectable and sorted above the default release. PyPI contributes no `release-asset-*`. A PyPI-only repo sets `expect_release_assets: false` at the caller. Test: S7 default leg publishes a release, non-default a `.dev0`; S9 is a `skip-existing` no-op; 5C inspects the `dist/*` filenames and the compute-version log. - **Docker image.** The leaf pushes the default branch multi-arch (amd64+arm64) and any other branch `amd64`-only, with a per-branch registry buildcache (`buildcache-`; a multi-image repo adds a per-image tag) (`cache-to` only the built branch and only on push, `cache-from` both branches); no `release-asset-*`, so a Docker-only repo's caller passes `expect_release_assets: false`; the readme (`peter-evans/dockerhub-description`, `DOCKER_HUB_ACCESS_TOKEN`) and date-badge jobs run **only** when the default branch publishes; the docker-readme task validates `repositories` XOR `manifest`+`manifest-jq` and a multi-image repo derives its publish matrix from the manifest. Docker **always re-pushes** the image, independently of a skipped release-create (S9). A **wrapper** repo tracks an upstream release: the upstream tracker writes a `name -> version` state file and the merge-bot auto-merges the bump PR (S11), and the leaf MUST read that file for the immutable tag instead of `SemVer2` (the tracker ships without this consumer wiring). Test: S7 default leg pushes `latest` + the version tag and updates readme/badge. Non-default pushes the develop tag (amd64 only). S9 still re-pushes. S11 ships the bumped upstream version next publish. 5C Docker probe needs `DOCKER_HUB_*` secrets and same-repo (not fork) runs. -- **Data / asset library.** A single new leaf: validate -> zip -> upload `release-asset--library` (`retention-days: 1`, upload gated `!smoke` - mirror the nugetlibrary leaf's shape). Because no such leaf ships, you **add a target** (D6.4): a new `enable_library` input + `build-library` job + `github-release` `needs:` entry in the release task, and a `library` paths-filter entry + `changes` output + `smoke-build` enable-forward in the PR workflow (without it, D1.1 never smoke-builds the library). Keep `expect_release_assets: true` (it has a file target, unlike Docker). The .NET `unit-test` job is replaced by a type-appropriate validator with the aggregator **and** `smoke-build` both re-pointed to it (D1.2/D1.5). `version.json` + the NBGV `get-version` step are retained (they own the tag). Test: S1 smoke runs validate+zip and uploads nothing; S7 attaches the zip, prerelease on the non-default leg; S9 on a *scheduled* re-run release-create + asset-delete skip (the existing zip is untouched, no registry push), while a `workflow_dispatch` re-run **refreshes** the release and re-runs the asset-delete (the asset is re-uploaded then re-deleted). N/A: the nuget/pypi/docker/executable 5A addenda and their scenario clauses. -- **Source-only / no build.** There is no `build-release-task.yml` (its `appliesTo` excludes source-only) and no package/image leaf, so nothing is edited down. The release is a standalone dispatch-only `publish-release.yml` that inlines NBGV for the tag and `action-gh-release` for the release - tag + source zip + README + LICENSE, no reusable release task and no asset download. With no target the paths-filter matches nothing, so `smoke-build` is **structurally always skipped** - validation is carried solely by the (replaced, non-.NET) validation job that the aggregator and `smoke-build`'s own `needs:` must both point at (D1.2; or drop the never-running `smoke-build` job). NBGV and `version.json` are still retained (they own the tag). Its publish job gates on the repo's reusable validation task (`needs:` the same `workflow_call` job the PR workflow runs), so a dispatch cannot release a ref that fails validation. Applicable scenarios: S1 (validation only), S5/S6 (publish gating), S7 (tag-only release), S8 (dispatch guard), S9 (no-op republish), S10 (classification gate). N/A: S2-S4 (assume a smoke-built target), the artifact-lifecycle and registry clauses of S7/S9, the D5/D6 artifact items, and all per-type 5A addenda - recorded N/A, not failed. -- **Operational (workflow model, not a build target).** A `workflowModel: operational` repo layers a direct-commit `develop` onto the **source-only** release shape (above). Two workflows: (1) a **lint/validation** PR workflow feeding the required `Check pull request workflow status job` - the generic linters (editorconfig/EOL, markdownlint, cspell, actionlint) plus a domain validator (Home Assistant `hass --script check_config`, `esphome config`, a firmware build), **no unit tests**; its triggers differ from the `release` model - `push` to `develop` (advisory feedback on direct commits) plus `pull_request` to `main` (the enforced promotion gate) plus `workflow_dispatch`. (2) the standard **source-only publisher** on `workflow_dispatch` only (`releaseTrigger: dispatch-only`): NBGV + `version.json` own the tag, and a manual dispatch cuts a GitHub release (tag + source zip + README + LICENSE, via the standalone publisher's inlined `action-gh-release`). Applicable scenarios: S1 (validation) on the promotion PR, plus the source-only release set - S7 (tag-only release), S8 (dispatch guard), S9 (no-op republish), S10 (classification). N/A: the auto-publish paths (S5/S6 bot-push and schedule - operational repos have neither) and every build/registry scenario. See the branch-model note in Section 3 and [GOVERNANCE.md "Branching Model"][governance-branching-model]. +- **Data / asset library.** A single new leaf: validate -> zip -> upload `release-asset--library` (`retention-days: 1`, upload gated `!smoke`, mirroring the nugetlibrary leaf's shape). Because no such leaf ships, you **add a target** (D6.4): a new `enable_library` input + `build-library` job + `github-release` `needs:` entry in the release task, and a `library` paths-filter entry + `changes` output + `smoke-build` enable-forward in the PR workflow (without it, D1.1 never smoke-builds the library). Keep `expect_release_assets: true` (it has a file target, unlike Docker). The .NET `unit-test` job is replaced by a type-appropriate validator with the aggregator **and** `smoke-build` both re-pointed to it (D1.2/D1.5). `version.json` + the NBGV `get-version` step are retained (they own the tag). Test: S1 smoke runs validate+zip and uploads nothing; S7 attaches the zip, prerelease on the non-default leg; S9 on a *scheduled* re-run release-create + asset-delete skip (the existing zip is untouched, no registry push), while a `workflow_dispatch` re-run **refreshes** the release and re-runs the asset-delete (the asset is re-uploaded then re-deleted). N/A: the nuget/pypi/docker/executable 5A addenda and their scenario clauses. +- **Source-only / no build.** There is no `build-release-task.yml` (its `appliesTo` excludes source-only) and no package/image leaf, so nothing is edited down. The release is a standalone dispatch-only `publish-release.yml` that inlines NBGV for the tag and `action-gh-release` for the release: tag + source zip + README + LICENSE, with no reusable release task and no asset download. With no target the paths-filter matches nothing, so `smoke-build` is **structurally always skipped**, and validation is carried solely by the (replaced, non-.NET) validation job that the aggregator and `smoke-build`'s own `needs:` must both point at (D1.2; or drop the never-running `smoke-build` job). NBGV and `version.json` are still retained (they own the tag). Its publish job gates on the repo's reusable validation task (`needs:` the same `workflow_call` job the PR workflow runs), so a dispatch cannot release a ref that fails validation. Applicable scenarios: S1 (validation only), S5/S6 (publish gating), S7 (tag-only release), S8 (dispatch guard), S9 (no-op republish), S10 (classification gate). N/A: S2-S4 (assume a smoke-built target), the artifact-lifecycle and registry clauses of S7/S9, the D5/D6 artifact items, and all per-type 5A addenda, all recorded N/A, not failed. +- **Operational (workflow model, not a build target).** A `workflowModel: operational` repo layers a direct-commit `develop` onto the **source-only** release shape (above). Two workflows: (1) a **lint/validation** PR workflow feeding the required `Check pull request workflow status job`, built from the generic linters (editorconfig/EOL, markdownlint, cspell, actionlint) plus a domain validator (Home Assistant `hass --script check_config`, `esphome config`, a firmware build), with **no unit tests**; its triggers differ from the `release` model: `push` to `develop` (advisory feedback on direct commits) plus `pull_request` to `main` (the enforced promotion gate) plus `workflow_dispatch`. (2) the standard **source-only publisher** on `workflow_dispatch` only (`releaseTrigger: dispatch-only`): NBGV + `version.json` own the tag, and a manual dispatch cuts a GitHub release (tag + source zip + README + LICENSE, via the standalone publisher's inlined `action-gh-release`). Applicable scenarios: S1 (validation) on the promotion PR, plus the source-only release set: S7 (tag-only release), S8 (dispatch guard), S9 (no-op republish), S10 (classification). N/A: the auto-publish paths (S5/S6 bot-push and schedule, neither of which an operational repo has) and every build/registry scenario. See the branch-model note in Section 3 and [GOVERNANCE.md "Branching Model"][governance-branching-model]. diff --git a/cspell.json b/cspell.json index adc66a5..a3a694a 100644 --- a/cspell.json +++ b/cspell.json @@ -17,6 +17,7 @@ "astral", "autoremove", "betz", + "BMFF", "buildcache", "buildmetadata", "buildtransitive", @@ -54,6 +55,7 @@ "eeuo", "Emby", "envsubst", + "esac", "exif", "exiftool", "extensionless", @@ -67,6 +69,7 @@ "gyan", "HACS", "hatchling", + "heif", "heredocs", "homeassistant", "icloudpd", @@ -74,14 +77,17 @@ "idempotently", "immich", "immichcli", - "ISOBMFF", + "IPTC", "isort", "Jellyfin", "Keychain", "kicad", "lastbuild", "lavfi", + "libheif", + "libraw", "libsecret", + "libvips", "libx", "lightroom", "LINQ", @@ -136,6 +142,7 @@ "remuxes", "Remuxing", "resharper", + "rhoopr", "rhysd", "Rubba", "ruff", @@ -156,6 +163,7 @@ "subdirs", "subsetting", "tagpath", + "thumbhash", "timonwong", "trashdb", "Triaging", @@ -167,6 +175,7 @@ "USERPROFILE", "venv", "Viljoen", + "vips", "winget", "Xsession", "xunit", diff --git a/repo-config/README.md b/repo-config/README.md index 2ccbbd7..c38bbfd 100644 --- a/repo-config/README.md +++ b/repo-config/README.md @@ -1,35 +1,35 @@ # repo-config -Repository and branch configuration held as committed files, kept out of `.github/` (which holds the GitHub-consumed configuration - workflows, Dependabot). +Repository and branch configuration held as committed files, kept out of `.github/` (which holds the GitHub-consumed configuration: workflows, Dependabot). -- `main.json` plus one `develop` variant - the branch rulesets as the writable API subset (`name`, `target`, `enforcement`, `bypass_actors`, `conditions`, `rules`). The `develop` payload is `develop.json` (`release` repos) or `operational/develop.json` (`operational` repos). These are the canonical expected payloads that the self-audit (`AUDIT.md`) diffs the live rulesets against. -- `operational/develop.json` - the `develop` ruleset for **operational** repos (registry `workflowModel: operational`): direct signed pushes, no PR gate. Present in operational repos only - a `release` repo does not have it. See "Rulesets" below. -- `configure.sh` - two modes over the GitHub API. `configure.sh apply [owner/repo] [release|operational]` creates-or-updates the settings, the Dependabot security features, and the rulesets idempotently (a full-payload update). `configure.sh check [owner/repo] [release|operational]` is the read-only inverse and exits non-zero on any drift, with the ruleset and settings assertions driven by the committed payloads so they stay repo-agnostic (rule presence, merge methods, and required checks, not a byte diff - so a GitHub-normalized stored ruleset does not false-positive). The command defaults to `apply`, the repo to the current one, and the model to the registry `workflowModel` lookup (or, absent a registry, inference from the carried `develop` payload - an ambiguous layout aborts rather than guesses). The model may be passed as the sole positional (`configure.sh check operational`). +- `main.json` plus one `develop` variant: the branch rulesets as the writable API subset (`name`, `target`, `enforcement`, `bypass_actors`, `conditions`, `rules`). The `develop` payload is `develop.json` (`release` repos) or `operational/develop.json` (`operational` repos). These are the canonical expected payloads that the self-audit (`AUDIT.md`) diffs the live rulesets against. +- `operational/develop.json`: the `develop` ruleset for **operational** repos (registry `workflowModel: operational`), taking direct signed pushes with no PR gate. Present in operational repos only, since a `release` repo does not have it. See "Rulesets" below. +- `configure.sh`: two modes over the GitHub API. `configure.sh apply [owner/repo] [release|operational]` creates-or-updates the settings, the Dependabot security features, and the rulesets idempotently (a full-payload update). `configure.sh check [owner/repo] [release|operational]` is the read-only inverse and exits non-zero on any drift, with the ruleset and settings assertions driven by the committed payloads so they stay repo-agnostic (rule presence, merge methods, and required checks, not a byte diff, so a GitHub-normalized stored ruleset does not false-positive). The command defaults to `apply`, the repo to the current one, and the model to the registry `workflowModel` lookup (or, absent a registry, inference from the carried `develop` payload, where an ambiguous layout aborts rather than guesses). The model may be passed as the sole positional (`configure.sh check operational`). ## Rulesets Two workflow models share `main.json` but differ on `develop` (registry `workflowModel`, default `release`): -- **`release`** (`develop.json`): `develop` requires squash merges with linear history and a PR - the feature-branch pipeline. -- **`operational`** (`operational/develop.json`): `develop` takes **direct signed pushes** - only `deletion`, `non_fast_forward`, and `required_signatures`; no PR, no status-check, no Copilot-on-push. CI runs on the push as advisory feedback. This is for live-service config repos that edit `develop` directly and promote a known-good snapshot to `main` via an occasional PR (see [GOVERNANCE.md "Branching Model"][governance-branching-model]). +- **`release`** (`develop.json`): `develop` requires squash merges with linear history and a PR, the feature-branch pipeline. +- **`operational`** (`operational/develop.json`): `develop` takes **direct signed pushes**, carrying only `deletion`, `non_fast_forward`, and `required_signatures`; no PR, no status-check, no Copilot-on-push. CI runs on the push as advisory feedback. This is for live-service config repos that edit `develop` directly and promote a known-good snapshot to `main` via an occasional PR (see [GOVERNANCE.md "Branching Model"][governance-branching-model]). -`main` (both models) requires merge-commit merges (no linear-history rule), signed commits, a passing `Check pull request workflow status job`, resolved review threads, and Copilot review, and blocks force-pushes and deletion - so a `develop -> main` promotion is always gated even when `develop` takes direct commits. Every ruleset intentionally leaves "Require branches to be up to date before merging" **off** - see [GOVERNANCE.md "Branching Model"][governance-branching-model]. +`main` (both models) requires merge-commit merges (no linear-history rule), signed commits, a passing `Check pull request workflow status job`, resolved review threads, and Copilot review, and blocks force-pushes and deletion, so a `develop -> main` promotion is always gated even when `develop` takes direct commits. Every ruleset intentionally leaves "Require branches to be up to date before merging" **off**, per [GOVERNANCE.md "Branching Model"][governance-branching-model]. -The result is **exactly two rulesets named `develop` and `main`** - the names are load-bearing (`GOVERNANCE.md` and the workflows reference them); only the `develop` *content* varies by model. The required check binds by name and only turns green after the repo's PR workflow runs once. +The result is **exactly two rulesets named `develop` and `main`**, and the names are load-bearing (`GOVERNANCE.md` and the workflows reference them). Only the `develop` *content* varies by model. The required check binds by name and only turns green after the repo's PR workflow runs once. ## Secrets -Publish credentials required per mechanism are enumerated in `spec/secrets.json`. A repo needs only the mechanisms its own publish targets use - a source-only repo needs none of the publish credentials below. NuGet and PyPI use keyless OIDC Trusted Publishing (no stored key; the publish job needs `id-token: write`, and PyPI additionally an `environment: pypi` gate). Docker Hub has no OIDC equivalent and uses a stored `DOCKER_HUB_USERNAME` + `DOCKER_HUB_ACCESS_TOKEN` in both the Actions and Dependabot secret stores. Codegen and merge-bot repos add a GitHub App (`CODEGEN_APP_CLIENT_ID` + `CODEGEN_APP_PRIVATE_KEY` in both stores; the app must be installed, not just created). App-token call sites use `client-id`, never the deprecated `app-id`. +Publish credentials required per mechanism are enumerated in `spec/secrets.json`. A repo needs only the mechanisms its own publish targets use, so a source-only repo needs none of the publish credentials below. NuGet and PyPI use keyless OIDC Trusted Publishing (no stored key; the publish job needs `id-token: write`, and PyPI additionally an `environment: pypi` gate). Docker Hub has no OIDC equivalent and uses a stored `DOCKER_HUB_USERNAME` + `DOCKER_HUB_ACCESS_TOKEN` in both the Actions and Dependabot secret stores. Codegen and merge-bot repos add a GitHub App (`CODEGEN_APP_CLIENT_ID` + `CODEGEN_APP_PRIVATE_KEY` in both stores; the app must be installed, not just created). App-token call sites use `client-id`, never the deprecated `app-id`. ## Repo Settings -The fleet-standard general settings live in [`settings.json`][settings-json] and are applied idempotently by `configure.sh apply` alongside the rulesets (`gh api PATCH /repos/{owner}/{repo}`). The two settings that depend on per-repo state - `has_discussions` (visibility) and `default_branch` (main-must-exist) - are computed by the script, not stored in the file. `configure.sh apply` also enables Dependabot vulnerability alerts and automated security updates - fleet policy applied via the API, not a `settings.json` key. `configure.sh check` validates all of these and exits non-zero on drift. +The fleet-standard general settings live in [`settings.json`][settings-json] and are applied idempotently by `configure.sh apply` alongside the rulesets (`gh api PATCH /repos/{owner}/{repo}`). The two settings that depend on per-repo state (`has_discussions` for visibility and `default_branch` for main-must-exist) are computed by the script, not stored in the file. `configure.sh apply` also enables Dependabot vulnerability alerts and automated security updates, which are fleet policy applied via the API rather than a `settings.json` key. `configure.sh check` validates all of these and exits non-zero on drift. - **Default branch `main`** (the script sets it only when a `main` branch exists, never pointing the default at a missing branch). -- **Merge methods**: `Allow merge commits` and `Allow squash merging` on, **rebase off** - each branch ruleset then picks its method (merge on `main`, squash on `develop`). +- **Merge methods**: `Allow merge commits` and `Allow squash merging` on, **rebase off**, and each branch ruleset then picks its method (merge on `main`, squash on `develop`). - **Auto-merge on** (the merge-bot needs it) and **`Always suggest updating pull request branches` on**. -- **`Automatically delete head branches` OFF - deliberately.** With it on, a `develop -> main` promotion (whose PR head is `develop`) would delete `develop`. There is no per-branch exemption, so the repo-wide toggle stays off to protect `develop`. **The CLI has the same trap: never `gh pr merge --delete-branch` a promotion PR whose head is `develop`** - the explicit flag deletes `develop` regardless of this setting (see [GOVERNANCE.md "Branching Model"][governance-branching-model]). -- **Wikis and Projects off. Discussions on public repos only** (off on private). **Sponsorships off** - the button is driven by `.github/FUNDING.yml`, not a REST toggle, and the fleet ships none. +- **`Automatically delete head branches` is OFF, deliberately.** With it on, a `develop -> main` promotion (whose PR head is `develop`) would delete `develop`. There is no per-branch exemption, so the repo-wide toggle stays off to protect `develop`. **The CLI has the same trap: never `gh pr merge --delete-branch` a promotion PR whose head is `develop`**, since the explicit flag deletes `develop` regardless of this setting (see [GOVERNANCE.md "Branching Model"][governance-branching-model]). +- **Wikis and Projects off. Discussions on public repos only** (off on private). **Sponsorships off**, since the button is driven by `.github/FUNDING.yml` rather than a REST toggle, and the fleet ships none. - **Actions / General**: allow GitHub Actions to create and approve pull requests (for the bots). From 671812b15f85b5e932ba1760fc6e7a491e0980b9 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 3 Aug 2026 10:45:08 -0700 Subject: [PATCH 05/12] Re-vendor the six verbatim fleet-law regions from the hub (#39) The hub advanced on six byte-locked regions since the last conformance round, and the audit reports each as a stale carry: a region matching a past hub revision rather than the current canonical. AGENTS.md "Context and Delegation Discipline" no longer ends a session on a third review round, since a loop still producing findings is the deliverable in progress, and it gains the rule that a wait separates "met", "not yet", and "cannot be reached" instead of rendering all three as silence. "Where the Rules Live" gains the router row for quoting data into agent-authored text. GOVERNANCE.md gains "Representative Data in Agent-Authored Text", absent here entirely, which binds agent-authored text to constructed data rather than data observed in the maintainer's environment. "Git and Commit Rules" gains the rule that an authorization to commit carries the push, because nothing reviews a local commit. "Verification Discipline" gains the rule that a launched process is not a result. "PR Review Etiquette" gains the five outcomes that close a finding, the rule that a low-confidence finding is not a low-value one, the requirement that a decline carry proof rather than an assertion, the answer format for a suppressed finding, and the warning that a review's own overview cannot be trusted to say whether findings exist. The new "PR Review Etiquette" text points at scripts/pr_review.py, which this repo does not carry. That is a known open question at the hub, and a verbatim region is carried unchanged rather than locally patched. Audit run 2026-08-03T16:52:36Z, hub 1ed0cc8, against develop@39c896b. All six regions are promoted to hub main a9cd154. Co-authored-by: Claude Opus 5 (1M context) --- AGENTS.md | 4 +++- GOVERNANCE.md | 35 ++++++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 27e033c..7e34d22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ An agent session is billed on the context it carries, not the work it does. Ever ### Session Scope - **One deliverable, one session.** A session covers one branch and one deliverable, and ends when that work merges. A multi-step task is one deliverable and stays in one session. Two unrelated tasks are two sessions even when they run back to back. -- **End a session at any of these, without being asked:** the branch changes, the pull request merges, the next task is unrelated to the last, or a third review round opens on the same pull request. +- **End a session at any of these, without being asked:** the branch changes, the pull request merges, or the next task is unrelated to the last. A review round is none of them. A loop still producing findings is the deliverable in progress, and a round count is not a reason to leave one open. - **Hand off in a file, never in context.** Close a session by writing at most 2 KB to a scratch file: branch, pull request link, what is done, the next command. A summary held in context is re-billed until the session ends, and a summary on disk is read once by whoever needs it. - **Re-derive state, do not carry it.** "This session already has the context" is the signal to split, not to continue. Context that has gone stale is worse than absent, because a file read hundreds of requests ago no longer describes the file. - **Compaction is a fallback, not the strategy.** It restarts context from a floor and climbs again, where a fresh session starts from zero. @@ -45,6 +45,7 @@ If a rule you were given does not cover what you find, stop and report it. Do no ``` - **Wait in a background process, not in a poll loop.** A review or CI wait is a sequence of near-identical requests, each billed for whatever context it happens to carry. Run the wait as one backgrounded command that returns when the condition is met. +- **A wait separates three outcomes, and says which one it reached.** The condition was met, it has not been met yet, and the wait cannot reach it at all are three different results, and a backgrounded wait that emits nothing renders all three identically. Run the command once in the foreground and read its output before backgrounding it, because a wait is only as good as the command inside it, and an unsupported flag on the installed tool version exits non-zero with an empty stdout that every naive test reads as "nothing yet". Never let a fallback stand in for a failed command, since `|| echo '[]'`, `|| true`, and `2>/dev/null` convert an error into that same reading, which is the suppression the write-safety rules already forbid on a mutation. Make the wait emit on failure as loudly as on success, so silence means "still running" and nothing else, and bound it, so a condition that is never coming ends in a report rather than in another wait. ## Where the Rules Live @@ -55,6 +56,7 @@ Every rule below is a level-two section of [`GOVERNANCE.md`](./GOVERNANCE.md). R | Why the rules are shaped this way | `Foundational Principles` | | Recording a durable lesson or updating governance | `Durable Knowledge and Self-Improvement` | | Any push, API mutation, comment, label, or merge | `Repository Boundaries and Write Safety` | +| Quoting data into a comment, commit, test, or doc | `Representative Data in Agent-Authored Text` | | Committing, signing, rebasing, force-pushing | `Git and Commit Rules` | | Branch choice, promotion, keeping branches in sync | `Branching Model` | | Releasing, version bumps, publishing | `Release Model` | diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 875e823..c14e1f0 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -26,9 +26,18 @@ A state-changing GitHub call is the highest-blast-radius thing an agent does her - **Never fabricate, guess, or reuse an identifier passed to a write.** Every id a state-changing call consumes (a node id, a numeric id, a thread or comment id) is captured from a live query in the **same** session into a variable and passed from there. Do not hand-type an id, guess it, recall it from memory or an earlier session, or copy it from documentation or an example. Ids commonly resolve **globally**, so a wrong-but-valid id does not fail. It writes to the wrong target, in someone else's repository. If a query returns no id, stop rather than invent one to proceed. - **A write is never a probe, and a write's output is never suppressed.** Never fire a state-changing call to see whether it works: decide it should happen, make it happen, and read the result. Never append output-discarding redirection or a force-success tail to a mutation (for example `>/dev/null`, `2>/dev/null`, `&>/dev/null`, `|| true`, `|| :`, `|| echo`), because the write's output is exactly what must be read. A write that appears to fail is **verified, not assumed harmless**, because the operation may have succeeded on the server while the client reported an error, so confirm the actual state before retrying or moving on. The ban targets hiding a *failure*. An ad-hoc call's response is the only signal you get, so `>/dev/null 2>&1`, `|| true`, and `|| echo`, which swallow the error stream or force success, are never acceptable on one. A committed script under `set -e` is a narrow exception: it may send a write's *stdout* to `/dev/null` to drop the success-response noise, because stderr stays visible and a failed write still aborts loudly (`repo-config/configure.sh` does exactly this). The exception is stdout-only suppression inside a reviewed, fail-loud script, never `2>&1` or a force-success tail, and never an ad-hoc command. +## Representative Data in Agent-Authored Text + +Agent-authored text illustrates with data the agent constructed, never with data it observed in the maintainer's environment. This binds every surface an agent writes: pull request and issue comments, review replies, commit messages, code, tests, fixtures, and docs. Reading real data is unrestricted, and what is bounded is what an agent copies out of the environment into text that is committed or posted. The rule holds for a private repository as much as a public one, since a repository's audience changes with one settings toggle while the text stays exactly where it was written, and it holds where the data is the maintainer's own, since the exposure happens on their behalf before they can weigh it. + +- **Synthetic evidence is the better evidence, not a weaker substitute.** A case constructed to carry the defect demonstrates it exactly and any reader can re-run it, where observed data proves the same thing and can never be reproduced by anyone else. A filename built to contain a newline is a complete proof of a newline-handling defect, and the real directory it was found in adds nothing the proof needed. Reaching for observed data to make a finding more convincing inverts which of the two is the stronger evidence. Where observed data is what revealed the defect, name its shape, meaning the property that triggers the fault, and construct a case that carries that property. +- **The exposure is one-way.** A public comment is fetched, cached, and indexed the moment it posts, so editing it afterwards is mitigation rather than a fix, and the edit leaves the original readable in the comment's edit history to anyone who can read the repository. Text that has already landed is reported to the maintainer rather than quietly rewritten, since the decision on what to do about it, deletion included, is theirs. Do not quote the exposed data again while reporting or investigating it, because a transcript, an issue, or a commit message written about the exposure reproduces it somewhere new. +- **No checker closes this.** A pattern finds an absolute home path or a drive letter, and gating that subset is worth doing as a floor. The exposure this rule exists for was name-shaped, and a name is not pattern-detectable, so a search of the offending text for path-shaped strings returns nothing while the names sit in plain sight. A gate here catches the easy half, and mistaking it for the answer is what stops anyone looking at the other half, which is why this is a judgment an agent applies rather than a check it waits for. + ## Git and Commit Rules - **Default to staging, not committing.** Stage changes with `git add` and leave `git commit` to the developer unless the developer has explicitly authorized the agent to commit for the current ask ("commit this", "open a PR", etc.). Authorization is scope-bound: it covers the commits needed for that specific task, not a blanket commit license for the rest of the session. +- **"Commit" means commit and push.** An authorization to commit carries the push to the feature branch the work belongs on, because nothing reviews a local commit. The Copilot review loop, the required status checks, and the maintainer all read the remote, so work that stops at `git commit` leaves the review unstarted and the branch's state private to one machine, which reads as progress while none of the gates have run. Push to the feature branch, never to a protected branch (see the Branching Model), and never with `--force`. Holding a commit locally is the narrower case, so it happens when the developer asks for it rather than by default. - **Check the working tree for the maintainer's own uncommitted edits before committing.** The maintainer hand-edits files live (often `README.md`/`HISTORY.md`, sometimes with the editor's LF->CRLF flip on top). Review `git status` first. If there are changes you did not make, ask whether to include them rather than bundling half-finished work or stranding it in an unrelated commit. - **All commits must be cryptographically signed (SSH or GPG).** Branch protection enforces this on both branches, and unsigned commits are rejected on push. Signing depends on environment configuration: `git config commit.gpgsign true`, a configured `user.signingkey`, and a working signing agent (loaded `ssh-agent` for SSH, or `gpg-agent` for GPG). If signing is not configured in the environment, **do not commit**. Surface the missing config to the developer and stop at `git add`. Verify before any agent-authored commit (`git config --get commit.gpgsign && ssh-add -L` or the GPG equivalent). **Signing must be live before the *first* commit, not retrofitted.** Turning on `Require signed commits` against a branch that already has unsigned commits forces a rewrite of that entire history to re-sign it, changing every commit SHA and making whoever does the rewrite the committer and signer of every commit (a rebase preserves the `author` field but not the original signatures, and you cannot sign another contributor's commits for them). During new-repo setup, never create commits until signing is verified. - **Commit under the committing account's own GitHub `noreply` identity, never a private, personal, or invented address.** The `author` and `committer` on every agent-authored commit are the GitHub `noreply` address of the account whose key signs the commit (above). GitHub issues these in a `username@users.noreply.github.com` or `ID+username@users.noreply.github.com` form, and for this single-maintainer fleet it is the owner's `ptr727@users.noreply.github.com`. Do not set `user.name`/`user.email` to a fabricated persona, bot name, or product name, and do not commit under whatever identity the environment happens to carry: verify `git config --get user.email` is that GitHub `noreply` address before committing. **Verify it, do not set it.** The identity is host configuration, set globally once, so a repo-local `user.email` is redundant where the global is right and a wrong identity where it is not, and it silently shadows the global it overrides. A mismatch is a host fault to surface to the maintainer rather than to patch per repo, because a local override hides a broken host that then commits under the wrong identity in every other repo on that machine. A wrong identity is not cosmetic: a private email trips GitHub's email-privacy push protection (GH007), and an unrecognized or invented author pollutes history. Identity is separate from signing: a wrong author does not by itself fail the signature rule, but the ad-hoc identities that produce it are typically also unsigned, which the signing rule above then rejects on push. @@ -212,6 +221,7 @@ The checks that separate work actually done from work that merely reports succes - **Never edit source through a shell heredoc when the text carries backslash escapes.** The shell consumes the escape and writes an invisible control character in its place, so a `\b` inside a regex becomes a backspace and the pattern silently matches nothing while every test still passes. Use a file-editing tool for such text. When a check inspects text for control characters, use `str.isprintable()` rather than a codepoint floor, since DEL and the Unicode format characters sit above 32 and are equally invisible in a diff. - **Never edit an active `.code-workspace` file.** A workspace file rewritten on disk can make VS Code reload the window, and a reload destroys the running agent session's context, so the work in flight is lost with nothing to catch it, and the trigger is not fully characterized (an agent's edit has caused the reload where a human's identical edit did not). Surface the needed change for the maintainer to apply by hand. - **A green check is not evidence the work happened.** A skipped job and a passing job are indistinguishable in the aggregated required check. When a job exists to exercise something, confirm from its log that it ran and produced the output it promises. +- **A launched process is not a result, and a cause nobody observed is not a diagnosis.** "The watcher is armed" names a process rather than a finding, so what gets reported is the output that process produced, and where it produced none, that absence is the report. The failure it prevents is an agent standing still on a condition that was met half an hour earlier, having announced the wait and never read it. Naming an external cause for such a stall afterwards, a throttle or a quota that appears nowhere in the record, turns a local defect into a story about someone else and closes the investigation on the wrong party, so read the record for the cause before naming one, and where the record does not carry it, report the cause as unknown. - **A workflow change is only fully exercised by CI.** Extracting a `run:` block and executing it locally validates the script and nothing else, because `secrets: inherit`, `permissions:`, `needs:` wiring, and reusable-workflow inputs resolve only in a real run. - **A review flags an instance, so fix the class.** When a reviewer cites one stale claim, one silent-narrowing pattern, or one mis-worded contract, sweep for its siblings before replying. Reviewers sample rather than enumerate. @@ -248,28 +258,47 @@ Drive the loop to green, meaning a review confirmed on the latest head SHA and e For provider-specific mechanics (how to request review, query review state, post replies, resolve threads), see the **GitHub Copilot Review Runbook** in [.github/copilot-instructions.md](./.github/copilot-instructions.md). This file owns the contract, and that file owns the mechanics. +### Every Finding Ends in an Action + +**A finding is closed by one of five outcomes, and a round count is never one of them.** The loop runs until no finding stands, however many rounds that takes, because the number of rounds measures how much was found rather than whether the work is done. A finding parked, waited out, or superseded by a push is still open. + +1. **It is real, so fix it.** Reply with the fixing commit SHA. +2. **It is not real, so disprove it in the thread**, with the command and its output, the code path that makes it impossible, or the rule that governs it. The proof is addressed to the reviewer as much as to the maintainer, since a decline it can read is what stops it raising the same thing next round. An assertion is not a proof and does not close a finding. +3. **It is real and deliberately not being fixed, which is the maintainer's call and not the agent's.** Say what the finding is, why the fix is unwanted, and get an explicit answer. Never suppress one by silence, by resolving the thread, or by an answer that reads as a decline while conceding the point. +4. **It is real and worth doing later, so file the issue first and reply with its link.** A deferral recorded only in a thread is lost the moment the pull request merges, so the issue is what carries it and the link is what proves it exists rather than being intended. This is for work the change did not create: an adjacent defect the reviewer noticed in passing, or a fix too large to ride along. It does not cover a defect in the change under review, because filing an issue about a bug you are about to merge is outcome 3 in other clothes, and that one is the maintainer's to decide. +5. **It keeps coming back, so fix the class rather than the instance.** A finding raised repeatedly against correct code is a defect in what the code communicates, not in the reviewer. Give it what it lacks: the non-obvious *why* as a comment where the code cannot state it, a clearer name, a narrower interface, or the rule change where the rule is what is wrong. A comment written for this earns its place under the comment rules like any other, so it states the why, stays short, and never cites a rule or addresses the reviewer. Making the noise stop is worth doing well, because a reviewer that repeats itself trains the reader to skim it, and skimming is how a real finding gets missed. + ### Triaging Review Comments +**A low-confidence finding is not a low-value one.** Copilot collapses the findings it is least sure of into the review body instead of raising a thread, and in this fleet's experience those are right the large majority of the time. Judge each one against the code, never against its confidence label. They are also the easiest to lose, because they appear in no thread, so a loop that polls threads alone reports a clean pass while they stand (see the Merge Gate, condition 3). + For each comment, classify before responding: - **Bug** - wrong behavior, missing test coverage, or a real divergence between code and docs. Fix it. Reply with the fixing commit SHA when done. - **Style/convention** - the comment cites a rule from this file or a language-specific style guide. Two cases: - The cited rule matches what the existing codebase already does -> fix the offending code. - - The cited rule contradicts what's in the tree, or industry norm -> **update the rule instead of the code**. The rule is wrong, not the code. Bouncing the same code across rounds is the symptom of a wrong rule. Heuristic: three rounds on the same style category means the rule needs adjusting and the user should authorize the rule change. + - The cited rule contradicts what's in the tree, or industry norm -> **update the rule instead of the code**. The rule is wrong, not the code. Bouncing the same code across rounds is the symptom of a wrong rule, so treat the recurrence itself as the finding and take it to the user for the rule change (outcome 5 above), rather than counting rounds until some threshold licenses it. - **Architectural opinion** - the comment proposes a different design ("constrain this to disabled-by-default", "move it elsewhere", "add a runtime guardrail"). This is judgment, not a bug. Surface it to the user with a recommendation, and don't apply it unilaterally. ### Responding and Resolution Expectations -Reply inline with either the fixing commit SHA (for accepted issues) or a concise rationale (for declines). Resolve review threads when addressed or intentionally declined with rationale. Issue-level comments (those at `repos/.../issues//comments` rather than tied to a specific line) have no resolution action, so acknowledge with a reply if needed and move on. +Reply inline with either the fixing commit SHA (for accepted issues) or the evidence that disproves it (for declines). **A decline carries proof rather than an assertion**, meaning the command and its output, the code path that makes the concern impossible, or the rule that governs it. "This is fine" is not a reply, and disagreeing without evidence is not addressing a finding, so a thread is not resolved on one. Resolve review threads when addressed, or when declined with that evidence recorded in the thread. Issue-level comments (those at `repos/.../issues//comments` rather than tied to a specific line) have no resolution action, so acknowledge with a reply if needed and move on. After the final push on a PR, sweep older threads from earlier rounds whose code paths no longer exist, or stale unresolved markers remain in the review UI. +**Answering a suppressed finding is a different act from replying in a thread, and it carries its own pairing.** A threaded reply sits under the comment it answers and the UI shows whether it is resolved. A suppressed finding has neither, so an answer that does not carry its own context is unverifiable: the maintainer cannot tell that it was seen, which finding it addresses, or whether any were skipped, and has to ask. An answer therefore **quotes the finding** in a blockquote, with its `file:line` anchor and enough of Copilot's own words to identify it, **carries one bold verdict per finding** (`Fixed in `, `Disproven`, or `No change needed`) so the outcomes are scannable without reading prose, **states the `(N)` count** the block heading gives so N answers can be checked against N findings, and **links the review** that raised them, since a PR accumulates rounds and an unlinked answer is ambiguous about which one it closes. One comment per review round keeps the answers together. + +**Read every round, not only the head.** A suppressed finding has no resolved state, so a push does not retire it: the finding simply stops appearing in a head-scoped query while remaining unanswered. Treating "superseded by a push" as "answered" is how rounds of findings go unanswered. `scripts/pr_review.py status ` reports every round and marks which are from earlier ones. + +**The review's own overview cannot be trusted to say whether findings exist.** A body that reads "Copilot reviewed N out of N changed files and generated no new comments" routinely carries a collapsed block of suppressed findings directly beneath that sentence. Read the body for the block rather than the summary line, because the summary line and `reviewDecision` and an empty unresolved-thread list all agree that a review with four outstanding findings is clean. + ### Escalating to the User Bring the user in when: - **Genuine design trade-off** surfaces (fail-open vs fail-closed, narrow vs broad refactor scope, "should we add a guardrail or trust the docstring"). Triage, recommend, ask. -- **Repeated friction** across rounds without convergence, which is the rule-needs-updating signal. Stop, summarize the pattern, and let the user authorize the rule change. +- **A recurring finding** the code keeps attracting, which is the fix-the-class signal. Summarize the pattern and bring the remedy, whether that is the rule change or what the code has to say differently to stop earning it. +- **A finding you judge real but do not want fixed**, which is outcome 3 above and is never the agent's call to make quietly. - **Architectural redesign** is requested rather than a bug fix. Surface with a recommendation, and never apply it unilaterally. Anti-pattern: don't keep flipping the code on the same style point. Flip the rule once and stick to the rule. From 65227ed2745fb973fcc7d8ecb352955cfacb7945 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 3 Aug 2026 10:45:18 -0700 Subject: [PATCH 06/12] Refresh the carried intent docs against the hub (#40) The audit's mechanical check only presence-checks the sections of an intent-fidelity carried file, so these four drifted silently while every byte-locked region stayed green. Hand-diffing each against the hub found stale paragraphs in all four, and one guarantee this repo describes but does not implement. CODESTYLE.md regains the Python section it had trimmed. The file's own intro says a repo keeps it whole and that an unused-language section costs nothing, so trimming it contradicted the sentence directly above the cut. Four paragraphs re-vendor: the clean-compile bullet now says it is not the whole gate and points at the repo's whole lint gate, MD033 now permits details and summary, HISTORY.md is now framed as mirroring the README opening rather than sharing a header, and the intro names the verification discipline. The Python tasks-mirror reference pointed at a hub catalog path this repo does not have, so it names the repo's own .vscode/tasks.json instead, which is what the file's VS Code config rule requires. WORKFLOW.md re-vendors D2.2, which now explains that the smoke check exits early while the job still reports success, and why a job-level if: would be wrong. D4.2 names GitCommitId, which is what get-version-task.yml already outputs and what github-release already passes as target_commitish. D1.1, D1.2, D1.4, D1.5 and D4.1 described machinery this repo does not have: a paths-filter changes job, smoke-build needing the validation job, and a publish-plan-task.yml. Each is adapted to what the tree actually does. Both absences are stricter than the guarantee rather than looser, since every push smoke-builds and no push publishes at all, so the fix is to state the repo's shape rather than build machinery it does not want. The S1, S2, S3, S5 and S6 trace scenarios are corrected the same way. .github/copilot-instructions.md re-vendors two runbook sections. Triggering and Polling still scoped the suppressed-finding query to the current head, which is how a finding stops appearing without ever being answered, so it now reads every round and marks which round each came from. The reviewer login note gains the third spelling, the REST timeline's bare Copilot. Bounded Retry Workflow gains the pending-request-nobody-picked-up state, which is invisible from the reviews alone and indistinguishable from patience, along with the clear-and-re-request recovery. repo-config/README.md re-vendors one paragraph and is now byte-identical to the hub. D4.5 is absent from this repo's WORKFLOW.md and is deliberately not added here. The pipeline does not satisfy it, so stating the guarantee before implementing it would replace a silent gap with a false claim. It lands with its fix. Audit run 2026-08-03T16:52:36Z, hub 1ed0cc8, against develop@39c896b. Co-authored-by: Claude Opus 5 (1M context) --- .github/copilot-instructions.md | 53 +++++++++- CODESTYLE.md | 167 ++++++++++++++++++++++++++++++-- WORKFLOW.md | 26 ++--- repo-config/README.md | 2 +- 4 files changed, 225 insertions(+), 23 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 02e75ab..3dc6b49 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -44,15 +44,22 @@ Auto-review on push is configured (via the branch ruleset's `copilot_code_review gh api repos///pulls//reviews --jq \ '.[] | select(.body | test("Suppressed comments|low confidence")) | .body' -# Scope it to the current head, so an answered finding from an earlier round does not re-open. +# Read every round, not only the head. A suppressed finding has no resolved state, so a push +# does not retire it: it simply stops appearing in a head-scoped query while still unanswered. +# Head-scoping this query is how four rounds went unanswered across three pull requests in a day. +gh api repos///pulls//reviews --jq \ + '[.[] | select(.body | test("Suppressed comments|low confidence"))] | length' + +# Mark which round each came from, since a finding on an older round may since be moot. PR_HEAD=$(gh pr view --json headRefOid --jq '.headRefOid') gh api repos///pulls//reviews --jq \ - "[.[] | select(.commit_id==\"$PR_HEAD\") | select(.body | test(\"Suppressed comments|low confidence\"))] | length" + "[.[] | select(.body | test(\"Suppressed comments|low confidence\")) + | {round: (if .commit_id == \"$PR_HEAD\" then \"head\" else \"earlier\" end), id}]" ``` **Round 1 is normally auto-seeded, so poll for it before trying to self-trigger.** Auto-review-on-open supplies the first review with no `botIds` call needed, but it can lag one to three minutes. After opening a PR (or the first push), **poll** for a Copilot review on the head SHA (see [Verify Review Covered Current Head](#verify-review-covered-current-head)) before concluding none ran. The `requestReviews` mutation below is for **re-requesting on later pushes** (a new head SHA). By then a prior review exists, so its bot node id is readable. A missing bot node id on round 1 therefore means "the auto-review has not landed yet - wait and poll," **not** "ask the maintainer to kick it off." -> **The reviewer login differs by API.** In **GraphQL** (`gh api graphql` and `gh pr view --json reviews`, which is GraphQL-backed) the `Bot.login` is `copilot-pull-request-reviewer`, with **no `[bot]` suffix**. In the **REST** API (`gh api repos/.../issues|pulls/...`) the same account's `user.login` is `copilot-pull-request-reviewer[bot]`, **with** the suffix. Each query below uses the correct form for its API, so match the API, not a single spelling, when adapting them. +> **The reviewer login differs by API, in three forms rather than two.** In **GraphQL** (`gh api graphql` and `gh pr view --json reviews`, which is GraphQL-backed) the `Bot.login` is `copilot-pull-request-reviewer`, with **no `[bot]` suffix**. In the **REST** API (`gh api repos/.../issues|pulls/...`) the same account's `user.login` is `copilot-pull-request-reviewer[bot]`, **with** the suffix. In a REST **timeline** `review_requested` event the `requested_reviewer` is a third spelling again, login `Copilot` with `type` `Bot`, so a filter written against either of the other two selects nothing there and reports a pull request with requests as having none. Match on the type plus a loose login test rather than on any one spelling, and each query below uses the correct form for its API. ```sh # 1. PR node id + the Copilot reviewer's bot node id (read from any existing @@ -141,6 +148,46 @@ This path is only for a **genuinely missing** review, meaning no Copilot review **A slow review is pending, not missing, so poll with backoff and never escalate on a timeout alone.** Copilot can lag far beyond the usual one-to-three minutes when it has been re-requested many times in quick succession, because it throttles under load, and a re-review landing tens of minutes after the request is normal. A poll that times out is therefore evidence only that the review has not landed *yet*, not that Copilot is done or unresponsive. Report the status as "review still pending" and keep polling on a widening interval (for example 20s steps, then a few minutes) rather than stopping. Enter the escalation step below only when the `requestReviews` mutation itself no-ops or errors, or after a genuinely long wait with the request confirmed accepted, never merely because one fixed poll window elapsed. +**Bound each wait, and read what Copilot actually posted before opening another one.** A poll that widens forever is indistinguishable from a poll that has stopped, and "still pending" is the honest report for exactly as long as evidence supports it. Two readings decide whether waiting again is warranted. Compare the request's timestamp against the newest Copilot activity of **any** kind on the pull request, since a reviewer that has already answered on a later head, or that posted an issue comment instead of a formal review, is not a reviewer running late, and a wait that keeps reporting "pending" against a landed review is a broken wait rather than a slow reviewer. Then read that newest response, because a Copilot answer naming a quota or a rate limit is a **terminal** outcome rather than a pending one: no formal review will land, so path (1) never matches the head and path (2) is correctly never confirmed, both paths behave exactly as specified, and the agent waits for something that is not coming. The fix is account-side and re-requesting does not change it, so report it to the maintainer and stop waiting. Where the newest response is neither a review nor a refusal you recognize, that too goes to the maintainer with its text, rather than being waited through. + +**A pending request nothing picked up is a third state, and it is the one that looks most like patience.** Copilot raises a `copilot_work_started` timeline event within about half a minute of accepting a request, and submits its review a few minutes later. A request that never draws one is not a slow review, it is a request nothing is acting on, and it stays that way indefinitely: one sat for thirteen and a half hours while the pull request read as waiting on the reviewer. Elapsed time cannot tell the two apart, since a genuinely slow round also shows no review, so read the event rather than the clock. `copilot_work_started` appears in the REST timeline only, and no GraphQL timeline item carries it: + +```sh +# The pending set (GraphQL, since the `gh pr view` projection cannot see a Bot reviewer). +gh api graphql -f query=' +{ repository(owner:"",name:""){ pullRequest(number:){ + reviewRequests(first:10){ totalCount + nodes{ requestedReviewer{ __typename ... on Bot{login} ... on User{login} } } } } } }' + +# The request and pickup events, newest last. A `review_requested` with no later +# `copilot_work_started` is the stuck state. Requests are filtered to the reviewer's own, +# since a human requested afterwards is a different request and reading it as this one +# reports a picked-up review as never picked up. `per_page` is the pagination cost. +gh api --paginate 'repos///issues//timeline?per_page=100' \ + --jq '.[] | select(.event == "copilot_work_started" or (.event == "review_requested" + and .requested_reviewer.type == "Bot" + and ((.requested_reviewer.login // "") | ascii_downcase | test("copilot")))) + | "\(.event) \(.created_at)"' +``` + +**Recover it by clearing the request and requesting again**, because the pull request UI offers no re-request control while a request is pending, and `requestReviews` with `union: true` adds a reviewer already in the set, which changes nothing. Read the pending set first, since `union: false` replaces the whole set and would drop a human reviewer requested alongside the bot. Where the clear-and-request does not draw a `copilot_work_started` within a minute or so, push a commit instead, since a new head raises a fresh request rather than poking a stale one. + +```sh +PR_NODE=$(gh pr view --json id --jq '.id') +# 1. Clear. `union: false` replaces the set, so an empty botIds removes the pending request. +gh api graphql -f query=' +mutation($pr: ID!) { + requestReviews(input: { pullRequestId: $pr, botIds: [], union: false }) { + pullRequest { reviewRequests(first: 10) { totalCount } } } +}' -F pr="$PR_NODE" +# 2. Request again, against a now-empty set, with $BOT_ID read as in "Triggering and Polling". +gh api graphql -f query=' +mutation($pr: ID!, $bot: ID!) { + requestReviews(input: { pullRequestId: $pr, botIds: [$bot], union: true }) { + pullRequest { reviewRequests(first: 10) { totalCount } } } +}' -F pr="$PR_NODE" -F bot="$BOT_ID" +``` + If a review did not run on the current head, retry: 1. Wait briefly and check head-SHA coverage (see above). diff --git a/CODESTYLE.md b/CODESTYLE.md index 6962f21..a874791 100644 --- a/CODESTYLE.md +++ b/CODESTYLE.md @@ -2,7 +2,7 @@ This is the single code-style guide for the fleet. The **General** section applies to every language. Each **language section** (.NET, Python) is self-contained: a repo follows only the section(s) for the languages it ships and ignores the rest. A repo keeps the whole file rather than trimming it. An unused-language section costs nothing, the same whole-file model as [`.editorconfig`][root], whose inert `[*.cs]` block a non-.NET repo keeps. -Cross-cutting *process* rules (PR titles, branching, US English, markdown style, comments philosophy, workflow YAML, PR review etiquette) live in [GOVERNANCE.md][governance] and are not repeated here. +Cross-cutting *process* rules (PR titles, branching, US English, markdown style, comments philosophy, workflow YAML, PR review etiquette, and the verification discipline that defines the pre-push lint gate) live in [GOVERNANCE.md][governance] and are not repeated here. ## General @@ -16,7 +16,7 @@ Use each tool's official casing in task labels, docs, and prose: `.NET` (not `.N Each language defines a **clean-compile** verification: the combination of build, formatter, linter, and code-analysis tools that must report clean before a commit. It is exposed as one or more **named** VS Code tasks (or, where a language ships no tasks, documented commands), and those definitions are the same across the fleet. The concrete names live in each language section below. -- **Run it after every code change.** The relevant language's clean-compile must pass before you commit, and CI runs the same checks as a backstop. +- **Run it after every code change, and it is not the whole gate.** The relevant language's clean-compile must pass before you commit. CI runs those same language checks as a backstop **plus everything else its validation workflow runs**, and all of it reports into the one required status, so a green clean-compile does not predict a green CI. That remainder is at least the doc-lint set (markdownlint, cspell, actionlint, `editorconfig-checker`) and whatever spec, config, and script gates the repo carries, so read the workflow for the full list rather than assuming this sentence enumerates it. What has to pass before a push is the repo's **whole** lint gate, per [GOVERNANCE.md "Verification Discipline"][governance-verification-discipline]. Each linter's known-working invocation is in [GOVERNANCE.md "Running the Linters Locally"][governance-running-the-linters-locally]. - **The named task definition is the canonical spec** - its exact command sequence, arguments, and strictness. You may run it through the VS Code task **or** by invoking the equivalent native commands directly, and either is fine **only if the sequence, arguments, and strictness match exactly**. No shortcuts and no more-lenient options (for example, never drop `--verify-no-changes` or loosen a `--severity`). - **A local commit/pre-commit gate is the repo's choice.** No single hook runner fits every language (a `dotnet`-tool runner like Husky.Net suits .NET but not Python), so none is mandated, but that is **not** a recommendation against commit gates. CI is the authoritative backstop regardless, and a local gate is an additive convenience a repo may wire and keep: Husky.Net (and `dotnet husky run` as a style step) for .NET, `pre-commit` for Python. Keeping a working gate is not drift. @@ -33,10 +33,10 @@ Each language defines a **clean-compile** verification: the combination of build These apply repo-wide, in every directory: -1. **Markdown linting**: All `.md` files must be lint-clean (error and warning free) via the VS Code `markdownlint` extension. [`.markdownlint-cli2.jsonc`][markdownlint-cli2] at the repo root is the single source of truth, since the davidanson `markdownlint` extension and a command-line `markdownlint-cli2` run both read it, so the IDE and CLI stay in lock-step. Rules it deliberately disables (e.g. `MD013` line-length) are **intentional**, so do not "fix" them. `MD033` inline HTML stays **enabled**: HTML comments are permitted (markdownlint does not flag them), HTML elements are flagged, and anything with a native markdown equivalent uses the markdown. Fix violations at the source rather than disabling rules. +1. **Markdown linting**: All `.md` files must be lint-clean (error and warning free) via the VS Code `markdownlint` extension. [`.markdownlint-cli2.jsonc`][markdownlint-cli2] at the repo root is the single source of truth, and the davidanson `markdownlint` extension and a command-line `markdownlint-cli2` run both read it, so the IDE and CLI stay in lock-step. Rules it deliberately disables (e.g. `MD013` line-length) are **intentional**, so do not "fix" them. `MD033` inline HTML stays **enabled**: HTML comments are permitted (markdownlint does not flag them), `details` and `summary` are allowed because a GitHub collapsible has no markdown equivalent, every other element is flagged, and anything with a native markdown equivalent uses the markdown. Fix violations at the source rather than disabling rules. 2. **Spelling**: All spelling must be clean via the CSpell VS Code integration, and words must be correctly spelled in **US English** (the repo-wide convention, per [GOVERNANCE.md][governance]). The shared `cspell.json` sets `"language": "en-US"` so British spellings are flagged, where a bare `"en"` accepts both US and British and silently passes the wrong spelling. Project-specific terms go in the shared `cspell.json` `words` list, the single source of truth the extension, CLI, and CI all read. The `.code-workspace` must **not** carry its own `cspell.words`/`cSpell.words` block, and when externalizing words into `cspell.json`, delete any word list left in the workspace (a leftover one duplicates the list and silently drifts). 3. **Spelling CI scope**: The enforced CI spell-check gate covers **`README.md` and `HISTORY.md` only**, because these are the files every repo visitor sees, so they must be clean. It is deliberately **not** all `**/*.md`: repos carry many markdown files full of technical terms, and gating every one of them would mean endlessly padding `cspell.json` just to keep CI green. Broad, live spell-checking across any file (source, markdown, text) is the **cspell editor extension's** job, so typos still surface to whoever is editing. A repo owner **may** widen their own CI file list, but README + HISTORY are the default; keep the CI workflow, the `Lint: Spelling` VS Code task, and the GOVERNANCE.md cspell one-liner on the same file list. The list is explicit (not a glob), so a repo that ships no `HISTORY.md` (e.g. one with no changelog) must drop it from all three surfaces and gate on `README.md` alone, since cspell errors on a listed file that does not exist. Markdown *linting* (item 1) stays repo-wide `**/*.md`, which does not choke on technical terms. -4. **`README.md` and `HISTORY.md` share the same header**: the `# PhotoCleaner` title and the one-line description under it match exactly in both files, so a reader landing on the changelog sees the same project identity. A change to one is applied to the other in the same commit, and the description is also what the GitHub About panel and the Docker Hub short description carry (see [GOVERNANCE.md][governance] "Repository Details"). +4. **`HISTORY.md` mirrors the README opening**: `HISTORY.md` is the maintainer-curated changelog and opens as the README's twin, carrying the same `# ` (without the README's ToC-omit comment) and the same intro paragraph copied verbatim, then a `## Release History` section. The mirrored opening keeps the project identity consistent for a reader who lands on the changelog directly. The audit checks that the title and intro match the README, with HTML comments stripped. ## .NET @@ -56,7 +56,7 @@ This is the style guide for any **.NET projects** in this repo. 2. **Analyzer configuration** - `<EnableNETAnalyzers>true</EnableNETAnalyzers>` with `<AnalysisLevel>latest-all</AnalysisLevel>` and `<AnalysisMode>All</AnalysisMode>` (full analyzer set enabled) - - `<TreatWarningsAsErrors>true</TreatWarningsAsErrors>`, so any diagnostic surfaced as a warning fails the build, so it must be fixed or deliberately suppressed, not left to accumulate (see [Analyzer Diagnostics and Suppressions][analyzer-diagnostics-and-suppressions]) + - `<TreatWarningsAsErrors>true</TreatWarningsAsErrors>`, so any diagnostic surfaced as a warning fails the build and must be fixed or deliberately suppressed, not left to accumulate (see [Analyzer Diagnostics and Suppressions][analyzer-diagnostics-and-suppressions]) 3. **CI lint backstop** - CI runs the clean-compile checks on every PR as the authoritative backstop @@ -75,7 +75,7 @@ The clean-compile task above is necessary and not sufficient. After every code c Shared MSBuild configuration is centralized at the repository root, never duplicated per project: - **`Directory.Build.props`** carries the properties every project shares: the analyzer set and `TreatWarningsAsErrors` from the Zero Warnings Policy above, plus `LangVersion`, `TargetFramework` where uniform, and any repo-wide build metadata. A csproj carries only what is genuinely project-specific (`OutputType`, `IsPackable`, project references). -- **`Directory.Packages.props`** enables central package management (`ManagePackageVersionsCentrally` true): every dependency version is declared once as a `PackageVersion` item, and a csproj's `PackageReference` items are versionless. One file to review on a bump, one Dependabot surface, and no version skew between projects. +- **`Directory.Packages.props`** owns central package management: it sets `ManagePackageVersionsCentrally` to `true` (in this file, not `Directory.Build.props`) and declares every dependency version once as a `PackageVersion` item, so a csproj's `PackageReference` items are versionless. One file to review on a bump, one Dependabot surface, and no version skew between projects. A repo whose projects still carry per-project analyzer settings or versioned `PackageReference` items is drifted, so move the shared property or version up to the root file rather than editing it in place. @@ -356,12 +356,167 @@ Follow the scope hierarchy in [Analyzer Diagnostics and Suppressions][analyzer-d 1. **Code reviews**: All changes go through pull requests +## Python + +*This section applies only to the Python side. A repo with no Python projects still carries it (the file is carried whole) and ignores it.* + +This is the style guide for any **Python project(s)** in this repo. + +**Adapt before propagating.** The rules below describe the default Python profile: a package that publishes to PyPI, type-checked by `pyright` in strict mode, dependencies in `[dependency-groups]`. A derived repo often differs; when it does, **adapt these fields to match the repo's actual toolchain rather than copying verbatim** (a verbatim copy that misdescribes the repo is inaccurate and gets rejected in review). The axes that commonly vary per repo: + +- **Type checker in CI** - `pyright` strict, **`mypy` in CI with `pyright` editor-only** (Pylance), or both. Whichever runs in CI is the one the clean-compile and the CI gate invoke. +- **Dependency declaration** - `[dependency-groups]`, or PEP 621 `[project.optional-dependencies]` (dev tools installed with `uv sync --extra <group>`). +- **Versioning / publishing** - a published package (`_version.py` + a version source + `uv build` + a PyPI publish step), or a **source-only** repo with a static `version` and no publish step (see [Versioning][versioning-section]). +- **Disabled markdownlint rules** - repo-specific. `.markdownlint-cli2.jsonc` at the repo root is the source of truth, not any example rule named here. +- **VS Code config home** - editor **settings/extensions** may live in `.vscode/*.json` **or** the `<Repo>.code-workspace`, while **tasks / launch / debug** configs can only be external `.vscode/*.json` (they cannot live in the workspace file). A `[vscode-tasks]` reference must point wherever the repo actually keeps `tasks.json`. + +**Two profiles.** A repo's Python is one of two shapes, declared as the `build` or `lint-only` profile and validated against the `pyproject.toml` shape. The rest of this section (uv project, `uv.lock`, `uv run`, `src` layout, pytest coverage) describes the **Project** shape (the `build` profile). The two differ by whether the Python has **third-party runtime dependencies**, which shows up structurally in `pyproject.toml`, so the audit reads the shape there (`python.profile.detect`): + +- **Project** (the `build` profile): the Python has third-party runtime dependencies, or is the repo's deliverable. It is a PEP 621 uv project: `[project]` with `dependencies` (dev tools in `[project.optional-dependencies]` or `[dependency-groups]`), a `[build-system]`, and a committed `uv.lock` (pinned LF, per [Line Endings][line-endings]). CI runs `uv sync --frozen` + `uv run <tool>`, so the lockfile pins tool versions. +- **Scripts** (the `lint-only` profile): stdlib-only utility scripts embedded in a **non-Python** repo (e.g. a Python tooling subtree of a `csharp` app). Run the tools with **`uvx`** (no project install, no lockfile): the `pyproject.toml` carries **only** tool config (`[tool.ruff]`, `[tool.mypy]`, and an optional `[tool.pyright]` editor block), with no `[project]`, no `[build-system]`, and no `uv.lock` (that metadata would misrepresent it as a shippable package). **mypy** is the type-check gate (there is no first-party package for pyright strict to anchor on), and a `[tool.pyright]` block in **standard** mode keeps Pylance quiet in the editor, the same mypy-gate/pyright-editor split the build profile uses. There is no lockfile, and a `uvx <tool>@<ver>` pin in a `run:` step is not something Dependabot tracks, so **CI runs `uvx ruff@latest` / `uvx mypy@latest`** rather than a manual pin that would silently go stale. The fleet rule is to pin only what Dependabot auto-updates (SHA-pinned actions, package deps) and otherwise run latest, so the VS Code tasks, README, and CI all run the unpinned latest here. `.py` files follow the repo's line-ending default (CRLF in a CRLF-default repo, and a shebang-executed script is LF-pinned by path, per [Line Endings][line-endings]). There is no pytest suite and no coverage gate. A script that carries a gate still earns tests, written with the standard library's `unittest` so they run under bare `python3` with nothing installed, as `test_<script>.py` beside the script it exercises; measure them with `uvx coverage@latest run -m unittest discover -s <dir>` when a number is wanted, without adopting a threshold. A co-present `csharp` type still carries `codecov.yml` for its own tests. + +### Toolchain + +| Tool | Role | Config | +|---|---|---| +| [uv][uv-link] | env, deps, build, publish (build/publish only where the repo ships a package) | `pyproject.toml` `[dependency-groups]` or `[project.optional-dependencies]`, `uv.lock` | +| [hatchling][latest-link] | build backend (published packages) | `pyproject.toml` `[build-system]` | +| [ruff][ruff-link] | lint + format + import sort | `pyproject.toml` `[tool.ruff]` | +| [pyright][pyright-link] | type checker (the default, a strict baseline) | `pyproject.toml` `[tool.pyright]` | +| [mypy][mypy-link] | additional/alternate type checker (optional, the CI checker in a mypy-in-CI repo, required for Home Assistant) | `pyproject.toml` `[tool.mypy]` (or per home-assistant/core) | +| [pytest][docs-link] | test runner | `pyproject.toml` `[tool.pytest.ini_options]` | + +**Type checking targets strongly typed, deterministic code.** `pyright` in **strict** mode is the default baseline on first-party code (a repo may instead run `mypy` in CI and keep `pyright` editor-only via Pylance, per the next paragraph) (`[tool.pyright]` `strict = ["src"]`, or the integration package for a Home Assistant repo, with tests run in standard mode). pyright is the anchor because **Pylance embeds it**, so the editor and the CLI/CI (`uv run pyright`) run the *same* engine and never disagree. The standalone `ms-pyright.pyright` extension stays in `unwantedRecommendations` because Pylance covers it. Relax strictness on **third-party** code only when a dependency has no usable types and no alternative (e.g. `pandas`): a targeted, commented `# pyright: ignore[...]` or a scoped `[tool.pyright]` override, never a blanket relaxation. + +**`mypy` is allowed, and required where the ecosystem demands it. It is not banned.** Running more than one checker is normal when each serves a purpose (the .NET side pairs `CSharpier` and `dotnet format` the same way), and pyright's inference and mypy's plugin ecosystem (e.g. `pydantic.mypy`) catch different classes of error. A **Home Assistant** integration runs `mypy --strict` because the platinum `strict-typing` quality-scale tier requires it; a pydantic-heavy library may opt in for the plugin. When a repo uses mypy it runs in **CI and the editor** (the `ms-python.mypy-type-checker` extension) so the two stay consistent, and its mypy command joins the clean-compile; a repo with no such need stays pyright-only, which is lighter and inherently consistent. + +### Local Development Loop + +From inside the Python project directory: + +```sh +uv sync # creates .venv, installs deps + dev group +uv run ruff format # auto-format +uv run ruff check --fix # auto-fix lint +uv run ruff check # verify lint clean +uv run ruff format --check # verify format clean +uv run pyright # verify types +uv run pytest # run tests +uv build # produce wheel + sdist in ./dist (published packages only) +``` + +The Python clean-compile (see [Clean-Compile Verification][clean-compile-verification]) is `uv run ruff format` + `uv run ruff check` + the repo's type checker: `uv run pyright`, or `uv run mypy src` where mypy is the CI checker, or both where the repo runs both (see Type checking above); run it (plus `uv run pytest`) before committing. These are documented commands; an optional VS Code tasks mirror (all `type: process`, no `&&` shell chaining, so it runs the same on any task shell) belongs in the repo's own `.vscode/tasks.json`. CI runs the same clean-compile commands as the authoritative backstop. Git hooks are opt-in; wire `pre-commit` for `ruff` and the type checker yourself if you want local enforcement. + +### Layout + +`src` layout, which keeps the package out of the repo root and prevents accidental imports of unbuilt code: + +```text +<python-project>/ + pyproject.toml + README.md + uv.lock # committed for reproducible CI + src/ + <package_name>/ + __init__.py + _version.py # published packages; a source-only repo uses a static version instead + <modules>.py + tests/ + __init__.py + test_<module>.py +``` + +### Code Style + +#### Formatting and Linting + +- **`ruff format` is authoritative.** Don't argue with the formatter, and if it reformats your code, that's the final form. Configure (line length, target version) in `pyproject.toml` `[tool.ruff]`, not via inline `# fmt:` directives. +- **Run `ruff check --fix` before committing.** Most ruff lint rules have safe autofixes, so let the tool handle them. The configured rule families are listed under `[tool.ruff.lint]` `select`. Add new rule families project-wide rather than scattering inline `# noqa` markers. +- **`# noqa` is a last resort.** When you must use one, scope it narrowly (`# noqa: E501`, not bare `# noqa`) and add a short comment on the same line explaining why. False-positive patterns that recur across the codebase belong in `[tool.ruff.lint]` `ignore` or per-file `[tool.ruff.lint.per-file-ignores]`, with a comment. Porting an existing codebase is not a license to add `ignore` / `per-file-ignores` blocks to mute newly surfaced lint. Fix it (see [Analyzer Diagnostics and Suppressions][analyzer-diagnostics-and-suppressions]). + +#### Comments + +- **Inline `#` comments**: keep tight and local. One line is preferred, but multi-line is fine when you need to document a non-obvious implementation constraint, a local trade-off, or coupling that future edits could easily break. Keep that rationale next to the affected block so the reviewer/maintainer sees it at edit-time. +- **Don't explain *what* the code does.** Well-named identifiers handle that. Don't reference the current task ("added for X", "used by Y"), which belongs in the PR description. + +#### Docstrings + +- Follow [PEP 257][pep-0257-link]. Focus docstrings primarily on the **behavior contract** (what callers and tests can rely on), public semantics, and edge-case expectations. Implementation-local rationale belongs in inline `#` comments, not docstrings. +- A short one-liner is fine for trivial functions and tests with self-documenting names. +- For non-trivial behavior (non-obvious test scenarios, contracts a test pins, edge cases callers must know about, design trade-offs that are load-bearing for future maintainers), write a one-line summary, blank line, then a details paragraph. Multi-paragraph docstrings are fine when the contract earns it. +- Design notes belong **in the code** (docstrings or inline comments). They do NOT belong in [`HISTORY.md`][history], which is end-user release notes, not a design log. + +#### Type Hints + +- **All public APIs are typed.** The repo's configured type checker runs on `src/` (pyright strict via `[tool.pyright]` `strict = ["src"]`, or `mypy` where that is the CI checker), and tests run in the checker's looser/standard mode. +- **Use modern syntax**: `list[int]` not `List[int]`, `dict[str, X]` not `Dict[str, X]`, `X | None` not `Optional[X]`, `from __future__ import annotations` only when needed for forward references. +- **Don't add `# type: ignore` to silence pyright errors without a comment** explaining the constraint. If a recurring false positive needs suppression, configure it project-wide in `[tool.pyright]`. A new port doesn't change this, so fix freshly surfaced type errors rather than muting them (see [Analyzer Diagnostics and Suppressions][analyzer-diagnostics-and-suppressions]). + +#### Naming + +- `snake_case` for functions, methods, variables, modules, package directories. +- `PascalCase` for classes, type aliases, type vars, enum members. +- `UPPER_SNAKE_CASE` for module-level constants. +- Single leading underscore for module-private, double leading underscore for name-mangled (rare, and usually means rethink the design). + +#### Imports + +- **Let ruff sort imports.** `[tool.ruff.lint]` `select` includes the `I` rule family (isort-equivalent). Don't hand-sort. +- Standard library first, then third-party, then first-party (the project itself), each block separated by a blank line, which ruff enforces automatically. +- Avoid wildcard imports (`from x import *`) outside `__init__.py` re-exports. + +#### Patterns to Avoid + +- **Don't add backward-compat shims, `# removed` markers, or rename-to-`_` for unused vars** - just delete. Git history is the audit trail. +- **Don't add error handling for impossible cases.** Trust internal code, and validate only at boundaries (user input, parsed config, external APIs). +- **Don't use exceptions for expected control flow.** Exceptions are for *unexpected* states. +- **Don't suppress errors silently** (`except Exception: pass`). Either handle the specific exception and document why it's safe, or let it propagate. + +### Tests + +- `pytest` with the configuration in `[tool.pytest.ini_options]`. Default invocation: `uv run pytest`. +- One test file per module under test, named `test_<module>.py`. +- Test functions named `test_<scenario>_<expected_behavior>`, descriptive and not numbered. +- Use fixtures (defined in `conftest.py` for shared ones, or per-test for narrowly-scoped) instead of setup/teardown methods. +- **Avoid mocking when fakes work.** Hand-rolled fakes that implement the protocol you depend on are usually clearer and break less than `unittest.mock` magic. +- **Test edge cases that the docstring promises**, not implementation details. If the test breaks when you refactor *without changing behavior*, the test is asserting on an implementation detail. + +### Versioning + +**Published packages.** `_version.py` ships with `__version__ = "0.0.0"` as a placeholder. Until you wire `_version.py` to something that increments (the usual options are `hatch-vcs`, a version.json bridge, or manual bumps), no new PyPI versions will land, and publishing with `skip-existing: true` keeps a stuck placeholder version from failing the run. + +**Source-only repos** (no PyPI publish, with a source-release on dispatch or no release at all) do not need `_version.py`: keep a static `version` in `pyproject.toml` `[project]`, or let the release pipeline's version source (e.g. NBGV + `version.json`) own the tag. There is no publish step to guard, so `skip-existing` does not apply. + +### Linter Cleanliness + +Before pushing or opening a PR: + +- VS Code's **Problems** pane should be quiet for the files you touched. The relevant linters are ruff (via the `charliermarsh.ruff` extension) and pyright (via the `ms-python.python` extension's bundled Pylance). +- The CI gate is `uv run ruff check`, `uv run ruff format --check`, the repo's type checker (`uv run pyright` or `uv run mypy src`), and `uv run pytest`, the same commands as the local loop above, run from the Python project directory. (Invoke them as separate steps, not `&&`-chained, so the runner shell is irrelevant.) +- Markdown in this directory follows the repo-wide [Markdown and Spelling][markdown-and-spelling] rules. + <!-- Repo --> [analyzer-diagnostics-and-suppressions]: #analyzer-diagnostics-and-suppressions [clean-compile-verification]: #clean-compile-verification [governance]: ./GOVERNANCE.md +[governance-running-the-linters-locally]: ./GOVERNANCE.md#running-the-linters-locally-known-working-invocations +[governance-verification-discipline]: ./GOVERNANCE.md#verification-discipline +[history]: ./HISTORY.md +[line-endings]: ./GOVERNANCE.md#line-endings +[markdown-and-spelling]: #markdown-and-spelling [markdownlint-cli2]: ./.markdownlint-cli2.jsonc [readme]: ./README.md [root]: ./.editorconfig +[versioning-section]: #versioning [vscode-tasks]: ./.vscode/tasks.json + +<!-- External --> + +[docs-link]: https://docs.pytest.org/ +[latest-link]: https://hatch.pypa.io/latest/ +[mypy-link]: https://mypy-lang.org/ +[pep-0257-link]: https://peps.python.org/pep-0257/ +[pyright-link]: https://microsoft.github.io/pyright/ +[ruff-link]: https://docs.astral.sh/ruff/ +[uv-link]: https://docs.astral.sh/uv/ diff --git a/WORKFLOW.md b/WORKFLOW.md index aeeeeb0..122e0cf 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -140,17 +140,17 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as input ### D1 - PR Fast-Feedback (Smoke) -- **D1.1 Only changed targets build.** Input: a PR touching some targets. Output: the paths-filter marks exactly those targets and only their smoke builds run. Unchanged targets skip. A repo's own targets MUST each have a filter entry (so a touched target is never silently skipped). *Prevents: rebuilding everything, and a changed target slipping through unbuilt.* -- **D1.2 A validation job always runs.** Input: any PR. Output: a type-appropriate validation job runs unconditionally and the aggregator `needs:` it. In a .NET repo this is the `unit-test` job (format/style/test). A non-.NET repo **replaces** it (not deletes) with its own validator (lint, schema-check) and re-points **every** `needs:` on it (both the aggregator and `smoke-build`, which `needs:` the validation job by name) to the replacement. *Prevents: a PR merging with no validation, or a dangling `needs:` that fails the whole workflow to load.* +- **D1.1 Only changed targets build.** Input: a PR touching some targets. Output: the paths-filter marks exactly those targets and only their smoke builds run. Unchanged targets skip. A repo's own targets MUST each have a filter entry (so a touched target is never silently skipped). *Prevents: rebuilding everything, and a changed target slipping through unbuilt.* **This repo runs no paths filter**, so every push builds both targets. That is stricter than the guarantee and prevents the same failure by never letting a changed target go unbuilt, at the cost of building an unchanged one. +- **D1.2 A validation job always runs.** Input: any PR. Output: a type-appropriate validation job runs unconditionally and the aggregator `needs:` it. In a .NET repo this is the `unit-test` job (format/style/test). A non-.NET repo **replaces** it (not deletes) with its own validator (lint, schema-check) and re-points **every** `needs:` on it to the replacement. *Prevents: a PR merging with no validation, or a dangling `needs:` that fails the whole workflow to load.* Here `validate` and `smoke-build` are siblings rather than a chain: both run unconditionally and the aggregator `needs:` both, so validation cannot be skipped and neither waits on the other. - **D1.3 Smoke never publishes and never uploads.** Input: `smoke: true`. Output: full compile/lint/test, but no registry/image push, no release, and **no** artifact uploads (every `upload-artifact`, including any aggregation job, is gated `!smoke`). *Prevents: a PR publishing; orphaned artifacts churning the storage quota.* -- **D1.4 Workflow-file changes are not smoke-built.** Input: a PR changing only `.github/workflows/**`. Output: the paths-filter excludes workflow files, so smoke-build skips. *Implication: a workflow-only change is not smoke-built, but actionlint still validates it in CI.* -- **D1.5 One required aggregator gates merge.** Input: any PR. Output: a single aggregator job must **succeed**, `needs:` the changes job and the validation job, treat a **skipped** smoke build as pass, and **block** on `failure`/`cancelled`. Its name is ruleset-bound: the job `name:` and the ruleset `context:` are the same string and MUST be renamed together, never independently. *Prevents: a paths-filter error letting a target-changing PR merge unbuilt.* +- **D1.4 Workflow-file changes are not smoke-built.** Input: a PR changing only `.github/workflows/**`. Output: the paths-filter excludes workflow files, so smoke-build skips. *Implication: a workflow-only change is not smoke-built, but actionlint still validates it in CI.* **N/A here**, since this repo runs no paths filter: a workflow-only change is smoke-built like any other. `test-pull-request.yml` runs on push so the reusable tasks resolve from the pushed head, which means such a change tests its own copy rather than the base branch's. +- **D1.5 One required aggregator gates merge.** Input: any PR. Output: a single aggregator job must **succeed**, `needs:` the changes job and the validation job, treat a **skipped** smoke build as pass, and **block** on `failure`/`cancelled`. Its name is ruleset-bound: the job `name:` and the ruleset `context:` are the same string and MUST be renamed together, never independently. *Prevents: a paths-filter error letting a target-changing PR merge unbuilt.* Here the aggregator is `check-workflow-status`, named `Check pull request workflow status job`, and it `needs: [validate, smoke-build]`. There is no changes job to need, and it demands **success** from both rather than treating a skip as pass, since neither is ever skipped. - **D1.6 Coverage is reported to Codecov (C# and Python).** Input: a C# or Python repo's validation/test job. Output: tests run with coverage collection (`dotnet test --collect:"XPlat Code Coverage"` or `pytest --cov-report=xml`) and a `codecov/codecov-action` step uploads it, **best-effort** (`continue-on-error` and/or `fail_ci_if_error: false`, so a Codecov outage or an absent token never reds the gate). `CODECOV_TOKEN` lives in the repo's **actions** secret store and reaches the reusable validator via `secrets: inherit`. Required for **every** C# and Python repo that has tests (see `spec/secrets.json` `typeMechanisms`). The repo also ships a **`codecov.yml`** that sets the project and patch statuses to **`informational: true`** so a coverage delta never gates a PR (a distinct knob from `fail_ci_if_error`, which only guards the upload step) and excludes intentionally-untested, non-shipped code (an example/demo or benchmark project) from the coverage denominator via `ignore`. A repo may override this to enforce a coverage threshold where its quality bar requires it. Coverage output is a build artifact, so `.gitignore` excludes it (e.g. `coverage/`, `*.cobertura.xml`; `.gitignore` is the full source of truth) so a blanket `git add -A` won't stage the untracked output. *Prevents: coverage silently going unreported; a stale, unused token; a coverage regression blocking an unrelated PR; a coverage artifact committed by a blanket add.* ### D2 - Input/State Validation at Entry - **D2.1 Validate before expensive work.** Output: a dedicated entry job/step asserts each cross-input/derived-state invariant and fails fast before builds. Downstream jobs `needs:` it. -- **D2.2 Release branch matches version classification.** Input: a real (non-smoke) release build. Output: the gate fails loudly if the default branch carries a prerelease suffix **or** a non-default branch carries none. It strips `+buildmetadata` before testing for the prerelease `-` (only a core/prerelease `-` counts), and it is **skipped on smoke** (a detached PR head always versions as prerelease). *Prevents: a non-default leg published as stable; a build-metadata false-positive; the gate blocking every default-base promotion PR.* +- **D2.2 Release branch matches version classification.** Input: a real (non-smoke) release build. Output: the gate fails loudly if the default branch carries a prerelease suffix **or** a non-default branch carries none. It strips `+buildmetadata` before testing for the prerelease `-` (only a core/prerelease `-` counts), and on a smoke build the **check exits early while the job still reports success** (a detached PR head always versions as prerelease). Read that as the validation being skipped rather than the job, because a job-level `if:` would skip the job itself, and a dependent skips with it unless that dependent opts out with `if: always()` and reads the result explicitly, the way the PR aggregator does. `github-release` carries `validate-release` in `needs:` and does **not** opt out, so a job-level skip there would couple the release to smoke through a second path on top of the `if:` it already carries. *Prevents: a non-default leg published as stable; a build-metadata false-positive; the gate blocking every default-base promotion PR.* - **D2.3 Publish only from main or develop.** Input: a dispatch publish. Output: a dispatch from any ref other than `main` or `develop` fails fast. *Prevents: cutting a release from an unintended branch.* - **D2.4 Mutually-exclusive / paired inputs are validated.** Input: a workflow with either/or or must-pair inputs (e.g. the docker-readme task's `repositories` XOR `manifest`+`manifest-jq`). Output: a half-filled or conflicting combination fails fast. *Prevents: a silent fall-through.* @@ -164,8 +164,8 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as input ### D4 - Release / Publish -- **D4.1 Gated single-branch publish.** Output: PRs smoke-test and publish nothing. A **human merge never auto-publishes**. A first `plan` job (`publish-plan-task.yml`) decides once and every job gates on it: publish on a **code-affecting bot push to `main`** (gated to the codegen App / Dependabot `github.actor`; an Actions-only bump matches no release path and publishes nothing), a **dispatch** of `main`/`develop`, or a **main-only weekly schedule** (Docker). A source-only repo publishes on dispatch only. Each run builds one branch. -- **D4.2 Tag the built commit.** Output: the release `target_commitish` is the built commit's SHA (NBGV's commit id), never `github.sha` or a moving branch ref. *Prevents: the tag landing on the default branch instead of the built tree.* +- **D4.1 Gated single-branch publish.** Output: PRs smoke-test and publish nothing. A **human merge never auto-publishes**. The fleet shape puts a first `plan` job (`publish-plan-task.yml`) in front and gates every job on it, covering a **code-affecting bot push to `main`**, a **dispatch** of `main`/`develop`, and a **main-only weekly schedule**. **This repo carries no plan job and no push trigger at all**: `publish-release.yml` runs on `workflow_dispatch` or the weekly `main`-only schedule, and its single job gates on `github.ref_name` being `main` or `develop`, so a dispatch from a feature branch is a no-op. Nothing publishes on a push, bot or human, and accumulated changes ship in the next scheduled or dispatched run. Each run builds one branch. +- **D4.2 Tag the built commit.** Output: the release `target_commitish` is the built commit's SHA (NBGV's `GitCommitId`), never a branch name or a separately re-resolved ref. *Prevents: the tag landing on the default branch instead of the built tree.* - **D4.3 Release contents.** Output: every release is a tag on the built commit plus the auto source zip, README, and LICENSE; file-producing targets attach `release-asset-*`; `prerelease` equals `branch != default`. A no-file-target repo that uses the release task (Docker-only, PyPI-only) reaches the tag-only shape **only** with `expect_release_assets: false` set by the caller (which relaxes `fail_on_unmatched_files` and skips the asset download). With the default `true` and no assets the release-create step fails. A source-only repo reaches the same shape through its inlined `action-gh-release` instead, with no release task or `expect_release_assets`. - **D4.4 No-op republish.** Input: a re-run whose version is unchanged. Output: nothing is re-pushed, because the release-create step is skipped when the tag exists (refreshed only on `workflow_dispatch`), and the paired asset-delete is skipped with it. Registry pushes are no-ops. The NuGet/PyPI publish steps are **not** statically gated on existence. They run and the **server** dedupes (`dotnet nuget push --skip-duplicate` turns a 409 into success; PyPI `skip-existing: true`). **Docker always re-pushes** the image (base-image refresh), independently of the release-create skip, within the same run. *Prevents: duplicate releases and wasted pushes.* @@ -238,13 +238,13 @@ For each *applicable* scenario, evaluate every job's `if:`/`needs:` against the | # | Input | Expected output | Exercises | | --- | --- | --- | --- | -| S1 | PR touching a build target | `changes` flags it; validation runs; that target's smoke build runs; no push, **no uploads**; validate-release **skipped (smoke), succeeds**; release **skipped**; aggregator **success**; version = prerelease; no release; no dangling artifacts | D1, D2.2, D3 | -| S2 | PR changing only docs | smoke-build **skipped**, validation runs, aggregator **success** | D1.1, D1.5 | -| S3 | PR changing only `.github/workflows/**` | filter excludes -> smoke-build **skipped**, aggregator **success** | D1.4 | +| S1 | PR touching a build target | validation runs; both smoke builds run; no push, **no uploads**; validate-release **skipped (smoke), succeeds**; release **skipped**; aggregator **success**; version = prerelease; no release; no dangling artifacts | D1, D2.2, D3 | +| S2 | PR changing only docs | validation and both smoke builds run (no filter to skip them), aggregator **success** | D1.1, D1.5 | +| S3 | PR changing only `.github/workflows/**` | validation and both smoke builds run against the pushed head, so the change tests its own copy, and the aggregator reports **success** | D1.4 | | S4 | PR base = default branch, carrying a build target | smoke versions as prerelease, validate-release **skipped (smoke)** so the default-branch arm does **not** fire, aggregator **success**, promotion not blocked | D1.3, D2.2 | -| S5 | bot push to `main` not touching a release path (e.g. an Actions bump) | the paths filter excludes it, so nothing publishes | D4.1 | -| S6 | code-affecting **bot** push to `main` (a human push/promotion, or any develop push, does not) | the `plan` job gates it to the App/Dependabot actor, and `main` publishes a release | D3, D4 | -| S7 | publish run (schedule, a bot push to main, or a dispatch) | builds the **one** trigger branch: `main` -> `X.Y.Z`, `prerelease=false`, registry stable, badge/readme run; `develop` -> `X.Y.Z-g<sha>`, `prerelease=true`, registry prerelease; `release-asset-*` consumed-then-deleted; PyPI build-artifact deleted after its publish; **no dangling artifacts** | D3, D4, D5, D6, D7 | +| S5 | bot push to `main` (e.g. an Actions bump) | `publish-release.yml` has no push trigger, so nothing publishes | D4.1 | +| S6 | code-affecting push to `main`, bot or human | still nothing publishes, and the change ships in the next scheduled or dispatched run | D3, D4 | +| S7 | publish run (the weekly `main` schedule, or a dispatch) | builds the **one** trigger branch: `main` -> `X.Y.Z`, `prerelease=false`, registry stable, badge/readme run; `develop` -> `X.Y.Z-g<sha>`, `prerelease=true`, registry prerelease; `release-asset-*` consumed-then-deleted; PyPI build-artifact deleted after its publish; **no dangling artifacts** | D3, D4, D5, D6, D7 | | S8 | dispatch from a ref other than `main` or `develop` | **fails fast** | D2.3 | | S9 | re-run publish, version unchanged | release-create **skipped**, `release-asset-*` delete **skipped**; NuGet/PyPI pushes no-op (server dedupe); **PyPI build-artifact still deleted** (its publish ran); **Docker still re-pushes** the image; no duplicate release | D4.4, D5.2 | | S10 | branch/version classification disagree | validate-release **fails loud**, build/publish skip | D2.2 | diff --git a/repo-config/README.md b/repo-config/README.md index c38bbfd..71fed17 100644 --- a/repo-config/README.md +++ b/repo-config/README.md @@ -23,7 +23,7 @@ Publish credentials required per mechanism are enumerated in `spec/secrets.json` ## Repo Settings -The fleet-standard general settings live in [`settings.json`][settings-json] and are applied idempotently by `configure.sh apply` alongside the rulesets (`gh api PATCH /repos/{owner}/{repo}`). The two settings that depend on per-repo state (`has_discussions` for visibility and `default_branch` for main-must-exist) are computed by the script, not stored in the file. `configure.sh apply` also enables Dependabot vulnerability alerts and automated security updates, which are fleet policy applied via the API rather than a `settings.json` key. `configure.sh check` validates all of these and exits non-zero on drift. +The fleet-standard general settings live in [`settings.json`][settings-json] and are applied idempotently by `configure.sh apply` alongside the rulesets (`gh api PATCH /repos/{owner}/{repo}`). The two settings that depend on per-repo state, `has_discussions` (visibility) and `default_branch` (main-must-exist), are computed by the script, not stored in the file. `configure.sh apply` also enables Dependabot vulnerability alerts and automated security updates, fleet policy applied via the API rather than a `settings.json` key. `configure.sh check` validates all of these and exits non-zero on drift. - **Default branch `main`** (the script sets it only when a `main` branch exists, never pointing the default at a missing branch). - **Merge methods**: `Allow merge commits` and `Allow squash merging` on, **rebase off**, and each branch ruleset then picks its method (merge on `main`, squash on `develop`). From 788f7b76af1a1054fba90731866f16484fbf1b4f Mon Sep 17 00:00:00 2001 From: Pieter Viljoen <ptr727@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:45:35 -0700 Subject: [PATCH 07/12] Restore the CRLF-default line-ending model and pin the Dockerfile (#41) The hub's model is a CRLF default with declared LF exceptions. This repo had dropped the `[*]` `end_of_line = crlf` default and enumerated CRLF per file type instead, which produces the same result for the types it listed and leaves every other type with no declared ending at all: `.slnx`, the `.code-workspace`, `LICENSE`, `.gitignore`, `.dockerignore`, and `.editorconfig` itself were all uncovered. Restoring the default covers them and makes the per-type CRLF lines redundant, so they go, along with the `[*.{json,jsonc}]` and `[*.{cmd,bat,ps1}]` sections that carried nothing else. Two LF pins were missing, and one of them matters. `Docker/Dockerfile` had no pin in either file although this repo ships one, and a CRLF there breaks RUN heredocs and line continuations. It is LF in the tree today, so intent held by accident rather than by rule; the pin is what keeps it that way through a checkout or a renormalize. `.husky/pre-commit` is an extensionless shebang script that matches no extension rule, so the CRLF default would have claimed it. `.gitattributes` already pinned it and `.editorconfig` did not, which is exactly the gap the restored default exposes. `.gitattributes` itself was LF while the hub's is CRLF, and under the restored default that is a violation of the repo's own rule, so it is rewritten as CRLF. Its comments regain the configure, renormalize, and inspect commands the hub carries. The hub's `catalog/`, `.github/actions/`, `scripts/*.py`, `host-setup/`, `spec/*.py` and `uv.lock` pins are hub-only paths and stay out. Verified: `editorconfig-checker` passes over the whole tree, and `git add --renormalize .` stages nothing beyond these two files, so the new pins describe the tree exactly rather than proposing a conversion. `git check-attr` confirms the Dockerfile pin resolves. Audit run 2026-08-03T16:52:36Z, hub 1ed0cc8, against develop@39c896b. Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> --- .editorconfig | 39 +++++++++++++++++++++++++-------------- .gitattributes | 28 +++++++++++++++++++--------- 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/.editorconfig b/.editorconfig index 0f13ec4..6af635d 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,9 +13,15 @@ # Root config root = true -# Defaults +# Defaults: CRLF is the default, and only the LF exceptions below are declared. +# Most LF pins are shared with `.gitattributes`, which git enforces: `*.sh`, the husky pre-commit shebang, and Dockerfiles. +# The workflow-YAML pin (`.github/workflows/*`) is `.editorconfig`-only. +# For that one, git stays passive (`* -text`) and CI (editorconfig-checker) enforces LF. +# Keep the `[*]` `end_of_line = crlf` default, which the Windows-GUI and WSL-engine workflow requires because Windows tooling misbehaves on LF. +# Every uncovered file type relies on that default too. [*] charset = utf-8 +end_of_line = crlf indent_size = 4 indent_style = space insert_final_newline = true @@ -23,39 +29,44 @@ trim_trailing_whitespace = true # Markdown files [*.md] -end_of_line = crlf trim_trailing_whitespace = false # Xml files [*.{xml,csproj,props,targets}] -end_of_line = crlf indent_size = 2 # Yaml files [*.{yml,yaml}] -end_of_line = crlf indent_size = 2 -# Workflow YAML is LF: Dependabot and Actions rewrite it with LF, so declaring LF keeps it consistent instead of -# mixed. git still leaves endings alone (`* -text`). This and CI (editorconfig-checker) enforce it. Other YAML is CRLF. +# Workflow YAML is LF, because Dependabot and Actions rewrite it with LF, so declaring LF keeps it consistent instead of mixed. +# Endings are still left alone by git (`* -text`), and this file plus CI (editorconfig-checker) enforce it. +# Other YAML stays CRLF. [.github/workflows/*.{yml,yaml}] end_of_line = lf -# Json files -[*.{json,jsonc}] -end_of_line = crlf - # Linux scripts [*.sh] end_of_line = lf -# Windows scripts -[*.{cmd,bat,ps1}] -end_of_line = crlf +# Husky.Net ships an extensionless pre-commit hook with a `/bin/sh` shebang, which matches no extension rule. +# Pin it by path, so a CRLF cannot break the hook's execution. +# `.gitattributes` carries the matching pin. +[.husky/pre-commit] +end_of_line = lf + +# Dockerfiles are LF, because CRLF breaks RUN heredocs and line continuations. +[{Dockerfile,*.Dockerfile}] +end_of_line = lf + +# .NET-only below, covering C# and ReSharper style. +# Everything above is the line-ending governance every derived repo carries. # C# files [*.cs] -end_of_line = crlf +# Suppressions follow CODESTYLE.md "Analyzer Diagnostics and Suppressions". +# Prefer a [SuppressMessage] attribute, or the owning project's .editorconfig. +# Relax a rule repo-wide here only when it applies to every project, never for a brownfield batch. dotnet_diagnostic.IDE0055.severity = none csharp_indent_block_contents = true csharp_indent_braces = false diff --git a/.gitattributes b/.gitattributes index cc3eb43..5aece30 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,9 +1,19 @@ -# Default: do not normalize line endings (`* -text`); .editorconfig end_of_line rules guide what the editor writes. -# The exception pins below are git enforcement - they force LF for execution-sensitive classes regardless of editor. -* -text - -# Scripts and extensionless executables must stay LF - a CRLF shebang breaks execution. -*.sh text eol=lf - -# Husky.Net ships an extensionless pre-commit hook with a /bin/sh shebang; pin it to LF so a CRLF cannot break execution. -.husky/pre-commit text eol=lf +# Default: git does not normalize line endings (`* -text`), and .editorconfig end_of_line rules guide what the editor writes. +# The exception pins below are git's own enforcement, forcing LF for execution-sensitive classes regardless of editor. +# Configure with: git config --global core.autocrlf false +# Renormalize with: git add --renormalize . +# Inspect with: git ls-files --eol +* -text + +# Exception: scripts must stay LF regardless of the `* -text` default, because a CRLF shebang breaks execution. +# `.editorconfig` covers `*.sh`, but an extensionless executable matches no extension rule. +# Pin those here, so git enforces LF on checkout and on `--renormalize`. +*.sh text eol=lf + +# Husky.Net ships an extensionless pre-commit hook with a `/bin/sh` shebang, so pin it to LF. +# A CRLF shebang would break the hook's execution. +.husky/pre-commit text eol=lf + +# Dockerfiles must be LF, because a CRLF breaks RUN heredocs and line continuations. +Dockerfile text eol=lf +*.Dockerfile text eol=lf From 252f5c598a713eff90e08ac51220202acf8711a2 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen <ptr727@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:46:18 -0700 Subject: [PATCH 08/12] Block a partial publish when a build fails (#42) WORKFLOW.md D4.5 requires that a build failure block every publish target. This repo's WORKFLOW.md was missing the guarantee entirely, and the pipeline did not satisfy it. build-docker needed only [get-version, validate, validate-release] and guarded solely on cancellation. The image builds from source and consumes no executable artifact, so the two builds were independent and nothing forced an ordering. On a real publish where build-executable failed, github-release skipped, so no tag and no release were cut, while build-docker was untouched and pushed the multi-arch image anyway, moving `latest`. An image shipped with no release behind it. build-docker becomes the terminal publish target: it needs build-executable, and its `if` gains `!failure()`. The distinction that matters is failed versus skipped. A failed upstream build must stop the push, and a skipped one must not, because `validate` is skipped on every smoke run and that run still has to build the image. The explicit get-version and validate-release result checks stay, so a job that somehow did not run cannot feed empty version inputs into a build. The cost is on smoke, where build-docker now waits for build-executable instead of running beside it, adding roughly the executable build's duration to PR feedback. Correctness over a subset of the wall clock. WORKFLOW.md gains D4.5 in the same commit rather than in the doc-refresh PR ahead of it, so the contract and the code that satisfies it land together instead of the file claiming a guarantee the pipeline breaks. The failure path is not reachable from a pull request, since smoke never publishes, so it is established from the needs graph rather than by observation. actionlint passes. Audit run 2026-08-03T16:52:36Z, hub 1ed0cc8, against develop@39c896b. Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> --- .github/workflows/build-release-task.yml | 10 ++++++++-- WORKFLOW.md | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-release-task.yml b/.github/workflows/build-release-task.yml index 6866be2..ef8404a 100644 --- a/.github/workflows/build-release-task.yml +++ b/.github/workflows/build-release-task.yml @@ -103,10 +103,16 @@ jobs: assembly_file_version: ${{ needs.get-version.outputs.AssemblyFileVersion }} assembly_informational_version: ${{ needs.get-version.outputs.AssemblyInformationalVersion }} + # The terminal publish target, so it needs every other build (D4.5). + # The image builds from source and consumes no executable artifact, so nothing else forces the ordering. + # Without it a failed build-executable skips github-release while the image still pushes and `latest` moves. + # That is a partial publish: an image with no release. + # `!failure()` is what blocks it, since a failed upstream build must stop the push where a skipped one must not. + # `validate` is skipped on smoke, which is the skip that must still build. build-docker: name: Build Docker job - needs: [get-version, validate, validate-release] - if: ${{ !cancelled() && needs.get-version.result == 'success' && needs.validate-release.result == 'success' && (needs.validate.result == 'success' || needs.validate.result == 'skipped') }} + needs: [get-version, validate, validate-release, build-executable] + if: ${{ !failure() && !cancelled() && needs.get-version.result == 'success' && needs.validate-release.result == 'success' && (needs.validate.result == 'success' || needs.validate.result == 'skipped') }} uses: ./.github/workflows/build-docker-task.yml secrets: inherit with: diff --git a/WORKFLOW.md b/WORKFLOW.md index 122e0cf..6e6aed6 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -168,6 +168,7 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as input - **D4.2 Tag the built commit.** Output: the release `target_commitish` is the built commit's SHA (NBGV's `GitCommitId`), never a branch name or a separately re-resolved ref. *Prevents: the tag landing on the default branch instead of the built tree.* - **D4.3 Release contents.** Output: every release is a tag on the built commit plus the auto source zip, README, and LICENSE; file-producing targets attach `release-asset-*`; `prerelease` equals `branch != default`. A no-file-target repo that uses the release task (Docker-only, PyPI-only) reaches the tag-only shape **only** with `expect_release_assets: false` set by the caller (which relaxes `fail_on_unmatched_files` and skips the asset download). With the default `true` and no assets the release-create step fails. A source-only repo reaches the same shape through its inlined `action-gh-release` instead, with no release task or `expect_release_assets`. - **D4.4 No-op republish.** Input: a re-run whose version is unchanged. Output: nothing is re-pushed, because the release-create step is skipped when the tag exists (refreshed only on `workflow_dispatch`), and the paired asset-delete is skipped with it. Registry pushes are no-ops. The NuGet/PyPI publish steps are **not** statically gated on existence. They run and the **server** dedupes (`dotnet nuget push --skip-duplicate` turns a 409 into success; PyPI `skip-existing: true`). **Docker always re-pushes** the image (base-image refresh), independently of the release-create skip, within the same run. *Prevents: duplicate releases and wasted pushes.* +- **D4.5 A build failure blocks every publish target.** Input: a real publish where one enabled build fails. Output: nothing publishes. `github-release` needs every build, so a failed build skips it (no tag, no release), and the terminal registry pusher (Docker) needs every other build and guards its `if` with `!failure() && !cancelled()`, so a failed build skips docker too (no image push) while a disabled or unchanged target (skipped, not failed) still lets docker build on smoke. *Prevents: a partial publish, e.g. a Docker image pushed while the executable build failed and no release was cut.* A repo pushing two registry targets at once would need a build/publish split behind an all-builds gate, which none does today. ### D5 - Resource Cleanup From 3d6ad1d6181344c7e0827f27651a5038e970dbc3 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen <ptr727@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:14:36 -0700 Subject: [PATCH 09/12] Raise the version floor to match the 1.1 declared in HISTORY.md (#43) HISTORY.md declares Version 1.1 for the verify command and the breaking exit-code changes, while version.json still floored at 1.0, so the next release cut from main would have been 1.0.18 and contradicted it. Only version.json moves. The Version and InformationalVersion properties in PhotoCleaner.csproj are local-build placeholders that CI overrides with the values Nerdbank.GitVersioning computes, so they do not track this file. Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 7cfe403..cd8a018 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", - "version": "1.0", + "version": "1.1", "publicReleaseRefSpec": [ "^refs/heads/main$" ] From 5e2eee0dc9a90b1954d36d0a9c21994678771984 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen <ptr727@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:37:26 -0700 Subject: [PATCH 10/12] Open the local gate chain with a dotnet tool restore (#44) * Open the local gate chain with a dotnet tool restore CSharpier and Husky.Net are local tools declared in .config/dotnet-tools.json, so on a clone whose package cache does not already hold them the documented chain failed on its first command with "Run dotnet tool restore to make the csharpier command available" and exit 1. CI already restored before its CSharpier step, and the prose called that out as a difference between the two. It is no longer one, so that clause is dropped. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * Attribute the restore to the clone, not to the .NET Format task The paragraph introduces the chain as the `.NET Format` task, so saying the chain opens with a restore read as a claim about that task. It is not one: `.NET Format` depends on `CSharpier Format` and `.NET Build`, and none of the three restores, so the task fails on a fresh clone exactly as the shell chain did. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> --- OPERATIONS.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 22adc3d..73e3e92 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -6,9 +6,10 @@ How this repository is run. It ships a .NET console application and a multi-arch ### Run the gates the way CI runs them -Local and CI runs read the same committed configuration, but they invoke it differently: locally the formatter writes, and in CI it only verifies. The [`.NET Format`](./.vscode/tasks.json) task is the local clean-compile chain, meaning `dotnet csharpier format`, then `dotnet build`, then the style verify. Run the chain and the suite before committing, since the chain never runs the tests and a change that compiles and formats cleanly can still be broken: +Local and CI runs read the same committed configuration, but they invoke it differently: locally the formatter writes, and in CI it only verifies. The [`.NET Format`](./.vscode/tasks.json) task is the local clean-compile chain, meaning `dotnet csharpier format`, then `dotnet build`, then the style verify. Run the chain and the suite before committing, since the chain never runs the tests and a change that compiles and formats cleanly can still be broken. CSharpier and Husky.Net are local tools declared in [`.config/dotnet-tools.json`](./.config/dotnet-tools.json), and neither the `.NET Format` task nor the two tasks it depends on restores them, so run `dotnet tool restore` on a fresh clone before that task or the chain below. A clone whose package cache does not already hold the tools otherwise fails on the format step: ```sh +dotnet tool restore dotnet csharpier format --log-level=debug . dotnet build dotnet format style --verify-no-changes --severity=info --verbosity=detailed @@ -16,7 +17,7 @@ dotnet test dotnet husky run ``` -CI differs in two places. It substitutes `dotnet csharpier check .` for the format step, after a `dotnet tool restore`, and it runs the suite as `dotnet test --collect:"XPlat Code Coverage" --results-directory ./coverage` so coverlet emits the report the Codecov upload consumes. The style verify is identical. So a local run that formats a file leaves CI clean, while an unformatted commit fails there rather than being fixed. +CI differs in two places. It substitutes `dotnet csharpier check .` for the format step, and it runs the suite as `dotnet test --collect:"XPlat Code Coverage" --results-directory ./coverage` so coverlet emits the report the Codecov upload consumes. The style verify is identical. So a local run that formats a file leaves CI clean, while an unformatted commit fails there rather than being fixed. The lint set runs in containers, matching the `Lint:` tasks in [`.vscode/tasks.json`](./.vscode/tasks.json): From 0335970c834aa88e7782bcd4713122e899f852b2 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen <ptr727@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:04:39 -0700 Subject: [PATCH 11/12] Correct the exit-code and Docker claims in OPERATIONS.md (#46) Two statements were written true and were left behind by the verify command. The exit-code section said a per-file failure does not change the exit code and listed only 0 and 1. ExitCode.Failed is 2 and every command returns it, so the note asserted the opposite of the behavior and the list was missing a code. It now points at the README table rather than restating it, and keeps only the operational reading: 0 and 2 both mean the command ran to completion. The tool section said the application needs no Docker daemon at runtime. verify runs the Immich decoder inside the immich-server image and its preflight throws when docker is missing, so Docker is a runtime dependency of that one command. Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> --- OPERATIONS.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 73e3e92..7cfad83 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -60,12 +60,9 @@ Workflow runs are the CI log. `gh run list --branch [branch]` and `gh run view [ The application logs to the console and to the file named by `--logfile`. Raise the level with `--loglevel debug` when a file is rejected and the reason is not obvious. -A calling script branches on the exit code rather than on output: +A calling script branches on the exit code rather than on output. Every command uses the same three codes, and [Exit Codes](./README.md#exit-codes) is the contract for what each one means. -- `0`: the command ran to completion. -- `1`: the command could not run, meaning a command-line parse or validation error, a cancellation, or an unhandled exception. A parse error short-circuits before any work starts, so nothing was touched. - -Note that a per-file failure does not currently change the exit code, so `0` means the command finished rather than that every file succeeded. Read the log to tell those apart. +The operational point is that `0` and `2` both mean the command ran to completion, and they differ only in whether every file succeeded. A pipeline that reads any non-zero code as "nothing happened" is therefore wrong: `1` is the only code that says the command reported nothing about the files, and a command-line parse error short-circuits before any work starts, so nothing was touched. ## Tool Usage @@ -73,8 +70,9 @@ The application shells out to external tools rather than reimplementing them, so - **exiftool** reads and writes metadata, invoked through `MediaUtilities.GetExifToolJsonAsync`. It is installed in the Docker image, and a native run needs it on `PATH`. - **ffmpeg** handles video, and is installed in the image alongside exiftool. +- **Docker** is a runtime dependency of `verify` alone, which runs the Immich decoder inside the `immich-server` image. The preflight exits `1` when the `docker` command is missing or the image cannot be prepared, so an unreachable daemon fails the run rather than condemning files. -The application itself needs no Docker daemon at runtime. Docker here is a packaging and tooling concern only, meaning the shipped image and the containerized linters above. +Every other command runs natively, so outside `verify` Docker is a packaging and tooling concern only, meaning the shipped image and the containerized linters above. The Immich API key can be given inline with `--apikey` or read from a file with `--apikey-file`, and the two are mutually exclusive. Prefer the file: an inline key lands in shell history and is visible in the process list for as long as the command runs. From 83f5101b1e36e9218b7123bb117884e89f44bd80 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen <ptr727@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:59:31 -0700 Subject: [PATCH 12/12] Kill the docker probe when its wait times out (#47) * Kill the docker probe when its wait times out ImmichImageAvailable started `docker image inspect` and returned false on a 60-second timeout without ending the child. Disposing a Process frees its handles only, so the probe outlived the test that started it and kept running alongside the rest of the suite. Confirmed the semantics with a standalone probe rather than assuming them: a child started the same way and left to Dispose was still present in /proc after the using block exited, and the same child was gone when Kill ran first. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * Wait out the killed probe and observe its drain tasks Kill signals termination rather than performing it, so the previous commit still returned while the child was alive. A bounded WaitForExit after the kill is what makes the child gone by the time the helper returns. The early return also left the two ReadToEndAsync drains unobserved. The kill closes their pipes, so a bounded WaitAll finishes them, and the AggregateException catch observes a drain that faulted on the closing pipe. Measured the asynchrony rather than reasoning about it: across five trials a killed child was still present in /proc when Kill returned, and gone after the bounded wait in every one. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> --- PhotoCleanerTests/VerifyTaskTests.cs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/PhotoCleanerTests/VerifyTaskTests.cs b/PhotoCleanerTests/VerifyTaskTests.cs index 9730da2..57c4bad 100644 --- a/PhotoCleanerTests/VerifyTaskTests.cs +++ b/PhotoCleanerTests/VerifyTaskTests.cs @@ -498,6 +498,30 @@ private static bool ImmichImageAvailable() Task<string> error = probe.StandardError.ReadToEndAsync(); if (!probe.WaitForExit(60_000)) { + // Disposing the probe frees its handles without ending the child. + // A wedged docker would otherwise outlive the test that started it. + try + { + probe.Kill(entireProcessTree: true); + + // Kill only signals, so the child is still alive when it returns. + probe.WaitForExit(5_000); + } + catch (InvalidOperationException) + { + // It exited between the timeout expiring and the kill. + } + + // The kill closes the pipes, so the drains finish rather than being left unobserved. + try + { + Task.WaitAll([output, error], 5_000); + } + catch (AggregateException) + { + // A drain that faulted on the closing pipe is observed here and discarded. + } + return false; }