diff --git a/.editorconfig b/.editorconfig index 0f13ec4..6af635d 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,9 +13,15 @@ # Root config root = true -# Defaults +# Defaults: CRLF is the default, and only the LF exceptions below are declared. +# Most LF pins are shared with `.gitattributes`, which git enforces: `*.sh`, the husky pre-commit shebang, and Dockerfiles. +# The workflow-YAML pin (`.github/workflows/*`) is `.editorconfig`-only. +# For that one, git stays passive (`* -text`) and CI (editorconfig-checker) enforces LF. +# Keep the `[*]` `end_of_line = crlf` default, which the Windows-GUI and WSL-engine workflow requires because Windows tooling misbehaves on LF. +# Every uncovered file type relies on that default too. [*] charset = utf-8 +end_of_line = crlf indent_size = 4 indent_style = space insert_final_newline = true @@ -23,39 +29,44 @@ trim_trailing_whitespace = true # Markdown files [*.md] -end_of_line = crlf trim_trailing_whitespace = false # Xml files [*.{xml,csproj,props,targets}] -end_of_line = crlf indent_size = 2 # Yaml files [*.{yml,yaml}] -end_of_line = crlf indent_size = 2 -# Workflow YAML is LF: Dependabot and Actions rewrite it with LF, so declaring LF keeps it consistent instead of -# mixed. git still leaves endings alone (`* -text`). This and CI (editorconfig-checker) enforce it. Other YAML is CRLF. +# Workflow YAML is LF, because Dependabot and Actions rewrite it with LF, so declaring LF keeps it consistent instead of mixed. +# Endings are still left alone by git (`* -text`), and this file plus CI (editorconfig-checker) enforce it. +# Other YAML stays CRLF. [.github/workflows/*.{yml,yaml}] end_of_line = lf -# Json files -[*.{json,jsonc}] -end_of_line = crlf - # Linux scripts [*.sh] end_of_line = lf -# Windows scripts -[*.{cmd,bat,ps1}] -end_of_line = crlf +# Husky.Net ships an extensionless pre-commit hook with a `/bin/sh` shebang, which matches no extension rule. +# Pin it by path, so a CRLF cannot break the hook's execution. +# `.gitattributes` carries the matching pin. +[.husky/pre-commit] +end_of_line = lf + +# Dockerfiles are LF, because CRLF breaks RUN heredocs and line continuations. +[{Dockerfile,*.Dockerfile}] +end_of_line = lf + +# .NET-only below, covering C# and ReSharper style. +# Everything above is the line-ending governance every derived repo carries. # C# files [*.cs] -end_of_line = crlf +# Suppressions follow CODESTYLE.md "Analyzer Diagnostics and Suppressions". +# Prefer a [SuppressMessage] attribute, or the owning project's .editorconfig. +# Relax a rule repo-wide here only when it applies to every project, never for a brownfield batch. dotnet_diagnostic.IDE0055.severity = none csharp_indent_block_contents = true csharp_indent_braces = false diff --git a/.gitattributes b/.gitattributes index cc3eb43..5aece30 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,9 +1,19 @@ -# Default: do not normalize line endings (`* -text`); .editorconfig end_of_line rules guide what the editor writes. -# The exception pins below are git enforcement - they force LF for execution-sensitive classes regardless of editor. -* -text - -# Scripts and extensionless executables must stay LF - a CRLF shebang breaks execution. -*.sh text eol=lf - -# Husky.Net ships an extensionless pre-commit hook with a /bin/sh shebang; pin it to LF so a CRLF cannot break execution. -.husky/pre-commit text eol=lf +# Default: git does not normalize line endings (`* -text`), and .editorconfig end_of_line rules guide what the editor writes. +# The exception pins below are git's own enforcement, forcing LF for execution-sensitive classes regardless of editor. +# Configure with: git config --global core.autocrlf false +# Renormalize with: git add --renormalize . +# Inspect with: git ls-files --eol +* -text + +# Exception: scripts must stay LF regardless of the `* -text` default, because a CRLF shebang breaks execution. +# `.editorconfig` covers `*.sh`, but an extensionless executable matches no extension rule. +# Pin those here, so git enforces LF on checkout and on `--renormalize`. +*.sh text eol=lf + +# Husky.Net ships an extensionless pre-commit hook with a `/bin/sh` shebang, so pin it to LF. +# A CRLF shebang would break the hook's execution. +.husky/pre-commit text eol=lf + +# Dockerfiles must be LF, because a CRLF breaks RUN heredocs and line continuations. +Dockerfile text eol=lf +*.Dockerfile text eol=lf diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 7476d19..3dc6b49 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -18,7 +18,7 @@ Summarized for VS Code's generators. The full rules, rationale, and examples are ## Reviewing Carried Fleet Content -Several of this repository's governance files are carried from a shared template and kept in sync across a fleet of sibling repositories, among them `AGENTS.md`, `GOVERNANCE.md`, `CODESTYLE.md`, `WORKFLOW.md`, this file, and the `repo-config/` rulesets. Most of `GOVERNANCE.md` is universal fleet law: every section that states a rule, as opposed to the two that describe this repository's own directory tree and devcontainer, is byte-locked and verified by an automated byte-for-byte match against the template canonical, not by line-by-line review. The two sections `AGENTS.md` carries are byte-locked on the same terms. +Several of this repository's governance files are carried from a shared template and kept in sync across a fleet of sibling repositories, among them `AGENTS.md`, `CODESTYLE.md`, `WORKFLOW.md`, this file, and the `repo-config/` rulesets. Most of `GOVERNANCE.md` is universal fleet law: every section that states a rule, as opposed to the two that describe this repository's own directory tree and devcontainer, is byte-locked and verified by an automated byte-for-byte match against the template canonical, not by line-by-line review. `AGENTS.md` is the thin router and carries two byte-locked sections of its own, with no repository-specific ones. Two constraints follow when reviewing that content. @@ -27,19 +27,39 @@ Two constraints follow when reviewing that content. ## GitHub Copilot Review Runbook -> This runbook implements the [GOVERNANCE.md "PR Review Etiquette"](../GOVERNANCE.md#pr-review-etiquette) review-loop contract for GitHub Copilot. Without it in-repo, an agent has no pointer to the reliable Copilot mechanics and falls back to known-broken paths (the no-op `POST /requested_reviewers`, the wrong bot-login filter). In the API snippets below, fill the `` placeholder (the PR number). +> This runbook implements the [GOVERNANCE.md "PR Review Etiquette"](../GOVERNANCE.md#pr-review-etiquette) review-loop contract for GitHub Copilot. Without it in-repo, an agent has no pointer to the reliable Copilot mechanics and falls back to known-broken paths (the no-op `POST /requested_reviewers`, the wrong bot-login filter). In the API snippets below, fill the `` / `` / `` placeholders. Use this section for provider-specific mechanics. The expected review loop *contract* (request review on every push, verify head-SHA coverage, triage findings, reply + resolve, escalate when stuck) is defined in [GOVERNANCE.md -> PR Review Etiquette](../GOVERNANCE.md#pr-review-etiquette). This section only describes how to make GitHub Copilot reliably execute it. ### Triggering and Polling -Auto-review on push is configured (via the branch ruleset's `copilot_code_review` rule with `review_on_push: true`) but fires inconsistently in practice - treat it as best-effort, not guaranteed. After every push, **re-request a review programmatically** via the GraphQL `requestReviews` mutation, passing the Copilot reviewer's bot node id in `botIds`. This drives the loop end-to-end without a UI hand-off. +Auto-review on push is configured (via the branch ruleset's `copilot_code_review` rule with `review_on_push: true`) but fires inconsistently in practice, so treat it as best-effort, not guaranteed. After every push, **re-request a review programmatically** via the GraphQL `requestReviews` mutation, passing the Copilot reviewer's bot node id in `botIds`. This drives the loop end-to-end without a UI hand-off. -**A review with no inline comments is still a completed review - not a failure, and not a reason to ask the maintainer to re-trigger.** Copilot very often posts a single formal review (GraphQL `state: COMMENTED`) whose body ends with "...reviewed N of N changed files ... and generated no comments" and adds **zero** inline threads. That review carries the head `commit.oid` and fully satisfies the loop - it is the clean-pass success case. Never read "no inline comments" as "the review didn't run," and never re-request or escalate to the maintainer because comments are absent. +**A review with no inline comments is still a completed review, not a failure, and not a reason to ask the maintainer to re-trigger.** Copilot very often posts a single formal review (GraphQL `state: COMMENTED`) whose body ends with "...reviewed N of N changed files ... and generated no comments" and adds **zero** inline threads. That review carries the head `commit.oid` and fully satisfies the loop, and it is the clean-pass success case. Never read "no inline comments" as "the review didn't run," and never re-request or escalate to the maintainer because comments are absent. -**Round 1 is normally auto-seeded - poll for it before trying to self-trigger.** Auto-review-on-open supplies the first review with no `botIds` call needed, but it can lag one to three minutes. After opening a PR (or the first push), **poll** for a Copilot review on the head SHA (see [Verify Review Covered Current Head](#verify-review-covered-current-head)) before concluding none ran. The `requestReviews` mutation below is for **re-requesting on later pushes** (a new head SHA); by then a prior review exists, so its bot node id is readable. A missing bot node id on round 1 therefore means "the auto-review has not landed yet - wait and poll," **not** "ask the maintainer to kick it off." +**Read the low-confidence findings, which are not inline threads.** A review body can carry a collapsed `
` block of findings Copilot withheld from the inline threads, and those findings appear nowhere in `reviewThreads`, so a loop that polls threads alone never sees them and reports a clean pass. **Match the block on more than one phrasing.** Its heading has appeared both as `Suppressed comments (N)` and as "Comments suppressed due to low confidence", so a filter keyed on either one alone silently reports zero suppressed findings on a review that has them, the same false clean this rule exists to prevent, one level up in the detection. They have been right repeatedly, including a rule stated more broadly than its check enforced and a check that skipped fenced blocks in every rule but one. Read the body of every review, investigate each suppressed finding on the same footing as an inline one, and answer it in the PR conversation, since a suppressed finding has no thread to reply on or resolve. -> **The reviewer login differs by API.** In **GraphQL** (`gh api graphql` and `gh pr view --json reviews`, which is GraphQL-backed) the `Bot.login` is `copilot-pull-request-reviewer` - **no `[bot]` suffix**. In the **REST** API (`gh api repos/.../issues|pulls/...`) the same account's `user.login` is `copilot-pull-request-reviewer[bot]` - **with** the suffix. Each query below uses the correct form for its API; match the API, not a single spelling, when adapting them. +```sh +# `test` with an alternation, not `contains` on one phrasing: the heading wording has changed. +gh api repos///pulls//reviews --jq \ + '.[] | select(.body | test("Suppressed comments|low confidence")) | .body' + +# Read every round, not only the head. A suppressed finding has no resolved state, so a push +# does not retire it: it simply stops appearing in a head-scoped query while still unanswered. +# Head-scoping this query is how four rounds went unanswered across three pull requests in a day. +gh api repos///pulls//reviews --jq \ + '[.[] | select(.body | test("Suppressed comments|low confidence"))] | length' + +# Mark which round each came from, since a finding on an older round may since be moot. +PR_HEAD=$(gh pr view --json headRefOid --jq '.headRefOid') +gh api repos///pulls//reviews --jq \ + "[.[] | select(.body | test(\"Suppressed comments|low confidence\")) + | {round: (if .commit_id == \"$PR_HEAD\" then \"head\" else \"earlier\" end), id}]" +``` + +**Round 1 is normally auto-seeded, so poll for it before trying to self-trigger.** Auto-review-on-open supplies the first review with no `botIds` call needed, but it can lag one to three minutes. After opening a PR (or the first push), **poll** for a Copilot review on the head SHA (see [Verify Review Covered Current Head](#verify-review-covered-current-head)) before concluding none ran. The `requestReviews` mutation below is for **re-requesting on later pushes** (a new head SHA). By then a prior review exists, so its bot node id is readable. A missing bot node id on round 1 therefore means "the auto-review has not landed yet - wait and poll," **not** "ask the maintainer to kick it off." + +> **The reviewer login differs by API, in three forms rather than two.** In **GraphQL** (`gh api graphql` and `gh pr view --json reviews`, which is GraphQL-backed) the `Bot.login` is `copilot-pull-request-reviewer`, with **no `[bot]` suffix**. In the **REST** API (`gh api repos/.../issues|pulls/...`) the same account's `user.login` is `copilot-pull-request-reviewer[bot]`, **with** the suffix. In a REST **timeline** `review_requested` event the `requested_reviewer` is a third spelling again, login `Copilot` with `type` `Bot`, so a filter written against either of the other two selects nothing there and reports a pull request with requests as having none. Match on the type plus a loose login test rather than on any one spelling, and each query below uses the correct form for its API. ```sh # 1. PR node id + the Copilot reviewer's bot node id (read from any existing @@ -47,7 +67,7 @@ Auto-review on push is configured (via the branch ruleset's `copilot_code_review PR_NODE=$(gh pr view --json id --jq '.id') BOT_ID=$(gh api graphql -f query=' { - repository(owner: "ptr727", name: "PhotoCleaner") { + repository(owner: "", name: "") { pullRequest(number: ) { reviews(first: 50) { nodes { author { __typename login ... on Bot { id } } } } } @@ -65,14 +85,14 @@ mutation($pr: ID!, $bot: ID!) { }' -F pr="$PR_NODE" -F bot="$BOT_ID" ``` -The bot node id is read from an existing Copilot **formal** review (`pullRequest.reviews`), so step 1 needs at least one prior formal review on the PR - the auto-review-on-open normally supplies the first one (it may have **no inline comments**; that still counts, and its bot node id is still readable). Poll for it (give auto-review-on-open a few minutes) before deciding it is missing. +The bot node id is read from an existing Copilot **formal** review (`pullRequest.reviews`), so step 1 needs at least one prior formal review on the PR, and the auto-review-on-open normally supplies the first one (it may have **no inline comments**, which still counts, and its bot node id is still readable). Poll for it (give auto-review-on-open a few minutes) before deciding it is missing. -**Cold start (round 1 not yet landed): read the id repo-wide, not from this PR.** The Copilot reviewer's bot node id is the reviewer bot *account's* node id and is **stable across every PR in the repo**. So a freshly opened PR that has neither a formal review nor an issue comment yet does **not** need UI seeding to bootstrap the id - read it from any prior Copilot review anywhere in the repo, then feed it into the `requestReviews` mutation to drive round 1. Query the **most recent** PRs (`first: 20` with an explicit newest-first order; plain `last: 20` returns the *oldest* PRs, which may predate Copilot on the repo), and **guard for an empty result** - an empty `$BOT_ID` means none of the sampled PRs carry a Copilot review. Widen the window (raise the count or paginate) before concluding the repo has never had one and falling back to UI seeding; never feed an empty id into the mutation: +**Cold start (round 1 not yet landed): read the id repo-wide, not from this PR.** The Copilot reviewer's bot node id is the reviewer bot *account's* node id and is **stable across every PR in the repo**. So a freshly opened PR that has neither a formal review nor an issue comment yet does **not** need UI seeding to bootstrap the id: read it from any prior Copilot review anywhere in the repo, then feed it into the `requestReviews` mutation to drive round 1. Query the **most recent** PRs (`first: 20` with an explicit newest-first order; plain `last: 20` returns the *oldest* PRs, which may predate Copilot on the repo), and **guard for an empty result**, since an empty `$BOT_ID` means none of the sampled PRs carry a Copilot review. Widen the window (raise the count or paginate) before concluding the repo has never had one and falling back to UI seeding; never feed an empty id into the mutation: ```sh BOT_ID=$(gh api graphql -f query=' { - repository(owner: "ptr727", name: "PhotoCleaner") { + repository(owner: "", name: "") { pullRequests(first: 20, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { reviews(first: 20) { nodes { author { __typename login ... on Bot { id } } } } } } @@ -86,21 +106,25 @@ if [ -z "$BOT_ID" ]; then fi ``` -If Copilot posted **only an issue comment** on this PR and no formal review, you can instead read the id from that comment's author (`pullRequest.comments` -> author `... on Bot { id }`). Manual UI seeding is the last resort - needed only for a repo that has **never** had a Copilot review, so no prior id exists anywhere to read; then use the mutation for every subsequent re-request. +If Copilot posted **only an issue comment** on this PR and no formal review, you can instead read the id from that comment's author (`pullRequest.comments` -> author `... on Bot { id }`). Manual UI seeding is the last resort, needed only for a repo that has **never** had a Copilot review, so no prior id exists anywhere to read. Use the mutation for every subsequent re-request. **Do NOT post `@Copilot review` as a PR comment.** That comment triggers the Copilot *coding agent* (`copilot-swe-agent[bot]`), which makes code changes rather than posting a review. -Known non-working request paths (don't rely on them - use the `requestReviews` mutation above instead): +Known non-working request paths (don't rely on them, and use the `requestReviews` mutation above instead): - `POST /requested_reviewers` with `reviewers=[Copilot]` can return 200 but no-op. - `copilot-pull-request-reviewer` as a requested reviewer slug returns 422. -- `requestReviews` with the reviewer's bot node id in **`userIds`** fails with `Could not resolve to User node` - the Copilot reviewer is a **Bot**, so its node id goes in **`botIds`** (as in the mutation above), never `userIds`. -- `suggestedActors(capabilities: [CAN_BE_ASSIGNED])` lists `copilot-swe-agent` (the coding agent), not `copilot-pull-request-reviewer` - do not source the reviewer's bot node id there. Read it from an existing review per step 1 above. -- There is no `removePullRequestFromReviewRequest` mutation, and removing the reviewer to force a fresh pass is unnecessary anyway - `requestReviews` with `union: true` re-fires the review on the current head. +- `requestReviews` with the reviewer's bot node id in **`userIds`** fails with `Could not resolve to User node`, because the Copilot reviewer is a **Bot**, so its node id goes in **`botIds`** (as in the mutation above), never `userIds`. +- `suggestedActors(capabilities: [CAN_BE_ASSIGNED])` lists `copilot-swe-agent` (the coding agent), not `copilot-pull-request-reviewer`, so do not source the reviewer's bot node id there. Read it from an existing review per step 1 above. +- There is no `removePullRequestFromReviewRequest` mutation, and removing the reviewer to force a fresh pass is unnecessary anyway, since `requestReviews` with `union: true` re-fires the review on the current head. ### Verify Review Covered Current Head -Before merging, confirm Copilot reviewed the current PR head SHA. Copilot may respond as either a formal review (carries an exact commit SHA) or an issue comment (no SHA - use the most recent Copilot comment for manual confirmation). Check both. +Before merging, confirm Copilot reviewed the current PR head SHA. Copilot may respond as either a formal review (carries an exact commit SHA) or an issue comment (no SHA, so use the most recent Copilot comment for manual confirmation). Check both. + +**Count matches and compare numerically, so an empty result cannot read as success.** A poll that captures a `gh api --jq` result and exits on `[ "$found" != "0" ]` treats an **empty** string as a landed review, and an empty string is exactly what a mis-written filter returns. Pipe the matches through `wc -l` and test `-gt 0`, so a query that finds nothing and a query that ran wrong both read as "not yet". A `gh` call that fails to run reaches the test the same way, because it writes its message to stderr and prints nothing to stdout, so the `$(...)` around it still yields the empty string. A mistyped or unsupported flag is the usual cause, and `gh` reports one as `accepts 1 arg(s), received 4` rather than as anything resembling a review verdict. + +**Check head coverage before reading merge-state, never the reverse.** A push makes the required checks go green before Copilot re-reviews the new head, so `mergeStateStatus` can read `CLEAN` in the window before any formal review covers the head. A poll that exits on `CLEAN` merges into that gap. Gate on a formal review whose `commit.oid` equals the current head SHA first, then on zero unresolved threads, and only then read merge-state. ```sh PR_HEAD=$(gh pr view --json headRefOid --jq '.headRefOid') @@ -112,33 +136,75 @@ gh pr view --json reviews --jq \ # 2. Issue comment - show the most recent Copilot comment for manual # confirmation. This is the REST API, so the login carries the `[bot]` suffix. -gh api repos/ptr727/PhotoCleaner/issues//comments --jq \ +gh api repos///issues//comments --jq \ '[.[] | select(.user.login=="copilot-pull-request-reviewer[bot]")] | last | {created_at, body: .body[:200]}' ``` -Coverage is confirmed when (1) exits 0 - **a formal review with no inline comments still satisfies path (1)**, because coverage is about the head SHA, not the comment count. For issue comments (path 2), body content is the only reliable signal - `created_at` is not: `git log -1 --format=%cI` is the **commit** timestamp, not the push timestamp, so amended or rebased commits can have an earlier timestamp and an older Copilot comment could satisfy a time check even though Copilot never saw the current head. Treat path (2) as confirmed only when the comment body explicitly refers to the current changes. +Coverage is confirmed when (1) exits 0, and **a formal review with no inline comments still satisfies path (1)**, because coverage is about the head SHA, not the comment count. For issue comments (path 2), body content is the only reliable signal, and `created_at` is not: `git log -1 --format=%cI` is the **commit** timestamp, not the push timestamp, so amended or rebased commits can have an earlier timestamp and an older Copilot comment could satisfy a time check even though Copilot never saw the current head. Treat path (2) as confirmed only when the comment body explicitly refers to the current changes. ### Bounded Retry Workflow -This path is only for a **genuinely missing** review - no Copilot review (formal *or* issue comment) covers the current head SHA after polling. A review that covered the head but produced no comments is a clean pass, not a missing review; do not enter this retry path for it. +This path is only for a **genuinely missing** review, meaning no Copilot review (formal *or* issue comment) covers the current head SHA after polling. A review that covered the head but produced no comments is a clean pass, not a missing review, so do not enter this retry path for it. + +**A slow review is pending, not missing, so poll with backoff and never escalate on a timeout alone.** Copilot can lag far beyond the usual one-to-three minutes when it has been re-requested many times in quick succession, because it throttles under load, and a re-review landing tens of minutes after the request is normal. A poll that times out is therefore evidence only that the review has not landed *yet*, not that Copilot is done or unresponsive. Report the status as "review still pending" and keep polling on a widening interval (for example 20s steps, then a few minutes) rather than stopping. Enter the escalation step below only when the `requestReviews` mutation itself no-ops or errors, or after a genuinely long wait with the request confirmed accepted, never merely because one fixed poll window elapsed. + +**Bound each wait, and read what Copilot actually posted before opening another one.** A poll that widens forever is indistinguishable from a poll that has stopped, and "still pending" is the honest report for exactly as long as evidence supports it. Two readings decide whether waiting again is warranted. Compare the request's timestamp against the newest Copilot activity of **any** kind on the pull request, since a reviewer that has already answered on a later head, or that posted an issue comment instead of a formal review, is not a reviewer running late, and a wait that keeps reporting "pending" against a landed review is a broken wait rather than a slow reviewer. Then read that newest response, because a Copilot answer naming a quota or a rate limit is a **terminal** outcome rather than a pending one: no formal review will land, so path (1) never matches the head and path (2) is correctly never confirmed, both paths behave exactly as specified, and the agent waits for something that is not coming. The fix is account-side and re-requesting does not change it, so report it to the maintainer and stop waiting. Where the newest response is neither a review nor a refusal you recognize, that too goes to the maintainer with its text, rather than being waited through. + +**A pending request nothing picked up is a third state, and it is the one that looks most like patience.** Copilot raises a `copilot_work_started` timeline event within about half a minute of accepting a request, and submits its review a few minutes later. A request that never draws one is not a slow review, it is a request nothing is acting on, and it stays that way indefinitely: one sat for thirteen and a half hours while the pull request read as waiting on the reviewer. Elapsed time cannot tell the two apart, since a genuinely slow round also shows no review, so read the event rather than the clock. `copilot_work_started` appears in the REST timeline only, and no GraphQL timeline item carries it: + +```sh +# The pending set (GraphQL, since the `gh pr view` projection cannot see a Bot reviewer). +gh api graphql -f query=' +{ repository(owner:"",name:""){ pullRequest(number:){ + reviewRequests(first:10){ totalCount + nodes{ requestedReviewer{ __typename ... on Bot{login} ... on User{login} } } } } } }' + +# The request and pickup events, newest last. A `review_requested` with no later +# `copilot_work_started` is the stuck state. Requests are filtered to the reviewer's own, +# since a human requested afterwards is a different request and reading it as this one +# reports a picked-up review as never picked up. `per_page` is the pagination cost. +gh api --paginate 'repos///issues//timeline?per_page=100' \ + --jq '.[] | select(.event == "copilot_work_started" or (.event == "review_requested" + and .requested_reviewer.type == "Bot" + and ((.requested_reviewer.login // "") | ascii_downcase | test("copilot")))) + | "\(.event) \(.created_at)"' +``` + +**Recover it by clearing the request and requesting again**, because the pull request UI offers no re-request control while a request is pending, and `requestReviews` with `union: true` adds a reviewer already in the set, which changes nothing. Read the pending set first, since `union: false` replaces the whole set and would drop a human reviewer requested alongside the bot. Where the clear-and-request does not draw a `copilot_work_started` within a minute or so, push a commit instead, since a new head raises a fresh request rather than poking a stale one. + +```sh +PR_NODE=$(gh pr view --json id --jq '.id') +# 1. Clear. `union: false` replaces the set, so an empty botIds removes the pending request. +gh api graphql -f query=' +mutation($pr: ID!) { + requestReviews(input: { pullRequestId: $pr, botIds: [], union: false }) { + pullRequest { reviewRequests(first: 10) { totalCount } } } +}' -F pr="$PR_NODE" +# 2. Request again, against a now-empty set, with $BOT_ID read as in "Triggering and Polling". +gh api graphql -f query=' +mutation($pr: ID!, $bot: ID!) { + requestReviews(input: { pullRequestId: $pr, botIds: [$bot], union: true }) { + pullRequest { reviewRequests(first: 10) { totalCount } } } +}' -F pr="$PR_NODE" -F bot="$BOT_ID" +``` If a review did not run on the current head, retry: 1. Wait briefly and check head-SHA coverage (see above). -1. Re-request the review via the `requestReviews` mutation (see "Triggering and Polling"); fall back to the GitHub PR UI only if the mutation no-ops. +1. Re-request the review via the `requestReviews` mutation (see "Triggering and Polling"), falling back to the GitHub PR UI only if the mutation no-ops. 1. Retry up to two more times (three total). 1. If still missing, mark review as blocked and escalate to the user/maintainer with what was attempted. ### Reply and Thread Resolution Workflow -Every id below is captured from a live query into a variable and passed from there - never hand-typed, guessed, or pasted as a `PRRT_...` literal. A node id resolves globally, so a fabricated or stale id does not fail, it writes to a real thread on an unrelated repository. This runbook implements [GOVERNANCE.md "Repository Boundaries and Write Safety"](../GOVERNANCE.md#repository-boundaries-and-write-safety): write only to this repo, capture every id from a live query, and never suppress a mutation's output. +Every id below is captured from a live query into a variable and passed from there, never hand-typed, guessed, or pasted as a `PRRT_...` literal. A node id resolves globally, so a fabricated or stale id does not fail, it writes to a real thread on an unrelated repository. This runbook implements [GOVERNANCE.md "Repository Boundaries and Write Safety"](../GOVERNANCE.md#repository-boundaries-and-write-safety): write only to this repo, capture every id from a live query, and never suppress a mutation's output. -List unresolved threads. Use `first: 100` with cursor-based pagination; if `hasNextPage` is true, re-run with `after: ""` to retrieve the next page: +List unresolved threads. Use `first: 100` with cursor-based pagination, and where `hasNextPage` is true, re-run with `after: ""` to retrieve the next page: ```sh gh api graphql -f query=' { - repository(owner: "ptr727", name: "PhotoCleaner") { + repository(owner: "", name: "") { pullRequest(number: ) { reviewThreads(first: 100) { nodes { @@ -156,12 +222,12 @@ gh api graphql -f query=' ' ``` -Reply on a thread, then resolve it. Capture the target thread's id into `$TID` from the listing query above - filter to the thread being answered by its `path`, and guard for an empty result so a mutation never runs on a guessed id. When a file carries more than one unresolved thread, `path` alone is ambiguous and `head -n 1` would pick the wrong one, so narrow by first-comment body - the query already fetches `comments(first: 1)` for this - by adding `and (.comments.nodes[0].body | contains(""))` to the `select`: +Reply on a thread, then resolve it. Capture the target thread's id into `$TID` from the listing query above, filtering to the thread being answered by its `path`, and guard for an empty result so a mutation never runs on a guessed id. When a file carries more than one unresolved thread, `path` alone is ambiguous and `head -n 1` would pick the wrong one, so narrow by first-comment body (the query already fetches `comments(first: 1)` for this) by adding `and (.comments.nodes[0].body | contains(""))` to the `select`: ```sh TID=$(gh api graphql -f query=' { - repository(owner: "ptr727", name: "PhotoCleaner") { + repository(owner: "", name: "") { pullRequest(number: ) { reviewThreads(first: 100) { nodes { id isResolved path comments(first: 1) { nodes { body } } } @@ -190,26 +256,27 @@ mutation($threadId: ID!) { }' -F threadId="$TID" ``` -Issue-level Copilot comments (those in `issues//comments`) have no resolution action - GitHub provides no API or UI to resolve them. Reply if the finding warrants it; no resolution step is needed or possible. +Issue-level Copilot comments (those in `issues//comments`) have no resolution action, since GitHub provides no API or UI to resolve them. Reply if the finding warrants it, but no resolution step is needed or possible. ### PR Edits and Merge-State Gotchas -- **`gh pr edit --title/--body` is broken here.** It touches the deprecated Projects-classic `projectCards` GraphQL field and **exits non-zero without applying the change** (a stale PR description then survives review rounds). Edit the title/body via the API and verify it took: GraphQL `updatePullRequest(input: { pullRequestId, title, body })`, or REST `gh api -X PATCH repos/ptr727/PhotoCleaner/pulls/ -F body=@body.md` (the `@` reads the body from a file - name it explicitly, not the literal `file`). -- **`main`/`develop` use rulesets, not classic branch protection.** The classic protection REST endpoint (`repos/.../branches//protection`) 404s - read the ruleset instead. A `mergeStateStatus` of `BLOCKED` on a green PR is usually just **unresolved review threads** (the ruleset requires thread resolution); resolving them moves it to `CLEAN`. (`BLOCKED` is a `mergeStateStatus` value; don't confuse it with the separate `mergeable` field's `MERGEABLE`/`CONFLICTING`, which reports merge conflicts, not review gates.) -- **Push -> head-SHA read race.** A `headRefOid` read taken immediately after a push can return the **old** head; re-read after the push registers, or a coverage poll evaluates the stale SHA. -- **Copilot is sometimes factually wrong** (e.g. it claimed `actionlint -color` "requires a value" - it is a boolean flag). Verify a finding before fixing; decline with evidence when it is wrong - that is distinct from dismissing a still-present finding as stale. +- **`gh pr edit --title/--body` is broken here.** It touches the deprecated Projects-classic `projectCards` GraphQL field and **exits non-zero without applying the change** (a stale PR description then survives review rounds). Edit the title/body via the API and verify it took: GraphQL `updatePullRequest(input: { pullRequestId, title, body })`, or REST `gh api -X PATCH repos///pulls/ -F body=@body.md` (the `@` reads the body from a file, so name it explicitly, not the literal `file`). +- **`main`/`develop` use rulesets, not classic branch protection.** The classic protection REST endpoint (`repos/.../branches//protection`) 404s, so read the ruleset instead. A `mergeStateStatus` of `BLOCKED` on a green PR is usually just **unresolved review threads** (the ruleset requires thread resolution); resolving them moves it to `CLEAN`. (`BLOCKED` is a `mergeStateStatus` value; don't confuse it with the separate `mergeable` field's `MERGEABLE`/`CONFLICTING`, which reports merge conflicts, not review gates.) +- **Push -> head-SHA read race.** A `headRefOid` read taken immediately after a push can return the **old** head, so re-read after the push registers, or a coverage poll evaluates the stale SHA. +- **Copilot is sometimes factually wrong** (e.g. it claimed `actionlint -color` "requires a value" when it is a boolean flag). Verify a finding before fixing, and decline with evidence when it is wrong, which is distinct from dismissing a still-present finding as stale. Reply-body conventions: - Accepted bug/style fix: include fixing commit SHA and a one-line summary. - Declined style comment: cite the rule (GOVERNANCE.md or the CODESTYLE.md language section) and the existing-tree precedent. - Declined architecture proposal: one-sentence rationale. +- Declined false positive on carried fleet content (a broken-link or dead-cross-reference flag inside byte-locked rule text): cite the "Reviewing Carried Fleet Content" section, since the reference is intentional and the text cannot be edited locally. After the final push, sweep-resolve stale older threads for removed code paths. ## When in Doubt -Read [AGENTS.md](../AGENTS.md) to find the section that governs your change, and [GOVERNANCE.md](../GOVERNANCE.md) for the rule text itself. For code-style rules, [`CODESTYLE.md`](../CODESTYLE.md) (its General section plus the relevant language section) is authoritative. Don't restate any of these files' rules in commit bodies or PR descriptions - keep those focused on the change itself. +Read [AGENTS.md](../AGENTS.md) to find the section that governs your change, and [GOVERNANCE.md](../GOVERNANCE.md) for the rule text itself. For code-style rules, [`CODESTYLE.md`](../CODESTYLE.md) (its General section plus the relevant language section) is authoritative. Don't restate any of these files' rules in commit bodies or PR descriptions, and keep those focused on the change itself. If you find a gap in the governance itself (this file, AGENTS.md, or GOVERNANCE.md is out of date, a rule is missing, something bit this repo and would bite the next), fix it in the governance docs as part of your change rather than only working around it locally. @@ -222,30 +289,35 @@ PhotoCleaner is a .NET 10 console application that processes media files in prep ### Project Structure - **Docker/**: Docker configuration - - `Dockerfile`: Two-stage build (SDK Alpine build -> runtime Alpine final); installs `exiftool` and `ffmpeg` in the final stage + - `Dockerfile`: Two-stage build (SDK Alpine build -> runtime Alpine final) that installs `exiftool` and `ffmpeg` in the final stage - **PhotoCleaner/**: Main console application - `Program.cs`: Entry point with logger setup (Main only) - - `CommandLine.cs`: System.CommandLine implementation for CLI parsing (`process`, `undo`, `import`, `index`, and `trash` subcommands) - - `MediaUtilities.cs`: Shared static utilities - `SupportedExtensions` (FrozenSet), `GetUniqueFileName`, `GetExifToolJsonAsync`, `SetCreateDateAsync`, video/duration constants - - `CommandRunner.cs`: Thin wrapper for command start/complete/error logging + - `CommandLine.cs`: System.CommandLine implementation for CLI parsing (`process`, `undo`, `import`, `index`, `trash`, and `verify` subcommands) + - `MediaUtilities.cs`: Shared static utilities, `SupportedExtensions` (FrozenSet), `GetUniqueFileName`, `GetExifToolJsonAsync`, `SetCreateDateAsync`, video/duration constants + - `CommandRunner.cs`: Thin wrapper for command start/complete/error logging, taking `Func>` and returning the command's exit code + - `ExitCode.cs`: The shared exit-code contract, being `Success` (0), `Error` (1, command could not run), and `Failed` (2, ran to completion with per-file failures) - `DatabaseScope.cs`: Generic async DB lifecycle helper (create, init, dispose) - `TrashDatabaseScope.cs`: Same lifecycle helper pattern for `TrashDatabase` - `FileEnumerator.cs`: Parallel file enumeration returning `(IReadOnlyList, int)` - - `DirectoryCleaner.cs`: Static helper that deletes empty subdirectories under a root (deepest-first; root itself is never deleted); used by `import` and `process` when `--deleteempty` is set - - `ProcessCommand.cs`: Process command orchestration - case conflict resolution, reprocessing loop, result reporting + - `DirectoryCleaner.cs`: Static helper that deletes empty subdirectories under a root (deepest-first, and the root itself is never deleted), used by `import` and `process` when `--deleteempty` is set + - `ProcessCommand.cs`: Process command orchestration, case conflict resolution, reprocessing loop, result reporting - `ImportCommand.cs`: Import command orchestration (formerly `OrganizeCommand`) - `IndexCommand.cs`: Index command orchestration - - `TrashCommand.cs`: Trash command orchestration - fetches trashed asset checksums from Immich API, stores SHA-1 hashes in a `TrashDatabase` + - `TrashCommand.cs`: Trash command orchestration, fetches trashed asset checksums from Immich API, stores SHA-1 hashes in a `TrashDatabase` - `UndoCommand.cs`: Undo command orchestration + - `VerifyCommand.cs`: Verify command orchestration, which enumerates, runs `VerifyTask`, and reports counts - `ProcessTask.cs`: Core file processing pipeline (validation, conversion, metadata) - - `UndoTask.cs`: Undo logic - two-pass algorithm that restores `.bak` files - - `ImportTask.cs`: Import logic - copies (default) or moves supported media files from source into date-based subdirectories under `--outpath`. Inserts a row keyed by SOURCE path into Import.db. Optional SQLite deduplication via `Database`. (Formerly `OrganizeTask`.) + - `UndoTask.cs`: Undo logic, two-pass algorithm that restores `.bak` files + - `ImportTask.cs`: Import logic, copies (default) or moves supported media files from source into date-based subdirectories under `--outpath`. Inserts a row keyed by SOURCE path into Import.db. Optional SQLite deduplication via `Database`. (Formerly `OrganizeTask`.) + - `VerifyTask.cs`: Verification logic, a decode pass that runs Immich's own `MediaRepository` inside `ghcr.io/immich-app/immich-server:release` via `docker run`, batching paths over stdin. Preflights the image before judging any file, so an infrastructure failure exits `Error` rather than marking files invalid + - `ImmichVerifyScript.cs`: The Node script run inside the Immich image, as const strings. Calls Immich's own compiled `MediaRepository`, `defaults`, and `ThumbnailConfig` rather than reimplementing the preview pipeline, so behavior tracks Immich across releases + - `VerifyResult.cs`: The AOT-compatible `ImmichVerifyLine` JSON model and its `ImmichVerifyJsonContext` source-generated context, together forming the container's output protocol - `IndexTask.cs`: Common DB upsert logic used by `process` and `index` commands; `IndexFileAsync` (single-file) returns `(IndexStatus, sha256, sha1, wasProcessed)`; `ExecuteAsync` (batch parallel) returns `(inserted, updated, unchanged, ignored, failed)`. When `options.MarkProcessed` is true, newly inserted rows are marked `is_processed=1` (used by `index --processed` to seed Process.db). - `Database.cs`: SQLite wrapper with a single `files` table (`path` PRIMARY KEY, `sha256`, `sha1`, `file_size`, `mtime_ticks`, `is_processed`); indexes on both hash columns; size/mtime caching via `ResolveHashesAsync` to skip rehashing unchanged files. Every write computes both sha256 and sha1 in a single read pass; both columns are non-null - `TrashDatabase.cs`: Simple SQLite wrapper for Immich trash hashes; single `trash_hashes` table (`sha1` PRIMARY KEY); used by `trash`, `import`, and `process` commands - `ImmichApiModels.cs`: AOT-compatible JSON models for Immich API (`ImmichSearchRequest`, `ImmichSearchResponse`, `ImmichAssetDto`) with `ImmichJsonContext` source generation - `DateFromPath.cs`: Static utility class for date inference from filenames/paths - - `ExifToolJson.cs`: JSON model for ExifTool metadata + - `ExifToolJson.cs`: JSON model for ExifTool metadata, including the `ExifTool:Validate` verdict and `ParseValidate` which splits it into error and warning counts - `SkippedExtensionTracker.cs`: Thread-safe tracker for unknown file extensions skipped during processing; used by all commands that filter by `MediaUtilities.SupportedExtensions` (`process`, `import`, `index`) - `HttpClientFactory.cs`: Polly resilience pipeline (retry, circuit breaker) and `SocketsHttpHandler` connection pooling - `AssemblyInfo.cs`: Assembly metadata (app name, version) used by `HttpClientFactory` for User-Agent header @@ -263,6 +335,7 @@ PhotoCleaner is a .NET 10 console application that processes media files in prep - `TrashDatabaseTests.cs`: TrashDatabase tests (8 tests) - `TrashCommandTests.cs`: TrashCommand tests with mock HTTP handler (6 tests) - `DirectoryCleanerTests.cs`: DirectoryCleaner static helper tests (6 tests) + - `VerifyTaskTests.cs`: Verify protocol parsing and script-contract tests (10 tests) ### Core Processing Pipeline @@ -277,11 +350,16 @@ if (!RenameMismatchedMimeExtensions() || !WarnDngVersion()) ``` +Before that chain runs, `CheckExifToolValidation` acts on the `ExifTool:Validate` verdict that +rides along with the metadata read. Only an error count fails the file (`ProcessResult.Invalid`). +Warnings are logged at debug level, because roughly three quarters of healthy files in a real +collection carry at least one. + ### State Management Pattern - **Primary Constructor Parameters**: Command and task classes use C# 12 primary constructors. All task classes take `CommandLine.Options options` as their first parameter, plus any non-option runtime params (e.g., `Database`, shared collections). Command classes take `(CommandLine.Options options, CancellationToken cancellationToken)` and pass `options` directly to task constructors. -- **Command/Task Separation**: Command classes (e.g., `ProcessCommand`) handle orchestration (file enumeration, DB lifecycle, result logging); task classes (e.g., `ProcessTask`) handle per-file business logic -- **Composable Infrastructure**: `CommandRunner`, `DatabaseScope`, and `FileEnumerator` are static helpers freely composed by command classes - no inheritance hierarchy +- **Command/Task Separation**: Command classes (e.g., `ProcessCommand`) handle orchestration (file enumeration, DB lifecycle, result logging), while task classes (e.g., `ProcessTask`) handle per-file business logic +- **Composable Infrastructure**: `CommandRunner`, `DatabaseScope`, and `FileEnumerator` are static helpers freely composed by command classes, no inheritance hierarchy - **Shared Collections**: `ConcurrentBag` for file names, `ConcurrentDictionary` for unknown extensions with case-insensitive comparison - **Parallel Processing**: Files processed using `Parallel.ForEachAsync` with `MaxDegreeOfParallelism` - **External Tool Integration**: Uses `CliWrap` for all external command execution (exiftool, ffmpeg, ffprobe) @@ -293,7 +371,7 @@ if (!RenameMismatchedMimeExtensions() ```csharp BufferedCommandResult result = await Cli.Wrap("exiftool") - .WithArguments(["-groupNames", "-json", _fileInfo.FullName]) + .WithArguments(["-groupNames", "-json", "-validate", "-all", _fileInfo.FullName]) .ExecuteBufferedAsync(); ``` @@ -304,7 +382,7 @@ BufferedCommandResult result = await Cli.Wrap("exiftool") ### Media File Processing Conventions - **FrozenSet Extensions**: Define supported extensions as `FrozenSet` with `StringComparer.OrdinalIgnoreCase` (e.g., `s_remuxExtensions`, `s_jpegExtensions`) -- **Case-Insensitive Matching**: Use FrozenSet `.Contains()` directly without `.ToLower()` - comparer handles case-insensitivity +- **Case-Insensitive Matching**: Use FrozenSet `.Contains()` directly without `.ToLower()`, comparer handles case-insensitivity - **File Type Categorization**: Group operations by file type requirements (remux vs re-encode vs audio-only) - **Single-Pass Optimizations**: Prefer single-loop iterations with early exit over multiple LINQ passes - **Skipped Extension Tracking**: Commands that filter files by `MediaUtilities.SupportedExtensions` pass a shared `SkippedExtensionTracker` instance to their task classes. The tracker collects unknown extensions (thread-safe via `Track()`), and the command calls `LogWarnings()` after processing to log them sorted. Used by `process`, `import`, and `index` commands. @@ -320,7 +398,7 @@ BufferedCommandResult result = await Cli.Wrap("exiftool") ### Date Inference System (DateFromPath.cs) - **Static Internal Methods**: All methods are `internal static` for testability with `InternalsVisibleTo` -- **DateFromPath.InferCreatedDate()**: Main entry point - tries filename first, then path fallback +- **DateFromPath.InferCreatedDate()**: Main entry point, tries filename first, then path fallback - **DateFromPath.ExtractDateFromFilename()**: Supports multiple filename patterns: - `YYYYMMDD_HHMMSS` format (e.g., `20210502_200152957_iOS-1747.jpg`) - `YYYYMMDD` format (e.g., `EX_20030219_3378.jpg`) @@ -332,16 +410,16 @@ BufferedCommandResult result = await Cli.Wrap("exiftool") ### Command Line Interface (CommandLine.cs) - **System.CommandLine Integration**: Uses modern .NET command line parsing -- **Five subcommands**: `process`, `undo`, `import`, `index`, `trash` - each with their own option set +- **Six subcommands**: `process`, `undo`, `import`, `index`, `trash`, `verify`, each with their own option set - **Required `--path` Parameter**: Single directory path using `Option`. Validated with `AcceptExistingOnly()` -- **Optional `--dryrun` Flag**: Non-destructive preview mode (process, undo, import - not index) +- **Optional `--dryrun` Flag**: Non-destructive preview mode (process, undo, import, not index) - **Optional `--threads` Parameter**: Controls parallel processing degree with `DefaultValueFactory = _ => Math.Min(Environment.ProcessorCount, 4)`. Validated to be > 0 and <= Environment.ProcessorCount using `Validators.Add()` (process, import, index) -- **Optional `--skipbackup` Flag** (process only): Skips all `.bak` file creation - originals are deleted/overwritten in-place. Logs a warning at startup. Disables undo. +- **Optional `--skipbackup` Flag** (process only): Skips all `.bak` file creation, originals are deleted/overwritten in-place. Logs a warning at startup. Disables undo. - **Optional `--deleteempty` Flag** (process, import): After the command completes, deletes empty child subdirectories from the target directory (deepest first; target root is never deleted). For `process` the target is `--path` (operated on in-place); for `import` it is `--outpath`. Implemented by `DirectoryCleaner.DeleteEmptyDirectories(root, dryRun)`. - **`import` subcommand** (formerly `organize`): Copies (default) or moves supported media files from `--path` sources into `--outpath/date/filename` directory structure. Date comes from EXIF metadata (falls back to `DateTime.MinValue` -> `"0001/01/01"` bucket when absent). `--format` (default `"yyyy/MM/dd"`) controls subdirectory naming and is validated as a date-only format (no time components). Uses `GetUniqueFileName` for collision handling (`foo_1.jpg` etc.). Parallel via `--threads` (same as `process`). `--deleteempty` (default `false`) deletes empty child subdirectories from `--outpath` after all files are imported. `--move` (default `false`) moves files instead of copying. `--tagpath` (default `false`) splits the source sub-directory path into tokens and writes each token as an `XMP:Subject` tag on the destination file using exiftool; filtered by `s_exiftoolWriteExtensions` (`.3gp`, `.arw`, `.cr2`, `.dng`, `.gif`, `.heic`, `.heif`, `.jpeg`, `.jpg`, `.mov`, `.mp4`, `.nef`, `.orf`, `.png`, `.psd`, `.rw2`, `.tif`, `.tiff`) checked via `meta.FileTypeExtension`; uses `-XMP:Subject-= / -XMP:Subject+=` to prevent duplicates while preserving existing tags. `--tags ` (optional) applies explicit comma-separated `XMP:Subject` tags to every imported file. `--datepath` (default `false`) infers the EXIF creation date from the source file path when no date is already embedded; applies the date to the destination file before restoring mtime. **`--db ` (Import.db) is the source-side dedup DB**: rows are keyed by `path = source_path` (NOT dest path) and hold the source file's hash/size/mtime. On each source file, import calls `GetByPathAsync(source_path)` for source-side hash caching, then `Sha256ExistsAsync(source_hash)` to skip already-imported sources. New imports insert a row at the source path. **No command outside `import` writes to source-keyed rows**, so dedup cannot be clobbered by later runs of `process`/`index`. `--trashdb ` skips files whose **source-file** SHA-1 is in Trash.db (Limitation: when import rewrites the dest via `--tags`/`--tagpath`/`--datepath`, the dest SHA-1 differs from the source SHA-1; Immich stored the dest SHA-1 from a prior upload, so the trash match is missed here and is caught later by `process --trashdb`). `--skipdb ` skips files whose SHA-256 matches a reference DB (read-only). Cross-collection dedup is typically implemented by pointing `--skipdb` at another collection's Import.db. `--rehash` forces recomputation of all hashes ignoring the size/mtime cache. -- **`index` subcommand**: Iterates all files in `--path`, upserts each into the `files` DB table via `IndexTask.ExecuteAsync` (insert new, update if hash changed, skip unchanged). `--db ` is **required**. No `--dryrun` (always writes to DB). Supports `--threads` and `--rehash`. `--processed` (optional) marks newly-INSERTED rows with `is_processed = 1`; useful when seeding a Process.db from existing files so `process` treats them as already-done. The flag does not flip the flag on existing rows. Reports `inserted`/`updated`/`unchanged`/`ignored`/`failed` counts. +- **`index` subcommand**: Iterates all files in `--path`, upserts each into the `files` DB table via `IndexTask.ExecuteAsync` (insert new, update if hash changed, skip unchanged). `--db ` is **required**. No `--dryrun` (always writes to DB). Supports `--threads` and `--rehash`. `--processed` (optional) marks newly-INSERTED rows with `is_processed = 1`, which is useful when seeding a Process.db from existing files so `process` treats them as already-done. The flag does not flip the flag on existing rows. Reports `inserted`/`updated`/`unchanged`/`ignored`/`failed` counts. - **`trash` subcommand**: Syncs trashed asset checksums from an Immich server into a local SQLite trash database. `--url` (Immich server URL, required), `--trashdb ` (trash database, required), and the API key supplied by exactly one of `--apikey` (inline) or `--apikey-file` (path to a file whose trimmed contents are the key). The two API-key options are mutually exclusive and exactly one must be provided; `--apikey-file` must reference an existing, non-empty, readable file (existence enforced by an option validator, non-empty/readable by a command-level validator; read failures are translated to validation errors, never thrown). The key is resolved at parse time by `CommandLine.ResolveApiKey`/`ReadApiKeyFile` (file contents preferred and `.Trim()`-med) and flows into `Options.ImmichApiKey`. Uses `POST /api/search/metadata` with `trashedAfter` to fetch all trashed assets, converts Base64 SHA-1 checksums to hex, and inserts them via `INSERT OR IGNORE`. Full sync (idempotent, append-only). No `--dryrun`. -- **`--trashdb` Flag** (import, process): SQLite database file with Immich trash hashes (synced by `trash`). In `import`, files matching the trash DB are skipped (this prevents re-importing photos the user trashed in Immich); the check is against the **source-file** SHA-1, so files whose dest SHA-1 was mutated by `import` itself (`--tags`/`--tagpath`/`--datepath`) will not match here even though Immich stored the mutated SHA-1 - `process --trashdb` catches those on the next pass. In `process`, matching files are **deleted from disk and from Process.db** before the per-file processing pipeline runs (cleanup of files trashed in Immich after upload, and the safety net for the import source-vs-dest SHA-1 drift). The Trash.db check is the durable safety net beyond Immich's ~30-day trash retention. +- **`--trashdb` Flag** (import, process): SQLite database file with Immich trash hashes (synced by `trash`). In `import`, files matching the trash DB are skipped (this prevents re-importing photos the user trashed in Immich); the check is against the **source-file** SHA-1, so files whose dest SHA-1 was mutated by `import` itself (`--tags`/`--tagpath`/`--datepath`) will not match here even though Immich stored the mutated SHA-1, `process --trashdb` catches those on the next pass. In `process`, matching files are **deleted from disk and from Process.db** before the per-file processing pipeline runs (cleanup of files trashed in Immich after upload, and the safety net for the import source-vs-dest SHA-1 drift). The Trash.db check is the durable safety net beyond Immich's ~30-day trash retention. - **Optional `--skipdb` Flag** (import only): SQLite database of files to skip (read-only SHA-256 check). Files whose SHA-256 matches a record in this DB are skipped without being recorded. Use this to skip files already present in another collection. - **Optional `--rehash` Flag** (process, import, index): Forces SHA-256 recomputation for every file, ignoring the size/mtime cache. SHA-1 is also recomputed when `--trashdb` is in use. Useful after filesystem operations that preserve mtime but change content. - **Optional `--duration` Flag** (process only): Overrides `ShortVideoDuration` (default `1.0`s). Videos in a live-photo-compatible format whose duration is <= this value are always deleted. Must be `> 0`. Stored in `CommandLine.Options.ShortVideoDuration` and read by `DeleteLivePhotosAsync`. @@ -379,18 +457,18 @@ See [`CODESTYLE.md`](../CODESTYLE.md) for build requirements, formatting command ### Video Conversion Logic - **Three-tier approach**: Remux (.mts, .m2ts, .mkv) -> Re-encode (.wmv, .avi, .3gp, .gif) -> Audio-only (.mov/.mp4 with PCM) -- **Backup Strategy**: Original files renamed to `.bak` extension after successful conversion; `BackupFile()` returns the backup path. A `{backup}.out` companion file (e.g. `img.gif.bak.out`) is written alongside the backup containing the full output path - this is needed when `GetUniqueFileName` appended a counter suffix (e.g. `img_1.mp4`) because the canonical name was already taken. When `options.SkipBackup` is true, no `.bak` or `.bak.out` files are created - the original is deleted after conversion. -- **Metadata Preservation**: After every ffmpeg conversion, `exiftool -TagsFromFile -all:all -overwrite_original` copies all source metadata to the output file. `ffmpeg -map_metadata` is not used - it is unreliable for Apple QuickTime-specific tags (e.g. `ContentIdentifier` in the `mdta`/`keys` atom). `TagsFromFile` handles cross-format date mapping, so no separate date-setting step is needed after conversion. +- **Backup Strategy**: Original files renamed to `.bak` extension after successful conversion, and `BackupFile()` returns the backup path. A `{backup}.out` companion file (e.g. `img.gif.bak.out`) is written alongside the backup containing the full output path, this is needed when `GetUniqueFileName` appended a counter suffix (e.g. `img_1.mp4`) because the canonical name was already taken. When `options.SkipBackup` is true, no `.bak` or `.bak.out` files are created, and the original is deleted after conversion. +- **Metadata Preservation**: After every ffmpeg conversion, `exiftool -TagsFromFile -all:all -overwrite_original` copies all source metadata to the output file. `ffmpeg -map_metadata` is not used, it is unreliable for Apple QuickTime-specific tags (e.g. `ContentIdentifier` in the `mdta`/`keys` atom). `TagsFromFile` handles cross-format date mapping, so no separate date-setting step is needed after conversion. - **Re-queue Pattern**: Converted files are added back to processing queue for validation ### Live Photo Detection -- **Short videos** (duration <= `options.ShortVideoDuration`, default `1.0s`; overridable via `--duration`): always deleted regardless of companion file +- **Short videos** (duration <= `options.ShortVideoDuration`, default `1.0s`, overridable via `--duration`): always deleted regardless of companion file - **Companion file search** (`FindCompanionImagePath()`): looks for a HEIC/JPG/JPEG file by: 1. Direct basename match (`IMG_1234.mov` -> `IMG_1234.heic`) - 2. Basename minus `_hevc` suffix (`IMG_1234_HEVC.mov` -> `IMG_1234.heic`) - new iPhone naming + 2. Basename minus `_hevc` suffix (`IMG_1234_HEVC.mov` -> `IMG_1234.heic`), the newer iPhone naming - **ContentIdentifier confirmation**: a candidate pair is only deleted when both files expose a `ContentIdentifier` tag that matches exactly. If either file lacks the tag, or the tags differ, the video is kept. There is no fallback to name-only deletion. -- **Long videos** (>= `LiveVideoDuration` = 4.0s): always kept even with a matching companion; a warning is logged +- **Long videos** (>= `LiveVideoDuration` = 4.0s): always kept even with a matching companion, and a warning is logged ### Undo Architecture (UndoTask.cs) @@ -404,9 +482,9 @@ See [`CODESTYLE.md`](../CODESTYLE.md) for build requirements, formatting command - Derived base: delete current file + all its backups - Non-derived base: delete current file if present, restore `X.bak` -> `X`; then locate the derived conversion output: if `X.bak.out` companion exists read the explicit output path from it and delete that file (handles uniquified names like `img_1.mp4`); otherwise fall back to checking whether `stem.mp4` exists and has no backup (legacy single-run heuristic) - **Internal static helpers** (testable via `InternalsVisibleTo`): - - `IsBackupFile(path)` - matches `.bak\d*$` - - `IsNumberedBackup(path)` - matches `.bak\d+$` - - `GetBackupBase(path)` - strips the `.bak\d*` suffix + - `IsBackupFile(path)`: matches `.bak\d*$` + - `IsNumberedBackup(path)`: matches `.bak\d+$` + - `GetBackupBase(path)`: strips the `.bak\d*` suffix - **Dry run**: logs all intended operations but performs no file I/O - **Known limitation**: extension renames to a previously non-existent filename create no backup and cannot be undone diff --git a/.github/workflows/build-release-task.yml b/.github/workflows/build-release-task.yml index 6866be2..ef8404a 100644 --- a/.github/workflows/build-release-task.yml +++ b/.github/workflows/build-release-task.yml @@ -103,10 +103,16 @@ jobs: assembly_file_version: ${{ needs.get-version.outputs.AssemblyFileVersion }} assembly_informational_version: ${{ needs.get-version.outputs.AssemblyInformationalVersion }} + # The terminal publish target, so it needs every other build (D4.5). + # The image builds from source and consumes no executable artifact, so nothing else forces the ordering. + # Without it a failed build-executable skips github-release while the image still pushes and `latest` moves. + # That is a partial publish: an image with no release. + # `!failure()` is what blocks it, since a failed upstream build must stop the push where a skipped one must not. + # `validate` is skipped on smoke, which is the skip that must still build. build-docker: name: Build Docker job - needs: [get-version, validate, validate-release] - if: ${{ !cancelled() && needs.get-version.result == 'success' && needs.validate-release.result == 'success' && (needs.validate.result == 'success' || needs.validate.result == 'skipped') }} + needs: [get-version, validate, validate-release, build-executable] + if: ${{ !failure() && !cancelled() && needs.get-version.result == 'success' && needs.validate-release.result == 'success' && (needs.validate.result == 'success' || needs.validate.result == 'skipped') }} uses: ./.github/workflows/build-docker-task.yml secrets: inherit with: diff --git a/.github/workflows/merge-bot-pull-request.yml b/.github/workflows/merge-bot-pull-request.yml new file mode 100644 index 0000000..4ee4f25 --- /dev/null +++ b/.github/workflows/merge-bot-pull-request.yml @@ -0,0 +1,95 @@ +name: Merge bot pull request action + +# Auto-merges in-repo bot pull requests: enable on opened/reopened, disable on a maintainer push. +# - Merge method by base: develop = squash, main = merge. +# - App token, not GITHUB_TOKEN: it fires downstream workflows and can write on a Dependabot PR. +# - pull_request_target, not pull_request: jobs hold the App key. +# The workflow and action SHAs then resolve from the trusted base, not the pull request head. +# No job checks out pull request code, since each runs gh pr merge by URL. +# +# The fleet canonical also carries merge-codegen and merge-upstream-version jobs. +# This repository runs neither codegen nor an upstream-version tracker, so both are N/A here. +# They are left out rather than carried as conditions that can never match. +on: + pull_request_target: + types: [opened, reopened, synchronize] + +# Concurrency keys on the pull request number rather than on github.ref. +# Under pull_request_target that ref is the base branch, serializing every bot pull request. +# Cancelling is off, so a follow-up synchronize cannot kill an opened run mid-enable. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: false + +jobs: + + merge-dependabot: + name: Merge dependabot pull request job + runs-on: ubuntu-latest + # Dependabot pull requests from this repository, not forks. + # Only on opened or reopened, so the disable job stays sticky. + if: >- + (github.event.action == 'opened' || github.event.action == 'reopened') && + github.event.pull_request.user.login == 'dependabot[bot]' && + github.event.pull_request.head.repo.full_name == github.repository + permissions: + contents: write + pull-requests: write + + steps: + + - name: Generate GitHub App token step + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ secrets.CODEGEN_APP_CLIENT_ID }} + private-key: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }} + + # Auto-merge every tier, semver-major included. + # The required checks are the gate, not the size of the bump. + - name: Merge pull request step + run: | + set -Eeuo pipefail + case "${{ github.event.pull_request.base.ref }}" in + develop) method=--squash ;; + main) method=--merge ;; + *) + echo "::error::Unsupported base branch: ${{ github.event.pull_request.base.ref }}" + exit 1 + ;; + esac + gh pr merge --auto "$method" "$PR_URL" + env: + PR_URL: ${{ github.event.pull_request.html_url }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + + disable-auto-merge-on-maintainer-push: + name: Disable auto-merge on maintainer push job + runs-on: ubuntu-latest + # Fires when a maintainer pushes to a bot's branch, meaning synchronize with a non-bot actor. + # Disables auto-merge so the maintainer's commits do not merge with the bot's. + # They re-enable it by hand. + # The disable call is idempotent. + if: >- + github.event.action == 'synchronize' && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login == 'dependabot[bot]' && + github.actor != github.event.pull_request.user.login + permissions: + pull-requests: write + + steps: + + - name: Generate GitHub App token step + # App token, because a Dependabot pull request's GITHUB_TOKEN is read-only whoever triggered it. + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ secrets.CODEGEN_APP_CLIENT_ID }} + private-key: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }} + + - name: Disable auto-merge step + run: gh pr merge --disable-auto "$PR_URL" + env: + PR_URL: ${{ github.event.pull_request.html_url }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index 626e12b..2c90e16 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -58,6 +58,16 @@ jobs: - name: Checkout code step uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + # The prose gate diffs against the base branch, so it needs that branch's history. + # Only the runs that reach the gate pay for it, since a publish run skips the gate. + # A full clone there is time and network a release does not need. + # The condition matches the gate's own below, so the two cannot drift apart silently. + # + # The depths are quoted, and must stay quoted. + # An unquoted 0 is falsy, so `cond && 0 || 1` collapses to 1 whenever cond is true. + # Every run would then shallow-clone, leaving the gate no base to diff against. + fetch-depth: ${{ (github.event_name == 'push' && github.ref_name != 'main') && '0' || '1' }} - name: Check C# formatting step run: | @@ -86,3 +96,26 @@ jobs: - name: Check EditorConfig step run: docker run --rm -v "$PWD":/check --workdir /check mstruebing/editorconfig-checker:latest + + # The fleet prose rules live in the hub, so this repo consumes them rather than vendoring them. + # The rules are read from hub develop, so a rule change is exercised here before promotion. + # Only lines a change touches are reported, so the existing backlog blocks nothing. + # + # The base is the branch this one merges into. + # The gate runs on push, so there is no pull_request event to read a base from. + # Runs on main are skipped, since it only receives promotion merges already gated on develop. + # + # A publish run reaches this workflow too, through build-release-task, and is skipped. + # Its content was already gated when it was pushed. + # Re-reading it would diff the whole unpromoted delta and could fail a release on prose. + - name: Check prose step + if: ${{ github.event_name == 'push' && github.ref_name != 'main' }} + # A SHA pin, containing the prose gate from ptr727/ProjectTemplate#520. + # It was hub develop when pinned, and says nothing about where develop is now. + # A pin that claims to be a branch goes stale the moment that branch moves. + # It carries no release tag yet, so Dependabot cannot compare it and will not bump it. + # Repoint it by hand at the next hub release. + # Dependabot tracks it like any other action pin from then on. + uses: ptr727/ProjectTemplate/.github/actions/prose-gate@cbd5eb3c9e079b640d5461400d89b390de2f0780 # hub develop + with: + base: origin/${{ github.ref_name == 'develop' && 'main' || 'develop' }} diff --git a/.vscode/launch.json b/.vscode/launch.json index 8925289..48d7908 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -100,6 +100,45 @@ "console": "internalConsole", "stopAtEntry": false }, + { + "name": "Verify", + "type": "coreclr", + "request": "launch", + "preLaunchTask": ".NET Build", + "program": "${workspaceFolder}/PhotoCleaner/bin/Debug/net10.0/PhotoCleaner.dll", + "args": [ + "verify", + "--path=/data/media/PhotoCleaner/Originals", + "--db=/data/media/PhotoCleaner/Verify.db", + "--rehash=false", + "--reprocess=true", + "--threads=4", + "--loglevel=verbose", + "--logfile=/data/media/PhotoCleaner/Verify.log", + "--logclear=true" + ], + "cwd": "${workspaceFolder}/PhotoCleaner/bin/Debug/net10.0", + "console": "internalConsole", + "stopAtEntry": false + }, + { + "name": "Verify No Database", + "type": "coreclr", + "request": "launch", + "preLaunchTask": ".NET Build", + "program": "${workspaceFolder}/PhotoCleaner/bin/Debug/net10.0/PhotoCleaner.dll", + "args": [ + "verify", + "--path=/data/media/PhotoCleaner/Originals", + "--threads=4", + "--loglevel=verbose", + "--logfile=/data/media/PhotoCleaner/Verify.log", + "--logclear=true" + ], + "cwd": "${workspaceFolder}/PhotoCleaner/bin/Debug/net10.0", + "console": "internalConsole", + "stopAtEntry": false + }, { "name": "Undo", "type": "coreclr", diff --git a/AGENTS.md b/AGENTS.md index 27e033c..7e34d22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ An agent session is billed on the context it carries, not the work it does. Ever ### Session Scope - **One deliverable, one session.** A session covers one branch and one deliverable, and ends when that work merges. A multi-step task is one deliverable and stays in one session. Two unrelated tasks are two sessions even when they run back to back. -- **End a session at any of these, without being asked:** the branch changes, the pull request merges, the next task is unrelated to the last, or a third review round opens on the same pull request. +- **End a session at any of these, without being asked:** the branch changes, the pull request merges, or the next task is unrelated to the last. A review round is none of them. A loop still producing findings is the deliverable in progress, and a round count is not a reason to leave one open. - **Hand off in a file, never in context.** Close a session by writing at most 2 KB to a scratch file: branch, pull request link, what is done, the next command. A summary held in context is re-billed until the session ends, and a summary on disk is read once by whoever needs it. - **Re-derive state, do not carry it.** "This session already has the context" is the signal to split, not to continue. Context that has gone stale is worse than absent, because a file read hundreds of requests ago no longer describes the file. - **Compaction is a fallback, not the strategy.** It restarts context from a floor and climbs again, where a fresh session starts from zero. @@ -45,6 +45,7 @@ If a rule you were given does not cover what you find, stop and report it. Do no ``` - **Wait in a background process, not in a poll loop.** A review or CI wait is a sequence of near-identical requests, each billed for whatever context it happens to carry. Run the wait as one backgrounded command that returns when the condition is met. +- **A wait separates three outcomes, and says which one it reached.** The condition was met, it has not been met yet, and the wait cannot reach it at all are three different results, and a backgrounded wait that emits nothing renders all three identically. Run the command once in the foreground and read its output before backgrounding it, because a wait is only as good as the command inside it, and an unsupported flag on the installed tool version exits non-zero with an empty stdout that every naive test reads as "nothing yet". Never let a fallback stand in for a failed command, since `|| echo '[]'`, `|| true`, and `2>/dev/null` convert an error into that same reading, which is the suppression the write-safety rules already forbid on a mutation. Make the wait emit on failure as loudly as on success, so silence means "still running" and nothing else, and bound it, so a condition that is never coming ends in a report rather than in another wait. ## Where the Rules Live @@ -55,6 +56,7 @@ Every rule below is a level-two section of [`GOVERNANCE.md`](./GOVERNANCE.md). R | Why the rules are shaped this way | `Foundational Principles` | | Recording a durable lesson or updating governance | `Durable Knowledge and Self-Improvement` | | Any push, API mutation, comment, label, or merge | `Repository Boundaries and Write Safety` | +| Quoting data into a comment, commit, test, or doc | `Representative Data in Agent-Authored Text` | | Committing, signing, rebasing, force-pushing | `Git and Commit Rules` | | Branch choice, promotion, keeping branches in sync | `Branching Model` | | Releasing, version bumps, publishing | `Release Model` | diff --git a/AUDIT.md b/AUDIT.md index 4d3b68d..ffbeb7b 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -6,7 +6,7 @@ The audit is read-only: it diffs live state against the committed baseline and r ## Scope -This is a release-model repo: the self-audit covers the `main` and `develop` rulesets, general repository settings, and secret names. Code-project conformance (analyzers, tests, coverage, publish workflows) is CI's job and the fleet hub's fleet-wide audit's, not this self-audit's - see [GOVERNANCE.md "Branching Model"][governance-branching-model] for the model this baseline encodes. +This is a release-model repo: the self-audit covers the `main` and `develop` rulesets, general repository settings, and secret names. Code-project conformance (analyzers, tests, coverage, publish workflows) is CI's job and the fleet hub's fleet-wide audit's, not this self-audit's. See [GOVERNANCE.md "Branching Model"][governance-branching-model] for the model this baseline encodes. ## General Settings @@ -35,7 +35,7 @@ for b in develop main; do done ``` -The result must be exactly two rulesets named `develop` and `main` - a missing ruleset or a divergent payload is a **defect**; a duplicate or stray ruleset is a **drift finding**. +The result must be exactly two rulesets named `develop` and `main`. A missing ruleset or a divergent payload is a **defect**, and a duplicate or stray ruleset is a **drift finding**. ## Secrets diff --git a/CODESTYLE.md b/CODESTYLE.md index 85e16f1..a874791 100644 --- a/CODESTYLE.md +++ b/CODESTYLE.md @@ -1,8 +1,8 @@ # Code Style and Formatting Rules -This is the single code-style guide for the fleet. The **General** section applies to every language. Each **language section** (.NET, Python) is self-contained: a repo follows only the section(s) for the languages it ships and ignores the rest. A repo keeps the whole file rather than trimming it - an unused-language section costs nothing, the same whole-file model as [`.editorconfig`][root], whose inert `[*.cs]` block a non-.NET repo keeps. +This is the single code-style guide for the fleet. The **General** section applies to every language. Each **language section** (.NET, Python) is self-contained: a repo follows only the section(s) for the languages it ships and ignores the rest. A repo keeps the whole file rather than trimming it. An unused-language section costs nothing, the same whole-file model as [`.editorconfig`][root], whose inert `[*.cs]` block a non-.NET repo keeps. -Cross-cutting *process* rules (PR titles, branching, US English, markdown style, comments philosophy, workflow YAML, PR review etiquette) live in [GOVERNANCE.md][governance] and are not repeated here. +Cross-cutting *process* rules (PR titles, branching, US English, markdown style, comments philosophy, workflow YAML, PR review etiquette, and the verification discipline that defines the pre-push lint gate) live in [GOVERNANCE.md][governance] and are not repeated here. ## General @@ -10,21 +10,21 @@ These rules apply to every language in the repo. ### Tooling Names and Casing -Use each tool's official casing in task labels, docs, and prose - `.NET` (not `.Net`), `CSharpier`, `ruff`, `pyright`, `uv`. Don't invent personal variants. +Use each tool's official casing in task labels, docs, and prose: `.NET` (not `.Net`), `CSharpier`, `ruff`, `pyright`, `uv`. Don't invent personal variants. ### Clean-Compile Verification -Each language defines a **clean-compile** verification - the combination of build, formatter, linter, and code-analysis tools that must report clean before a commit. It is exposed as one or more **named** VS Code tasks (or, where a language ships no tasks, documented commands), and those definitions are the same across the fleet. The concrete names live in each language section below. +Each language defines a **clean-compile** verification: the combination of build, formatter, linter, and code-analysis tools that must report clean before a commit. It is exposed as one or more **named** VS Code tasks (or, where a language ships no tasks, documented commands), and those definitions are the same across the fleet. The concrete names live in each language section below. -- **Run it after every code change.** The relevant language's clean-compile must pass before you commit; CI runs the same checks as a backstop. -- **The named task definition is the canonical spec** - its exact command sequence, arguments, and strictness. You may run it through the VS Code task **or** by invoking the equivalent native commands directly; either is fine **only if the sequence, arguments, and strictness match exactly**. No shortcuts and no more-lenient options (for example, never drop `--verify-no-changes` or loosen a `--severity`). -- **A local commit/pre-commit gate is the repo's choice.** No single hook runner fits every language (a `dotnet`-tool runner like Husky.Net suits .NET but not Python), so none is mandated - but that is **not** a recommendation against commit gates. CI is the authoritative backstop regardless; a local gate is an additive convenience a repo may wire and keep - Husky.Net (and `dotnet husky run` as a style step) for .NET, `pre-commit` for Python. Keeping a working gate is not drift. +- **Run it after every code change, and it is not the whole gate.** The relevant language's clean-compile must pass before you commit. CI runs those same language checks as a backstop **plus everything else its validation workflow runs**, and all of it reports into the one required status, so a green clean-compile does not predict a green CI. That remainder is at least the doc-lint set (markdownlint, cspell, actionlint, `editorconfig-checker`) and whatever spec, config, and script gates the repo carries, so read the workflow for the full list rather than assuming this sentence enumerates it. What has to pass before a push is the repo's **whole** lint gate, per [GOVERNANCE.md "Verification Discipline"][governance-verification-discipline]. Each linter's known-working invocation is in [GOVERNANCE.md "Running the Linters Locally"][governance-running-the-linters-locally]. +- **The named task definition is the canonical spec** - its exact command sequence, arguments, and strictness. You may run it through the VS Code task **or** by invoking the equivalent native commands directly, and either is fine **only if the sequence, arguments, and strictness match exactly**. No shortcuts and no more-lenient options (for example, never drop `--verify-no-changes` or loosen a `--severity`). +- **A local commit/pre-commit gate is the repo's choice.** No single hook runner fits every language (a `dotnet`-tool runner like Husky.Net suits .NET but not Python), so none is mandated, but that is **not** a recommendation against commit gates. CI is the authoritative backstop regardless, and a local gate is an additive convenience a repo may wire and keep: Husky.Net (and `dotnet husky run` as a style step) for .NET, `pre-commit` for Python. Keeping a working gate is not drift. ### Analyzer Diagnostics and Suppressions -- **A new port is not a license to silence diagnostics.** Brownfield / just-ported status never justifies relaxing analyzer or linter severities or muting newly surfaced warnings - fix them. (The only brownfield allowance is the one-time git-signing / line-ending migration described in [GOVERNANCE.md][governance] and [README.md][readme], which has nothing to do with code analysis.) +- **A new port is not a license to silence diagnostics.** Brownfield / just-ported status never justifies relaxing analyzer or linter severities or muting newly surfaced warnings. Fix them. (The only brownfield allowance is the one-time git-signing / line-ending migration described in [GOVERNANCE.md][governance] and [README.md][readme], which has nothing to do with code analysis.) - **Suppress only genuine false-positives or deliberate, documented exceptions**, always at the **narrowest scope that fits**, in this order of preference: - 1. An **in-code annotation on the specific symbol**, with a justification - the language's attribute/comment form, never a blanket pragma spanning a region. + 1. An **in-code annotation on the specific symbol**, with a justification, in the language's attribute/comment form, never a blanket pragma spanning a region. 2. The **owning project's local config** when the exception is project-wide for one project (e.g. a test project's own `.editorconfig` / `pyproject.toml`). 3. The **root / shared config** only when the suppression is genuinely applicable to **every** project in the repo. - **Never blanket-relax a batch of rules project-wide** to get a port to build. The per-language mechanics (which attribute, which config key) are in each language section. @@ -33,10 +33,10 @@ Each language defines a **clean-compile** verification - the combination of buil These apply repo-wide, in every directory: -1. **Markdown linting**: All `.md` files must be lint-clean (error and warning free) via the VS Code `markdownlint` extension. [`.markdownlint-cli2.jsonc`][markdownlint-cli2] at the repo root is the single source of truth - the davidanson `markdownlint` extension and a command-line `markdownlint-cli2` run both read it, so the IDE and CLI stay in lock-step. Rules it deliberately disables (e.g. `MD013` line-length) are **intentional** - do not "fix" them. `MD033` inline HTML stays **enabled**: HTML comments are permitted (markdownlint does not flag them), HTML elements are flagged, and anything with a native markdown equivalent uses the markdown. Fix violations at the source rather than disabling rules. -2. **Spelling**: All spelling must be clean via the CSpell VS Code integration; words must be correctly spelled in **US English** (the repo-wide convention - see [GOVERNANCE.md][governance]). The shared `cspell.json` sets `"language": "en-US"` so British spellings are flagged - a bare `"en"` accepts both US and British and silently passes the wrong spelling. Project-specific terms go in the shared `cspell.json` `words` list - it is the single source of truth the extension, CLI, and CI all read. The `.code-workspace` must **not** carry its own `cspell.words`/`cSpell.words` block; when externalizing words into `cspell.json`, delete any word list left in the workspace (a leftover one duplicates the list and silently drifts). -3. **Spelling CI scope**: The enforced CI spell-check gate covers **`README.md` and `HISTORY.md` only** - these are the files every repo visitor sees, so they must be clean. It is deliberately **not** all `**/*.md`: repos carry many markdown files full of technical terms, and gating every one of them would mean endlessly padding `cspell.json` just to keep CI green. Broad, live spell-checking across any file (source, markdown, text) is the **cspell editor extension's** job, so typos still surface to whoever is editing. A repo owner **may** widen their own CI file list, but README + HISTORY are the default; keep the CI workflow, the `Lint: Spelling` VS Code task, and the GOVERNANCE.md cspell one-liner on the same file list. The list is explicit (not a glob), so a repo that ships no `HISTORY.md` (e.g. one with no changelog) must drop it from all three surfaces and gate on `README.md` alone - cspell errors on a listed file that does not exist. Markdown *linting* (item 1) stays repo-wide `**/*.md` - it does not choke on technical terms. -4. **`README.md` and `HISTORY.md` share the same header**: the `# PhotoCleaner` title and the one-line description under it match exactly in both files, so a reader landing on the changelog sees the same project identity. A change to one is applied to the other in the same commit, and the description is also what the GitHub About panel and the Docker Hub short description carry (see [GOVERNANCE.md][governance] "Repository Details"). +1. **Markdown linting**: All `.md` files must be lint-clean (error and warning free) via the VS Code `markdownlint` extension. [`.markdownlint-cli2.jsonc`][markdownlint-cli2] at the repo root is the single source of truth, and the davidanson `markdownlint` extension and a command-line `markdownlint-cli2` run both read it, so the IDE and CLI stay in lock-step. Rules it deliberately disables (e.g. `MD013` line-length) are **intentional**, so do not "fix" them. `MD033` inline HTML stays **enabled**: HTML comments are permitted (markdownlint does not flag them), `details` and `summary` are allowed because a GitHub collapsible has no markdown equivalent, every other element is flagged, and anything with a native markdown equivalent uses the markdown. Fix violations at the source rather than disabling rules. +2. **Spelling**: All spelling must be clean via the CSpell VS Code integration, and words must be correctly spelled in **US English** (the repo-wide convention, per [GOVERNANCE.md][governance]). The shared `cspell.json` sets `"language": "en-US"` so British spellings are flagged, where a bare `"en"` accepts both US and British and silently passes the wrong spelling. Project-specific terms go in the shared `cspell.json` `words` list, the single source of truth the extension, CLI, and CI all read. The `.code-workspace` must **not** carry its own `cspell.words`/`cSpell.words` block, and when externalizing words into `cspell.json`, delete any word list left in the workspace (a leftover one duplicates the list and silently drifts). +3. **Spelling CI scope**: The enforced CI spell-check gate covers **`README.md` and `HISTORY.md` only**, because these are the files every repo visitor sees, so they must be clean. It is deliberately **not** all `**/*.md`: repos carry many markdown files full of technical terms, and gating every one of them would mean endlessly padding `cspell.json` just to keep CI green. Broad, live spell-checking across any file (source, markdown, text) is the **cspell editor extension's** job, so typos still surface to whoever is editing. A repo owner **may** widen their own CI file list, but README + HISTORY are the default; keep the CI workflow, the `Lint: Spelling` VS Code task, and the GOVERNANCE.md cspell one-liner on the same file list. The list is explicit (not a glob), so a repo that ships no `HISTORY.md` (e.g. one with no changelog) must drop it from all three surfaces and gate on `README.md` alone, since cspell errors on a listed file that does not exist. Markdown *linting* (item 1) stays repo-wide `**/*.md`, which does not choke on technical terms. +4. **`HISTORY.md` mirrors the README opening**: `HISTORY.md` is the maintainer-curated changelog and opens as the README's twin, carrying the same `# ` (without the README's ToC-omit comment) and the same intro paragraph copied verbatim, then a `## Release History` section. The mirrored opening keeps the project identity consistent for a reader who lands on the changelog directly. The audit checks that the title and intro match the README, with HTML comments stripped. ## .NET @@ -52,15 +52,15 @@ This is the style guide for any **.NET projects** in this repo. 1. **The `.NET Format` clean-compile task** (see [Clean-Compile Verification][clean-compile-verification]) - The .NET clean-compile is the **`.NET Format`** VS Code task, which chains `CSharpier Format` -> `.NET Build` -> `dotnet format style --verify-no-changes`. These three task definitions are carried verbatim in [`.vscode/tasks.json`][vscode-tasks]. - - After any code change it must pass before commit. Run the `.NET Format` task. To run it natively instead, reproduce that task chain from [`.vscode/tasks.json`][vscode-tasks] exactly - `CSharpier Format`, then `.NET Build`, then the `dotnet format style --verify-no-changes --severity=info ...` verify - without dropping or loosening any argument (tasks.json is the canonical command spec). Bare `dotnet format` alone, skipping CSharpier or the build, is not sufficient. + - After any code change it must pass before commit. Run the `.NET Format` task. To run it natively instead, reproduce that task chain from [`.vscode/tasks.json`][vscode-tasks] exactly (`CSharpier Format`, then `.NET Build`, then the `dotnet format style --verify-no-changes --severity=info ...` verify) without dropping or loosening any argument (tasks.json is the canonical command spec). Bare `dotnet format` alone, skipping CSharpier or the build, is not sufficient. 2. **Analyzer configuration** - `<EnableNETAnalyzers>true</EnableNETAnalyzers>` with `<AnalysisLevel>latest-all</AnalysisLevel>` and `<AnalysisMode>All</AnalysisMode>` (full analyzer set enabled) - - `<TreatWarningsAsErrors>true</TreatWarningsAsErrors>` - any diagnostic surfaced as a warning fails the build, so it must be fixed or deliberately suppressed, not left to accumulate (see [Analyzer Diagnostics and Suppressions][analyzer-diagnostics-and-suppressions]) + - `<TreatWarningsAsErrors>true</TreatWarningsAsErrors>`, so any diagnostic surfaced as a warning fails the build and must be fixed or deliberately suppressed, not left to accumulate (see [Analyzer Diagnostics and Suppressions][analyzer-diagnostics-and-suppressions]) 3. **CI lint backstop** - CI runs the clean-compile checks on every PR as the authoritative backstop - - Git hooks are optional; a repo may wire a local runner (Husky.Net) for pre-commit enforcement, but CI is the gate that matters + - Git hooks are optional, and a repo may wire a local runner (Husky.Net) for pre-commit enforcement, but CI is the gate that matters #### The Full Post-Change Set @@ -74,14 +74,14 @@ The clean-compile task above is necessary and not sufficient. After every code c Shared MSBuild configuration is centralized at the repository root, never duplicated per project: -- **`Directory.Build.props`** carries the properties every project shares - the analyzer set and `TreatWarningsAsErrors` from the Zero Warnings Policy above, plus `LangVersion`, `TargetFramework` where uniform, and any repo-wide build metadata. A csproj carries only what is genuinely project-specific (`OutputType`, `IsPackable`, project references). -- **`Directory.Packages.props`** enables central package management (`ManagePackageVersionsCentrally` true): every dependency version is declared once as a `PackageVersion` item, and a csproj's `PackageReference` items are versionless. One file to review on a bump, one Dependabot surface, and no version skew between projects. +- **`Directory.Build.props`** carries the properties every project shares: the analyzer set and `TreatWarningsAsErrors` from the Zero Warnings Policy above, plus `LangVersion`, `TargetFramework` where uniform, and any repo-wide build metadata. A csproj carries only what is genuinely project-specific (`OutputType`, `IsPackable`, project references). +- **`Directory.Packages.props`** owns central package management: it sets `ManagePackageVersionsCentrally` to `true` (in this file, not `Directory.Build.props`) and declares every dependency version once as a `PackageVersion` item, so a csproj's `PackageReference` items are versionless. One file to review on a bump, one Dependabot surface, and no version skew between projects. -A repo whose projects still carry per-project analyzer settings or versioned `PackageReference` items is drifted - move the shared property or version up to the root file rather than editing it in place. +A repo whose projects still carry per-project analyzer settings or versioned `PackageReference` items is drifted, so move the shared property or version up to the root file rather than editing it in place. #### Build Tasks -Available VS Code tasks (run them from VS Code's task runner - **Terminal -> Run Task** - or an agent's task-running tool). The three clean-compile tasks below are carried verbatim; a repo adds its own convenience tasks (tool updates, dependency upgrades, benchmarks) on top: +Available VS Code tasks (run them from VS Code's task runner, **Terminal -> Run Task**, or an agent's task-running tool). The three clean-compile tasks below are carried verbatim, and a repo adds its own convenience tasks (tool updates, dependency upgrades, benchmarks) on top: - `.NET Build`: Build with diagnostic verbosity *(clean-compile)* - `CSharpier Format`: Auto-format code with CSharpier *(clean-compile)* @@ -99,7 +99,7 @@ Available VS Code tasks (run them from VS Code's task runner - **Terminal -> Run - `dotnet-outdated-tool`: Dependency update checks - Nerdbank.GitVersioning: Version management -CI is the authoritative lint backstop. Local pre-commit hooks are optional - wire Husky.Net (or another runner) if you want local enforcement. +CI is the authoritative lint backstop. Local pre-commit hooks are optional, so wire Husky.Net (or another runner) if you want local enforcement. #### Editor Baseline @@ -127,7 +127,7 @@ Note: Code snippets are illustrative examples only. Replace namespaces/types to - Top-level statements for console apps - Pattern matching over traditional checks - Collection expressions when types loosely match - - Extension methods - the classic `this`-parameter form, or an `extension(<receiver>) { ... }` block on C# 14+ + - Extension methods, in the classic `this`-parameter form or an `extension(<receiver>) { ... }` block on C# 14+ - Implicit object creation when type is apparent - Range and index operators @@ -206,7 +206,7 @@ Note: Code snippets are illustrative examples only. Replace namespaces/types to - YAML files: 2 spaces - JSON files: 4 spaces -5. **Line endings**: not specified here - governed per repo by `.editorconfig` / `.gitattributes` per the [GOVERNANCE.md][governance] "Line Endings" section. +5. **Line endings**: not specified here, but governed per repo by `.editorconfig` / `.gitattributes` per the [GOVERNANCE.md][governance] "Line Endings" section. 6. **`#region`**: Do not use regions. Prefer logical file/folder/namespace organization. 7. **Member ordering (StyleCop SA1201)**: const -> static readonly -> static fields -> instance readonly fields -> instance fields -> constructors -> public (events -> properties -> indexers -> methods -> operators) -> non-public in same order -> nested types @@ -262,13 +262,13 @@ Follow the scope hierarchy in [Analyzer Diagnostics and Suppressions][analyzer-d #### Error Handling and Logging -1. **Structured logging**: Use structured message templates - Serilog is the **application's** concrete backend; a library never references it (see item 2) +1. **Structured logging**: Use structured message templates. Serilog is the **application's** concrete backend, and a library never references it (see item 2) ```csharp logger.LogError(exception, "{Function}", function); ``` -2. **Libraries log through abstractions, never a concrete backend.** A NuGet **library** depends only on `Microsoft.Extensions.Logging.Abstractions` and exposes an `ILoggerFactory` seam - a settable global factory defaulting to `NullLoggerFactory.Instance` (fallback `NullLogger.Instance`) with `SetFactory`/`TrySetFactory`, and/or an `ILoggerFactory`/`ILogger` parameter in its API. It must **not** reference Serilog or any sink - that forces a logging framework on every consumer and drags in AOT-incompatible dependencies. The consuming **application** owns the concrete logger (Serilog is fine there), bridges it to `ILoggerFactory` (e.g. `SerilogLoggerFactory` from `Serilog.Extensions.Logging`), and injects it. Reference pattern: a `LogOptions` seam in the library; the consuming CLI builds the Serilog-backed factory and injects it via `LogOptions.SetFactory`. +2. **Libraries log through abstractions, never a concrete backend.** A NuGet **library** depends only on `Microsoft.Extensions.Logging.Abstractions` and exposes an `ILoggerFactory` seam: a settable global factory defaulting to `NullLoggerFactory.Instance` (fallback `NullLogger.Instance`) with `SetFactory`/`TrySetFactory`, and/or an `ILoggerFactory`/`ILogger` parameter in its API. It must **not** reference Serilog or any sink, which would force a logging framework on every consumer and drag in AOT-incompatible dependencies. The consuming **application** owns the concrete logger (Serilog is fine there), bridges it to `ILoggerFactory` (e.g. `SerilogLoggerFactory` from `Serilog.Extensions.Logging`), and injects it. Reference pattern: a `LogOptions` seam in the library; the consuming CLI builds the Serilog-backed factory and injects it via `LogOptions.SetFactory`. 3. **CallerMemberName**: Use for automatic function name tracking @@ -288,20 +288,20 @@ Follow the scope hierarchy in [Analyzer Diagnostics and Suppressions][analyzer-d } ``` -5. **Exceptions**: Do not swallow exceptions; log and rethrow or translate to a domain-specific exception +5. **Exceptions**: Do not swallow exceptions, and either log and rethrow or translate to a domain-specific exception #### Code Patterns 1. **Guard clauses**: Prefer early returns for validation and error handling -2. **Async all the way**: Avoid blocking calls (`.Result`, `.Wait()`); use `async`/`await` +2. **Async all the way**: Avoid blocking calls (`.Result`, `.Wait()`) and use `async`/`await` 3. **Cancellation tokens**: Accept `CancellationToken` as the last parameter and pass it through 4. **ConfigureAwait**: In library code, use `ConfigureAwait(false)` unless context is required - Do not call `ConfigureAwait(false)` in xUnit tests (see xUnit1030) -5. **Disposables**: Use `await using` for async disposables; prefer `using` declarations +5. **Disposables**: Use `await using` for async disposables, and prefer `using` declarations 6. **LINQ vs loops**: Use LINQ for clarity, loops for hot paths or allocations -7. **HTTP**: Reuse `HttpClient` via factory; avoid per-request instantiation +7. **HTTP**: Reuse `HttpClient` via factory, never per-request instantiation 8. **Collections**: Prefer `IReadOnlyList<T>`/`IReadOnlyCollection<T>` for public APIs -9. **Immutability**: Prefer immutable records; use init-only setters when records are not suitable; prefer immutable or frozen collections for read-only data +9. **Immutability**: Prefer immutable records, use init-only setters when records are not suitable, and prefer immutable or frozen collections for read-only data 10. **Exceptions as control flow**: Avoid using exceptions for expected flow 11. **Sealing classes**: Seal classes that are not designed for inheritance 12. **Read-only data**: Use immutable or frozen collections for read-only data sets @@ -309,7 +309,7 @@ Follow the scope hierarchy in [Analyzer Diagnostics and Suppressions][analyzer-d #### Testing Conventions -1. **Framework**: **xUnit v3 or later** (the `xunit.v3` package, never the legacy v2 `xunit` package) with **AwesomeAssertions** for every assertion. Native xUnit asserts (`Assert.Equal`, `Assert.True`, ...) are not allowed - use the fluent `.Should()` API. Dynamic test skipping (`Assert.Skip`, `Assert.SkipWhen`) is control flow, not an assertion, and stays native. +1. **Framework**: **xUnit v3 or later** (the `xunit.v3` package, never the legacy v2 `xunit` package) with **AwesomeAssertions** for every assertion. Native xUnit asserts (`Assert.Equal`, `Assert.True`, ...) are not allowed, so use the fluent `.Should()` API. Dynamic test skipping (`Assert.Skip`, `Assert.SkipWhen`) is control flow, not an assertion, and stays native. ```csharp [Fact] @@ -356,12 +356,167 @@ Follow the scope hierarchy in [Analyzer Diagnostics and Suppressions][analyzer-d 1. **Code reviews**: All changes go through pull requests +## Python + +*This section applies only to the Python side. A repo with no Python projects still carries it (the file is carried whole) and ignores it.* + +This is the style guide for any **Python project(s)** in this repo. + +**Adapt before propagating.** The rules below describe the default Python profile: a package that publishes to PyPI, type-checked by `pyright` in strict mode, dependencies in `[dependency-groups]`. A derived repo often differs; when it does, **adapt these fields to match the repo's actual toolchain rather than copying verbatim** (a verbatim copy that misdescribes the repo is inaccurate and gets rejected in review). The axes that commonly vary per repo: + +- **Type checker in CI** - `pyright` strict, **`mypy` in CI with `pyright` editor-only** (Pylance), or both. Whichever runs in CI is the one the clean-compile and the CI gate invoke. +- **Dependency declaration** - `[dependency-groups]`, or PEP 621 `[project.optional-dependencies]` (dev tools installed with `uv sync --extra <group>`). +- **Versioning / publishing** - a published package (`_version.py` + a version source + `uv build` + a PyPI publish step), or a **source-only** repo with a static `version` and no publish step (see [Versioning][versioning-section]). +- **Disabled markdownlint rules** - repo-specific. `.markdownlint-cli2.jsonc` at the repo root is the source of truth, not any example rule named here. +- **VS Code config home** - editor **settings/extensions** may live in `.vscode/*.json` **or** the `<Repo>.code-workspace`, while **tasks / launch / debug** configs can only be external `.vscode/*.json` (they cannot live in the workspace file). A `[vscode-tasks]` reference must point wherever the repo actually keeps `tasks.json`. + +**Two profiles.** A repo's Python is one of two shapes, declared as the `build` or `lint-only` profile and validated against the `pyproject.toml` shape. The rest of this section (uv project, `uv.lock`, `uv run`, `src` layout, pytest coverage) describes the **Project** shape (the `build` profile). The two differ by whether the Python has **third-party runtime dependencies**, which shows up structurally in `pyproject.toml`, so the audit reads the shape there (`python.profile.detect`): + +- **Project** (the `build` profile): the Python has third-party runtime dependencies, or is the repo's deliverable. It is a PEP 621 uv project: `[project]` with `dependencies` (dev tools in `[project.optional-dependencies]` or `[dependency-groups]`), a `[build-system]`, and a committed `uv.lock` (pinned LF, per [Line Endings][line-endings]). CI runs `uv sync --frozen` + `uv run <tool>`, so the lockfile pins tool versions. +- **Scripts** (the `lint-only` profile): stdlib-only utility scripts embedded in a **non-Python** repo (e.g. a Python tooling subtree of a `csharp` app). Run the tools with **`uvx`** (no project install, no lockfile): the `pyproject.toml` carries **only** tool config (`[tool.ruff]`, `[tool.mypy]`, and an optional `[tool.pyright]` editor block), with no `[project]`, no `[build-system]`, and no `uv.lock` (that metadata would misrepresent it as a shippable package). **mypy** is the type-check gate (there is no first-party package for pyright strict to anchor on), and a `[tool.pyright]` block in **standard** mode keeps Pylance quiet in the editor, the same mypy-gate/pyright-editor split the build profile uses. There is no lockfile, and a `uvx <tool>@<ver>` pin in a `run:` step is not something Dependabot tracks, so **CI runs `uvx ruff@latest` / `uvx mypy@latest`** rather than a manual pin that would silently go stale. The fleet rule is to pin only what Dependabot auto-updates (SHA-pinned actions, package deps) and otherwise run latest, so the VS Code tasks, README, and CI all run the unpinned latest here. `.py` files follow the repo's line-ending default (CRLF in a CRLF-default repo, and a shebang-executed script is LF-pinned by path, per [Line Endings][line-endings]). There is no pytest suite and no coverage gate. A script that carries a gate still earns tests, written with the standard library's `unittest` so they run under bare `python3` with nothing installed, as `test_<script>.py` beside the script it exercises; measure them with `uvx coverage@latest run -m unittest discover -s <dir>` when a number is wanted, without adopting a threshold. A co-present `csharp` type still carries `codecov.yml` for its own tests. + +### Toolchain + +| Tool | Role | Config | +|---|---|---| +| [uv][uv-link] | env, deps, build, publish (build/publish only where the repo ships a package) | `pyproject.toml` `[dependency-groups]` or `[project.optional-dependencies]`, `uv.lock` | +| [hatchling][latest-link] | build backend (published packages) | `pyproject.toml` `[build-system]` | +| [ruff][ruff-link] | lint + format + import sort | `pyproject.toml` `[tool.ruff]` | +| [pyright][pyright-link] | type checker (the default, a strict baseline) | `pyproject.toml` `[tool.pyright]` | +| [mypy][mypy-link] | additional/alternate type checker (optional, the CI checker in a mypy-in-CI repo, required for Home Assistant) | `pyproject.toml` `[tool.mypy]` (or per home-assistant/core) | +| [pytest][docs-link] | test runner | `pyproject.toml` `[tool.pytest.ini_options]` | + +**Type checking targets strongly typed, deterministic code.** `pyright` in **strict** mode is the default baseline on first-party code (a repo may instead run `mypy` in CI and keep `pyright` editor-only via Pylance, per the next paragraph) (`[tool.pyright]` `strict = ["src"]`, or the integration package for a Home Assistant repo, with tests run in standard mode). pyright is the anchor because **Pylance embeds it**, so the editor and the CLI/CI (`uv run pyright`) run the *same* engine and never disagree. The standalone `ms-pyright.pyright` extension stays in `unwantedRecommendations` because Pylance covers it. Relax strictness on **third-party** code only when a dependency has no usable types and no alternative (e.g. `pandas`): a targeted, commented `# pyright: ignore[...]` or a scoped `[tool.pyright]` override, never a blanket relaxation. + +**`mypy` is allowed, and required where the ecosystem demands it. It is not banned.** Running more than one checker is normal when each serves a purpose (the .NET side pairs `CSharpier` and `dotnet format` the same way), and pyright's inference and mypy's plugin ecosystem (e.g. `pydantic.mypy`) catch different classes of error. A **Home Assistant** integration runs `mypy --strict` because the platinum `strict-typing` quality-scale tier requires it; a pydantic-heavy library may opt in for the plugin. When a repo uses mypy it runs in **CI and the editor** (the `ms-python.mypy-type-checker` extension) so the two stay consistent, and its mypy command joins the clean-compile; a repo with no such need stays pyright-only, which is lighter and inherently consistent. + +### Local Development Loop + +From inside the Python project directory: + +```sh +uv sync # creates .venv, installs deps + dev group +uv run ruff format # auto-format +uv run ruff check --fix # auto-fix lint +uv run ruff check # verify lint clean +uv run ruff format --check # verify format clean +uv run pyright # verify types +uv run pytest # run tests +uv build # produce wheel + sdist in ./dist (published packages only) +``` + +The Python clean-compile (see [Clean-Compile Verification][clean-compile-verification]) is `uv run ruff format` + `uv run ruff check` + the repo's type checker: `uv run pyright`, or `uv run mypy src` where mypy is the CI checker, or both where the repo runs both (see Type checking above); run it (plus `uv run pytest`) before committing. These are documented commands; an optional VS Code tasks mirror (all `type: process`, no `&&` shell chaining, so it runs the same on any task shell) belongs in the repo's own `.vscode/tasks.json`. CI runs the same clean-compile commands as the authoritative backstop. Git hooks are opt-in; wire `pre-commit` for `ruff` and the type checker yourself if you want local enforcement. + +### Layout + +`src` layout, which keeps the package out of the repo root and prevents accidental imports of unbuilt code: + +```text +<python-project>/ + pyproject.toml + README.md + uv.lock # committed for reproducible CI + src/ + <package_name>/ + __init__.py + _version.py # published packages; a source-only repo uses a static version instead + <modules>.py + tests/ + __init__.py + test_<module>.py +``` + +### Code Style + +#### Formatting and Linting + +- **`ruff format` is authoritative.** Don't argue with the formatter, and if it reformats your code, that's the final form. Configure (line length, target version) in `pyproject.toml` `[tool.ruff]`, not via inline `# fmt:` directives. +- **Run `ruff check --fix` before committing.** Most ruff lint rules have safe autofixes, so let the tool handle them. The configured rule families are listed under `[tool.ruff.lint]` `select`. Add new rule families project-wide rather than scattering inline `# noqa` markers. +- **`# noqa` is a last resort.** When you must use one, scope it narrowly (`# noqa: E501`, not bare `# noqa`) and add a short comment on the same line explaining why. False-positive patterns that recur across the codebase belong in `[tool.ruff.lint]` `ignore` or per-file `[tool.ruff.lint.per-file-ignores]`, with a comment. Porting an existing codebase is not a license to add `ignore` / `per-file-ignores` blocks to mute newly surfaced lint. Fix it (see [Analyzer Diagnostics and Suppressions][analyzer-diagnostics-and-suppressions]). + +#### Comments + +- **Inline `#` comments**: keep tight and local. One line is preferred, but multi-line is fine when you need to document a non-obvious implementation constraint, a local trade-off, or coupling that future edits could easily break. Keep that rationale next to the affected block so the reviewer/maintainer sees it at edit-time. +- **Don't explain *what* the code does.** Well-named identifiers handle that. Don't reference the current task ("added for X", "used by Y"), which belongs in the PR description. + +#### Docstrings + +- Follow [PEP 257][pep-0257-link]. Focus docstrings primarily on the **behavior contract** (what callers and tests can rely on), public semantics, and edge-case expectations. Implementation-local rationale belongs in inline `#` comments, not docstrings. +- A short one-liner is fine for trivial functions and tests with self-documenting names. +- For non-trivial behavior (non-obvious test scenarios, contracts a test pins, edge cases callers must know about, design trade-offs that are load-bearing for future maintainers), write a one-line summary, blank line, then a details paragraph. Multi-paragraph docstrings are fine when the contract earns it. +- Design notes belong **in the code** (docstrings or inline comments). They do NOT belong in [`HISTORY.md`][history], which is end-user release notes, not a design log. + +#### Type Hints + +- **All public APIs are typed.** The repo's configured type checker runs on `src/` (pyright strict via `[tool.pyright]` `strict = ["src"]`, or `mypy` where that is the CI checker), and tests run in the checker's looser/standard mode. +- **Use modern syntax**: `list[int]` not `List[int]`, `dict[str, X]` not `Dict[str, X]`, `X | None` not `Optional[X]`, `from __future__ import annotations` only when needed for forward references. +- **Don't add `# type: ignore` to silence pyright errors without a comment** explaining the constraint. If a recurring false positive needs suppression, configure it project-wide in `[tool.pyright]`. A new port doesn't change this, so fix freshly surfaced type errors rather than muting them (see [Analyzer Diagnostics and Suppressions][analyzer-diagnostics-and-suppressions]). + +#### Naming + +- `snake_case` for functions, methods, variables, modules, package directories. +- `PascalCase` for classes, type aliases, type vars, enum members. +- `UPPER_SNAKE_CASE` for module-level constants. +- Single leading underscore for module-private, double leading underscore for name-mangled (rare, and usually means rethink the design). + +#### Imports + +- **Let ruff sort imports.** `[tool.ruff.lint]` `select` includes the `I` rule family (isort-equivalent). Don't hand-sort. +- Standard library first, then third-party, then first-party (the project itself), each block separated by a blank line, which ruff enforces automatically. +- Avoid wildcard imports (`from x import *`) outside `__init__.py` re-exports. + +#### Patterns to Avoid + +- **Don't add backward-compat shims, `# removed` markers, or rename-to-`_` for unused vars** - just delete. Git history is the audit trail. +- **Don't add error handling for impossible cases.** Trust internal code, and validate only at boundaries (user input, parsed config, external APIs). +- **Don't use exceptions for expected control flow.** Exceptions are for *unexpected* states. +- **Don't suppress errors silently** (`except Exception: pass`). Either handle the specific exception and document why it's safe, or let it propagate. + +### Tests + +- `pytest` with the configuration in `[tool.pytest.ini_options]`. Default invocation: `uv run pytest`. +- One test file per module under test, named `test_<module>.py`. +- Test functions named `test_<scenario>_<expected_behavior>`, descriptive and not numbered. +- Use fixtures (defined in `conftest.py` for shared ones, or per-test for narrowly-scoped) instead of setup/teardown methods. +- **Avoid mocking when fakes work.** Hand-rolled fakes that implement the protocol you depend on are usually clearer and break less than `unittest.mock` magic. +- **Test edge cases that the docstring promises**, not implementation details. If the test breaks when you refactor *without changing behavior*, the test is asserting on an implementation detail. + +### Versioning + +**Published packages.** `_version.py` ships with `__version__ = "0.0.0"` as a placeholder. Until you wire `_version.py` to something that increments (the usual options are `hatch-vcs`, a version.json bridge, or manual bumps), no new PyPI versions will land, and publishing with `skip-existing: true` keeps a stuck placeholder version from failing the run. + +**Source-only repos** (no PyPI publish, with a source-release on dispatch or no release at all) do not need `_version.py`: keep a static `version` in `pyproject.toml` `[project]`, or let the release pipeline's version source (e.g. NBGV + `version.json`) own the tag. There is no publish step to guard, so `skip-existing` does not apply. + +### Linter Cleanliness + +Before pushing or opening a PR: + +- VS Code's **Problems** pane should be quiet for the files you touched. The relevant linters are ruff (via the `charliermarsh.ruff` extension) and pyright (via the `ms-python.python` extension's bundled Pylance). +- The CI gate is `uv run ruff check`, `uv run ruff format --check`, the repo's type checker (`uv run pyright` or `uv run mypy src`), and `uv run pytest`, the same commands as the local loop above, run from the Python project directory. (Invoke them as separate steps, not `&&`-chained, so the runner shell is irrelevant.) +- Markdown in this directory follows the repo-wide [Markdown and Spelling][markdown-and-spelling] rules. + <!-- Repo --> [analyzer-diagnostics-and-suppressions]: #analyzer-diagnostics-and-suppressions [clean-compile-verification]: #clean-compile-verification [governance]: ./GOVERNANCE.md +[governance-running-the-linters-locally]: ./GOVERNANCE.md#running-the-linters-locally-known-working-invocations +[governance-verification-discipline]: ./GOVERNANCE.md#verification-discipline +[history]: ./HISTORY.md +[line-endings]: ./GOVERNANCE.md#line-endings +[markdown-and-spelling]: #markdown-and-spelling [markdownlint-cli2]: ./.markdownlint-cli2.jsonc [readme]: ./README.md [root]: ./.editorconfig +[versioning-section]: #versioning [vscode-tasks]: ./.vscode/tasks.json + +<!-- External --> + +[docs-link]: https://docs.pytest.org/ +[latest-link]: https://hatch.pypa.io/latest/ +[mypy-link]: https://mypy-lang.org/ +[pep-0257-link]: https://peps.python.org/pep-0257/ +[pyright-link]: https://microsoft.github.io/pyright/ +[ruff-link]: https://docs.astral.sh/ruff/ +[uv-link]: https://docs.astral.sh/uv/ diff --git a/Docker/Dockerfile b/Docker/Dockerfile index 842069b..660ce58 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -35,7 +35,7 @@ FROM mcr.microsoft.com/dotnet/runtime:10.0-alpine AS final ARG LABEL_VERSION="1.0.0.0" LABEL name="PhotoCleaner" \ version="${LABEL_VERSION}" \ - description="An application that prepares photos and videos for import into photo managers." \ + description="Utility to prepare photos and videos for import into photo managers." \ maintainer="ptr727" # exiftool: EXIF metadata reader/writer (pulls in Perl) diff --git a/Docker/README.md b/Docker/README.md index 1f114fa..758d23d 100644 --- a/Docker/README.md +++ b/Docker/README.md @@ -1,14 +1,14 @@ # PhotoCleaner -An application that prepares photos and videos for import into photo managers. +Utility to prepare photos and videos for import into photo managers. ## Documentation Refer to the [project page][github] for complete usage and configuration. -- **Source Code**: [GitHub][github] - source code, issues, and CI/CD pipelines. -- **Binary Releases**: [GitHub Releases][releases] - pre-compiled executables for Windows, Linux, and macOS. -- **Docker Images**: [Docker Hub][dockerhub] - container images with exiftool and ffmpeg pre-installed. +- **Source Code**: [GitHub][github], holding the source, the issues, and the CI/CD pipelines. +- **Binary Releases**: [GitHub Releases][releases], carrying pre-compiled executables for Windows, Linux, and macOS. +- **Docker Images**: [Docker Hub][dockerhub], carrying container images with exiftool and ffmpeg pre-installed. ## Docker Tags diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 875e823..c14e1f0 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -26,9 +26,18 @@ A state-changing GitHub call is the highest-blast-radius thing an agent does her - **Never fabricate, guess, or reuse an identifier passed to a write.** Every id a state-changing call consumes (a node id, a numeric id, a thread or comment id) is captured from a live query in the **same** session into a variable and passed from there. Do not hand-type an id, guess it, recall it from memory or an earlier session, or copy it from documentation or an example. Ids commonly resolve **globally**, so a wrong-but-valid id does not fail. It writes to the wrong target, in someone else's repository. If a query returns no id, stop rather than invent one to proceed. - **A write is never a probe, and a write's output is never suppressed.** Never fire a state-changing call to see whether it works: decide it should happen, make it happen, and read the result. Never append output-discarding redirection or a force-success tail to a mutation (for example `>/dev/null`, `2>/dev/null`, `&>/dev/null`, `|| true`, `|| :`, `|| echo`), because the write's output is exactly what must be read. A write that appears to fail is **verified, not assumed harmless**, because the operation may have succeeded on the server while the client reported an error, so confirm the actual state before retrying or moving on. The ban targets hiding a *failure*. An ad-hoc call's response is the only signal you get, so `>/dev/null 2>&1`, `|| true`, and `|| echo`, which swallow the error stream or force success, are never acceptable on one. A committed script under `set -e` is a narrow exception: it may send a write's *stdout* to `/dev/null` to drop the success-response noise, because stderr stays visible and a failed write still aborts loudly (`repo-config/configure.sh` does exactly this). The exception is stdout-only suppression inside a reviewed, fail-loud script, never `2>&1` or a force-success tail, and never an ad-hoc command. +## Representative Data in Agent-Authored Text + +Agent-authored text illustrates with data the agent constructed, never with data it observed in the maintainer's environment. This binds every surface an agent writes: pull request and issue comments, review replies, commit messages, code, tests, fixtures, and docs. Reading real data is unrestricted, and what is bounded is what an agent copies out of the environment into text that is committed or posted. The rule holds for a private repository as much as a public one, since a repository's audience changes with one settings toggle while the text stays exactly where it was written, and it holds where the data is the maintainer's own, since the exposure happens on their behalf before they can weigh it. + +- **Synthetic evidence is the better evidence, not a weaker substitute.** A case constructed to carry the defect demonstrates it exactly and any reader can re-run it, where observed data proves the same thing and can never be reproduced by anyone else. A filename built to contain a newline is a complete proof of a newline-handling defect, and the real directory it was found in adds nothing the proof needed. Reaching for observed data to make a finding more convincing inverts which of the two is the stronger evidence. Where observed data is what revealed the defect, name its shape, meaning the property that triggers the fault, and construct a case that carries that property. +- **The exposure is one-way.** A public comment is fetched, cached, and indexed the moment it posts, so editing it afterwards is mitigation rather than a fix, and the edit leaves the original readable in the comment's edit history to anyone who can read the repository. Text that has already landed is reported to the maintainer rather than quietly rewritten, since the decision on what to do about it, deletion included, is theirs. Do not quote the exposed data again while reporting or investigating it, because a transcript, an issue, or a commit message written about the exposure reproduces it somewhere new. +- **No checker closes this.** A pattern finds an absolute home path or a drive letter, and gating that subset is worth doing as a floor. The exposure this rule exists for was name-shaped, and a name is not pattern-detectable, so a search of the offending text for path-shaped strings returns nothing while the names sit in plain sight. A gate here catches the easy half, and mistaking it for the answer is what stops anyone looking at the other half, which is why this is a judgment an agent applies rather than a check it waits for. + ## Git and Commit Rules - **Default to staging, not committing.** Stage changes with `git add` and leave `git commit` to the developer unless the developer has explicitly authorized the agent to commit for the current ask ("commit this", "open a PR", etc.). Authorization is scope-bound: it covers the commits needed for that specific task, not a blanket commit license for the rest of the session. +- **"Commit" means commit and push.** An authorization to commit carries the push to the feature branch the work belongs on, because nothing reviews a local commit. The Copilot review loop, the required status checks, and the maintainer all read the remote, so work that stops at `git commit` leaves the review unstarted and the branch's state private to one machine, which reads as progress while none of the gates have run. Push to the feature branch, never to a protected branch (see the Branching Model), and never with `--force`. Holding a commit locally is the narrower case, so it happens when the developer asks for it rather than by default. - **Check the working tree for the maintainer's own uncommitted edits before committing.** The maintainer hand-edits files live (often `README.md`/`HISTORY.md`, sometimes with the editor's LF->CRLF flip on top). Review `git status` first. If there are changes you did not make, ask whether to include them rather than bundling half-finished work or stranding it in an unrelated commit. - **All commits must be cryptographically signed (SSH or GPG).** Branch protection enforces this on both branches, and unsigned commits are rejected on push. Signing depends on environment configuration: `git config commit.gpgsign true`, a configured `user.signingkey`, and a working signing agent (loaded `ssh-agent` for SSH, or `gpg-agent` for GPG). If signing is not configured in the environment, **do not commit**. Surface the missing config to the developer and stop at `git add`. Verify before any agent-authored commit (`git config --get commit.gpgsign && ssh-add -L` or the GPG equivalent). **Signing must be live before the *first* commit, not retrofitted.** Turning on `Require signed commits` against a branch that already has unsigned commits forces a rewrite of that entire history to re-sign it, changing every commit SHA and making whoever does the rewrite the committer and signer of every commit (a rebase preserves the `author` field but not the original signatures, and you cannot sign another contributor's commits for them). During new-repo setup, never create commits until signing is verified. - **Commit under the committing account's own GitHub `noreply` identity, never a private, personal, or invented address.** The `author` and `committer` on every agent-authored commit are the GitHub `noreply` address of the account whose key signs the commit (above). GitHub issues these in a `username@users.noreply.github.com` or `ID+username@users.noreply.github.com` form, and for this single-maintainer fleet it is the owner's `ptr727@users.noreply.github.com`. Do not set `user.name`/`user.email` to a fabricated persona, bot name, or product name, and do not commit under whatever identity the environment happens to carry: verify `git config --get user.email` is that GitHub `noreply` address before committing. **Verify it, do not set it.** The identity is host configuration, set globally once, so a repo-local `user.email` is redundant where the global is right and a wrong identity where it is not, and it silently shadows the global it overrides. A mismatch is a host fault to surface to the maintainer rather than to patch per repo, because a local override hides a broken host that then commits under the wrong identity in every other repo on that machine. A wrong identity is not cosmetic: a private email trips GitHub's email-privacy push protection (GH007), and an unrecognized or invented author pollutes history. Identity is separate from signing: a wrong author does not by itself fail the signature rule, but the ad-hoc identities that produce it are typically also unsigned, which the signing rule above then rejects on push. @@ -212,6 +221,7 @@ The checks that separate work actually done from work that merely reports succes - **Never edit source through a shell heredoc when the text carries backslash escapes.** The shell consumes the escape and writes an invisible control character in its place, so a `\b` inside a regex becomes a backspace and the pattern silently matches nothing while every test still passes. Use a file-editing tool for such text. When a check inspects text for control characters, use `str.isprintable()` rather than a codepoint floor, since DEL and the Unicode format characters sit above 32 and are equally invisible in a diff. - **Never edit an active `.code-workspace` file.** A workspace file rewritten on disk can make VS Code reload the window, and a reload destroys the running agent session's context, so the work in flight is lost with nothing to catch it, and the trigger is not fully characterized (an agent's edit has caused the reload where a human's identical edit did not). Surface the needed change for the maintainer to apply by hand. - **A green check is not evidence the work happened.** A skipped job and a passing job are indistinguishable in the aggregated required check. When a job exists to exercise something, confirm from its log that it ran and produced the output it promises. +- **A launched process is not a result, and a cause nobody observed is not a diagnosis.** "The watcher is armed" names a process rather than a finding, so what gets reported is the output that process produced, and where it produced none, that absence is the report. The failure it prevents is an agent standing still on a condition that was met half an hour earlier, having announced the wait and never read it. Naming an external cause for such a stall afterwards, a throttle or a quota that appears nowhere in the record, turns a local defect into a story about someone else and closes the investigation on the wrong party, so read the record for the cause before naming one, and where the record does not carry it, report the cause as unknown. - **A workflow change is only fully exercised by CI.** Extracting a `run:` block and executing it locally validates the script and nothing else, because `secrets: inherit`, `permissions:`, `needs:` wiring, and reusable-workflow inputs resolve only in a real run. - **A review flags an instance, so fix the class.** When a reviewer cites one stale claim, one silent-narrowing pattern, or one mis-worded contract, sweep for its siblings before replying. Reviewers sample rather than enumerate. @@ -248,28 +258,47 @@ Drive the loop to green, meaning a review confirmed on the latest head SHA and e For provider-specific mechanics (how to request review, query review state, post replies, resolve threads), see the **GitHub Copilot Review Runbook** in [.github/copilot-instructions.md](./.github/copilot-instructions.md). This file owns the contract, and that file owns the mechanics. +### Every Finding Ends in an Action + +**A finding is closed by one of five outcomes, and a round count is never one of them.** The loop runs until no finding stands, however many rounds that takes, because the number of rounds measures how much was found rather than whether the work is done. A finding parked, waited out, or superseded by a push is still open. + +1. **It is real, so fix it.** Reply with the fixing commit SHA. +2. **It is not real, so disprove it in the thread**, with the command and its output, the code path that makes it impossible, or the rule that governs it. The proof is addressed to the reviewer as much as to the maintainer, since a decline it can read is what stops it raising the same thing next round. An assertion is not a proof and does not close a finding. +3. **It is real and deliberately not being fixed, which is the maintainer's call and not the agent's.** Say what the finding is, why the fix is unwanted, and get an explicit answer. Never suppress one by silence, by resolving the thread, or by an answer that reads as a decline while conceding the point. +4. **It is real and worth doing later, so file the issue first and reply with its link.** A deferral recorded only in a thread is lost the moment the pull request merges, so the issue is what carries it and the link is what proves it exists rather than being intended. This is for work the change did not create: an adjacent defect the reviewer noticed in passing, or a fix too large to ride along. It does not cover a defect in the change under review, because filing an issue about a bug you are about to merge is outcome 3 in other clothes, and that one is the maintainer's to decide. +5. **It keeps coming back, so fix the class rather than the instance.** A finding raised repeatedly against correct code is a defect in what the code communicates, not in the reviewer. Give it what it lacks: the non-obvious *why* as a comment where the code cannot state it, a clearer name, a narrower interface, or the rule change where the rule is what is wrong. A comment written for this earns its place under the comment rules like any other, so it states the why, stays short, and never cites a rule or addresses the reviewer. Making the noise stop is worth doing well, because a reviewer that repeats itself trains the reader to skim it, and skimming is how a real finding gets missed. + ### Triaging Review Comments +**A low-confidence finding is not a low-value one.** Copilot collapses the findings it is least sure of into the review body instead of raising a thread, and in this fleet's experience those are right the large majority of the time. Judge each one against the code, never against its confidence label. They are also the easiest to lose, because they appear in no thread, so a loop that polls threads alone reports a clean pass while they stand (see the Merge Gate, condition 3). + For each comment, classify before responding: - **Bug** - wrong behavior, missing test coverage, or a real divergence between code and docs. Fix it. Reply with the fixing commit SHA when done. - **Style/convention** - the comment cites a rule from this file or a language-specific style guide. Two cases: - The cited rule matches what the existing codebase already does -> fix the offending code. - - The cited rule contradicts what's in the tree, or industry norm -> **update the rule instead of the code**. The rule is wrong, not the code. Bouncing the same code across rounds is the symptom of a wrong rule. Heuristic: three rounds on the same style category means the rule needs adjusting and the user should authorize the rule change. + - The cited rule contradicts what's in the tree, or industry norm -> **update the rule instead of the code**. The rule is wrong, not the code. Bouncing the same code across rounds is the symptom of a wrong rule, so treat the recurrence itself as the finding and take it to the user for the rule change (outcome 5 above), rather than counting rounds until some threshold licenses it. - **Architectural opinion** - the comment proposes a different design ("constrain this to disabled-by-default", "move it elsewhere", "add a runtime guardrail"). This is judgment, not a bug. Surface it to the user with a recommendation, and don't apply it unilaterally. ### Responding and Resolution Expectations -Reply inline with either the fixing commit SHA (for accepted issues) or a concise rationale (for declines). Resolve review threads when addressed or intentionally declined with rationale. Issue-level comments (those at `repos/.../issues/<N>/comments` rather than tied to a specific line) have no resolution action, so acknowledge with a reply if needed and move on. +Reply inline with either the fixing commit SHA (for accepted issues) or the evidence that disproves it (for declines). **A decline carries proof rather than an assertion**, meaning the command and its output, the code path that makes the concern impossible, or the rule that governs it. "This is fine" is not a reply, and disagreeing without evidence is not addressing a finding, so a thread is not resolved on one. Resolve review threads when addressed, or when declined with that evidence recorded in the thread. Issue-level comments (those at `repos/.../issues/<N>/comments` rather than tied to a specific line) have no resolution action, so acknowledge with a reply if needed and move on. After the final push on a PR, sweep older threads from earlier rounds whose code paths no longer exist, or stale unresolved markers remain in the review UI. +**Answering a suppressed finding is a different act from replying in a thread, and it carries its own pairing.** A threaded reply sits under the comment it answers and the UI shows whether it is resolved. A suppressed finding has neither, so an answer that does not carry its own context is unverifiable: the maintainer cannot tell that it was seen, which finding it addresses, or whether any were skipped, and has to ask. An answer therefore **quotes the finding** in a blockquote, with its `file:line` anchor and enough of Copilot's own words to identify it, **carries one bold verdict per finding** (`Fixed in <SHA>`, `Disproven`, or `No change needed`) so the outcomes are scannable without reading prose, **states the `(N)` count** the block heading gives so N answers can be checked against N findings, and **links the review** that raised them, since a PR accumulates rounds and an unlinked answer is ambiguous about which one it closes. One comment per review round keeps the answers together. + +**Read every round, not only the head.** A suppressed finding has no resolved state, so a push does not retire it: the finding simply stops appearing in a head-scoped query while remaining unanswered. Treating "superseded by a push" as "answered" is how rounds of findings go unanswered. `scripts/pr_review.py status <n>` reports every round and marks which are from earlier ones. + +**The review's own overview cannot be trusted to say whether findings exist.** A body that reads "Copilot reviewed N out of N changed files and generated no new comments" routinely carries a collapsed block of suppressed findings directly beneath that sentence. Read the body for the block rather than the summary line, because the summary line and `reviewDecision` and an empty unresolved-thread list all agree that a review with four outstanding findings is clean. + ### Escalating to the User Bring the user in when: - **Genuine design trade-off** surfaces (fail-open vs fail-closed, narrow vs broad refactor scope, "should we add a guardrail or trust the docstring"). Triage, recommend, ask. -- **Repeated friction** across rounds without convergence, which is the rule-needs-updating signal. Stop, summarize the pattern, and let the user authorize the rule change. +- **A recurring finding** the code keeps attracting, which is the fix-the-class signal. Summarize the pattern and bring the remedy, whether that is the rule change or what the code has to say differently to stop earning it. +- **A finding you judge real but do not want fixed**, which is outcome 3 above and is never the agent's call to make quietly. - **Architectural redesign** is requested rather than a bug fix. Surface with a recommendation, and never apply it unilaterally. Anti-pattern: don't keep flipping the code on the same style point. Flip the rule once and stick to the rule. diff --git a/HISTORY.md b/HISTORY.md index 1fa24bf..bc217c8 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,9 +1,22 @@ # PhotoCleaner -An application that prepares photos and videos for import into photo managers. +Utility to prepare photos and videos for import into photo managers. ## Release History +**Version: 1.1**: + +- Added `verify` command to detect possibly corrupt media: + - I discovered thousands of files in my Immich library that failed to create thumbnails, all these images were [corrupt](https://github.com/ptr727/PhotoCleaner/issues/25) but passed the entire pipeline undetected. + - Runs the Immich decoder inside the Immich docker image to confirm that the file is usable, which makes that decoder the only judge of a file's health. + - Carries no container parser of its own, deliberately, because such a parser condemns whatever it fails to understand, and an unfamiliar but valid format looks the same as a damaged one from the inside. Docker is therefore required. +- The exiftool metadata read now always passes `-validate`: + - Files exiftool reports errors on are marked invalid by `process` and skipped by `import`. + - Validation warnings are logged at debug level only, because many healthy files do produce warnings. +- **Breaking**: + - Commands now exit `2` when they complete with per-file failures, where `process` previously exited `0`. `0` still means success and `1` still means the command could not run. + - `trash` exits `2` when pagination stops early and the trash database ends up short of the server, where it previously reported success. + **Version: 1.0**: - First published release, carrying the multi-arch Docker image and the GitHub release with the Linux and Windows executables attached. diff --git a/OPERATIONS.md b/OPERATIONS.md new file mode 100644 index 0000000..7cfad83 --- /dev/null +++ b/OPERATIONS.md @@ -0,0 +1,86 @@ +# Operations + +How this repository is run. It ships a .NET console application and a multi-architecture Docker image, so its operations are the local gates that mirror CI, the release pipeline, and the external tools the application drives at runtime. + +## Runbooks + +### Run the gates the way CI runs them + +Local and CI runs read the same committed configuration, but they invoke it differently: locally the formatter writes, and in CI it only verifies. The [`.NET Format`](./.vscode/tasks.json) task is the local clean-compile chain, meaning `dotnet csharpier format`, then `dotnet build`, then the style verify. Run the chain and the suite before committing, since the chain never runs the tests and a change that compiles and formats cleanly can still be broken. CSharpier and Husky.Net are local tools declared in [`.config/dotnet-tools.json`](./.config/dotnet-tools.json), and neither the `.NET Format` task nor the two tasks it depends on restores them, so run `dotnet tool restore` on a fresh clone before that task or the chain below. A clone whose package cache does not already hold the tools otherwise fails on the format step: + +```sh +dotnet tool restore +dotnet csharpier format --log-level=debug . +dotnet build +dotnet format style --verify-no-changes --severity=info --verbosity=detailed +dotnet test +dotnet husky run +``` + +CI differs in two places. It substitutes `dotnet csharpier check .` for the format step, and it runs the suite as `dotnet test --collect:"XPlat Code Coverage" --results-directory ./coverage` so coverlet emits the report the Codecov upload consumes. The style verify is identical. So a local run that formats a file leaves CI clean, while an unformatted commit fails there rather than being fixed. + +The lint set runs in containers, matching the `Lint:` tasks in [`.vscode/tasks.json`](./.vscode/tasks.json): + +```sh +docker run --rm --pull=always -v "$PWD":/workdir -w /workdir davidanson/markdownlint-cli2:latest "**/*.md" +docker run --rm --pull=always -v "$PWD":/workdir -w /workdir ghcr.io/streetsidesoftware/cspell:latest --no-progress README.md HISTORY.md +docker run --rm --pull=always -v "$PWD":/repo -w /repo rhysd/actionlint:latest -color +docker run --rm --pull=always -v "$PWD":/check -w /check mstruebing/editorconfig-checker:latest +``` + +Those four are the whole local lint surface, matching the four `Lint:` tasks. CI checks the same four rules from the same committed configuration, but not through the same tools: it reaches markdownlint, cspell and actionlint through SHA-pinned action wrappers, and only editorconfig-checker runs as a container there. The local commands pull `:latest` deliberately, so a local result can legitimately differ from CI once an upstream release lands ahead of the pinned wrapper. Treat CI as authoritative when the two disagree, and read the difference as a version gap rather than a rule change. + +The prose gate lives in the hub rather than here, so it is consumed from a hub checkout and reads only the lines a change touches: + +```sh +python3 [path-to-hub]/scripts/prose_lint.py . --diff origin/develop +``` + +### Cut a release + +Publishing never happens as a side effect of a merge. A release is a `workflow_dispatch` of [`publish-release.yml`](./.github/workflows/publish-release.yml), and the same workflow runs on a weekly schedule so the image picks up base-image and tool updates. Merging to `main` publishes nothing on its own. + +## Backup and Recovery + +The repository is the record, and GitHub holds it. Nothing here keeps state outside git. + +The application writes state the user owns rather than the repository: the SQLite databases named by `--db` and `--trashdb`, and the `.bak` files that `process` creates. `undo` restores those `.bak` files, and it is the recovery path for a processing run that went wrong. Running `process --skipbackup` makes that recovery impossible, which is the trade the flag names. + +A deleted branch is recoverable from any full clone that still has the commit: + +```sh +git push origin [sha]:refs/heads/[branch] +``` + +Never use `--depth 1` on a clone that will amend or force-push, because a shallow clone severs the merge base and orphans the branch. + +## Logs and Debugging + +Workflow runs are the CI log. `gh run list --branch [branch]` and `gh run view [id] --log-failed` reach them, and a local gate above reproduces a CI failure exactly, so reproduce locally before reading workflow logs. + +The application logs to the console and to the file named by `--logfile`. Raise the level with `--loglevel debug` when a file is rejected and the reason is not obvious. + +A calling script branches on the exit code rather than on output. Every command uses the same three codes, and [Exit Codes](./README.md#exit-codes) is the contract for what each one means. + +The operational point is that `0` and `2` both mean the command ran to completion, and they differ only in whether every file succeeded. A pipeline that reads any non-zero code as "nothing happened" is therefore wrong: `1` is the only code that says the command reported nothing about the files, and a command-line parse error short-circuits before any work starts, so nothing was touched. + +## Tool Usage + +The application shells out to external tools rather than reimplementing them, so their versions decide its behavior: + +- **exiftool** reads and writes metadata, invoked through `MediaUtilities.GetExifToolJsonAsync`. It is installed in the Docker image, and a native run needs it on `PATH`. +- **ffmpeg** handles video, and is installed in the image alongside exiftool. +- **Docker** is a runtime dependency of `verify` alone, which runs the Immich decoder inside the `immich-server` image. The preflight exits `1` when the `docker` command is missing or the image cannot be prepared, so an unreachable daemon fails the run rather than condemning files. + +Every other command runs natively, so outside `verify` Docker is a packaging and tooling concern only, meaning the shipped image and the containerized linters above. + +The Immich API key can be given inline with `--apikey` or read from a file with `--apikey-file`, and the two are mutually exclusive. Prefer the file: an inline key lands in shell history and is visible in the process list for as long as the command runs. + +## Configuration Layout + +- [PhotoCleaner/](./PhotoCleaner/) is the console application. +- [PhotoCleanerTests/](./PhotoCleanerTests/) is the xUnit suite, and [PhotoCleanerBenchmarks/](./PhotoCleanerBenchmarks/) is the benchmark project. +- [Docker/](./Docker/) holds the multi-architecture `Dockerfile` and the Docker Hub README. +- [repo-config/](./repo-config/) holds the branch rulesets and the apply script. It sits outside `.github/`, which is Actions-owned. +- [.github/workflows/](./.github/workflows/) holds the CI and release pipelines, with `validate-task.yml` as the single validation gate that the pull request check and the release both call. +- Analyzer and package configuration is central: `Directory.Build.props` carries the analyzer set, and `Directory.Packages.props` pins every package version. diff --git a/PhotoCleaner.code-workspace b/PhotoCleaner.code-workspace index f440ce9..8181455 100644 --- a/PhotoCleaner.code-workspace +++ b/PhotoCleaner.code-workspace @@ -26,20 +26,19 @@ "csharp.debug.symbolOptions.searchNuGetOrgSymbolServer": true, "csharp.debug.symbolOptions.searchMicrosoftSymbolServer": true, "files.encoding": "utf8", - "csharp.debug.justMyCode": false, - "claudeCode.preferredLocation": "sidebar", - "claudeCode.useCtrlEnterToSend": true, + "csharp.debug.justMyCode": false }, "extensions": { "recommendations": [ + "csharpier.csharpier-vscode", "davidanson.vscode-markdownlint", - "ms-dotnettools.csdevkit", - "streetsidesoftware.code-spell-checker", "editorconfig.editorconfig", - "csharpier.csharpier-vscode", + "fanaticpythoner.better-todo-tree", "github.vscode-github-actions", - "eamodio.gitlens", - "fanaticpythoner.better-todo-tree" + "ms-azuretools.vscode-docker", + "ms-dotnettools.csdevkit", + "streetsidesoftware.code-spell-checker", + "yzhang.markdown-all-in-one" ] } } diff --git a/PhotoCleaner/CommandLine.cs b/PhotoCleaner/CommandLine.cs index 76c730c..135a994 100644 --- a/PhotoCleaner/CommandLine.cs +++ b/PhotoCleaner/CommandLine.cs @@ -45,7 +45,7 @@ internal CommandLine(string[] args) internal RootCommand CreateRootCommand() { RootCommand command = new( - "PhotoCleaner - An application that prepares photos and videos for import into photo managers." + "PhotoCleaner - Utility to prepare photos and videos for import into photo managers." ) { _logLevelOption, @@ -57,6 +57,25 @@ internal RootCommand CreateRootCommand() command.Subcommands.Add(CreateImportCommand()); command.Subcommands.Add(CreateIndexCommand()); command.Subcommands.Add(CreateTrashCommand()); + command.Subcommands.Add(CreateVerifyCommand()); + + return command; + } + + private Command CreateVerifyCommand() + { + Command command = new("verify", "Verify that media files can be rendered by Immich") + { + _pathOption, + _threadsOption, + _dbFileOption, + _rehashOption, + _reprocessOption, + }; + command.SetAction( + (parseResult, cancellationToken) => + new VerifyCommand(CreateOptions(parseResult), cancellationToken).ExecuteAsync() + ); return command; } @@ -299,7 +318,7 @@ internal Options CreateOptions( private static Option<DirectoryInfo> CreatePathOption() => new Option<DirectoryInfo>("--path") { - Description = "The directory path to process", + Description = "The media directory path", Required = true, }.AcceptExistingOnly(); @@ -389,7 +408,7 @@ private static Option<bool> CreateRehashOption() => { Option<FileInfo?> option = new("--skipdb") { - Description = "SQLite database with indexed files to be skipped", + Description = "SQLite database with indexed files to be skipped (read-only)", }; option.Validators.Add(result => { @@ -403,10 +422,7 @@ private static Option<bool> CreateRehashOption() => } private static Option<bool> CreateReprocessOption() => - new("--reprocess") - { - Description = "Re-process files even if already marked as processed in the database", - }; + new("--reprocess") { Description = "Re-run every file even if the database marks it done" }; private static Option<bool> CreateMarkProcessedOption() => new("--processed") diff --git a/PhotoCleaner/CommandRunner.cs b/PhotoCleaner/CommandRunner.cs index 3fc248b..24ff2ee 100644 --- a/PhotoCleaner/CommandRunner.cs +++ b/PhotoCleaner/CommandRunner.cs @@ -2,23 +2,43 @@ namespace PhotoCleaner; internal static class CommandRunner { - internal static async Task<int> RunAsync(string commandName, Func<Task> work) + internal static async Task<int> RunAsync(string commandName, Func<Task<int>> work) { Log.Information("{CommandName} started", commandName); + int exitCode; try { - await work().ConfigureAwait(false); + exitCode = await work().ConfigureAwait(false); } catch (OperationCanceledException) { Log.Information("{CommandName} cancelled", commandName); - return 1; + return ExitCode.Error; } catch (Exception ex) when (Log.Logger.LogAndHandle(ex)) { - return 1; + return ExitCode.Error; } - Log.Information("{CommandName} complete", commandName); - return 0; + + // Each outcome is named rather than reading everything that is not Failed as success. + // Nothing returns Error from the work itself today, and the catch blocks return it directly. + // This branch is what keeps a future one from reporting completion. + if (exitCode == ExitCode.Success) + { + Log.Information("{CommandName} complete", commandName); + } + else if (exitCode == ExitCode.Failed) + { + Log.Warning("{CommandName} complete with failures", commandName); + } + else + { + Log.Error( + "{CommandName} did not complete, exit code {ExitCode}", + commandName, + exitCode + ); + } + return exitCode; } } diff --git a/PhotoCleaner/ExifToolJson.cs b/PhotoCleaner/ExifToolJson.cs index f3731b5..27d5a44 100644 --- a/PhotoCleaner/ExifToolJson.cs +++ b/PhotoCleaner/ExifToolJson.cs @@ -1,4 +1,5 @@ using System.Diagnostics.CodeAnalysis; +using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; @@ -153,6 +154,12 @@ public sealed class ExifToolJson [JsonPropertyName("Matroska:DateTimeOriginal")] public string? MatroskaDateTimeOriginal { get; set; } + [JsonPropertyName("ExifTool:Validate")] + public string? Validate { get; set; } + + [JsonPropertyName("ExifTool:Error")] + public string? ExifToolError { get; set; } + [JsonPropertyName("QuickTime:ContentIdentifier")] public string? QuickTimeContentIdentifier { get; set; } @@ -232,6 +239,56 @@ out DateTime date return !string.IsNullOrEmpty(RIFFDateTimeOriginal) ? RIFFDateTimeOriginal : null; } + // Verdicts read "OK", "4 Warnings (all minor)", or "2 Errors, 3 Warnings". + // Only the leading count of each clause matters, since the minor breakdown qualifies warnings. + internal static (int Errors, int Warnings) ParseValidate(string? validate) + { + if (string.IsNullOrWhiteSpace(validate)) + { + return (0, 0); + } + + int errors = 0, + warnings = 0; + foreach (Range clauseRange in validate.AsSpan().Split(',')) + { + ReadOnlySpan<char> clause = validate.AsSpan()[clauseRange].Trim(); + + // Strip any minor qualifier in parentheses before looking for the noun. + int parenIndex = clause.IndexOf('('); + if (parenIndex >= 0) + { + clause = clause[..parenIndex].Trim(); + } + + int spaceIndex = clause.IndexOf(' '); + if ( + spaceIndex <= 0 + || !int.TryParse( + clause[..spaceIndex], + NumberStyles.None, + CultureInfo.InvariantCulture, + out int count + ) + ) + { + continue; + } + + ReadOnlySpan<char> noun = clause[(spaceIndex + 1)..].Trim(); + if (noun.StartsWith("Error", StringComparison.OrdinalIgnoreCase)) + { + errors += count; + } + else if (noun.StartsWith("Warning", StringComparison.OrdinalIgnoreCase)) + { + warnings += count; + } + } + + return (errors, warnings); + } + internal static bool IsDngVersionNewer(string? versionString) { if (string.IsNullOrEmpty(versionString)) diff --git a/PhotoCleaner/ExitCode.cs b/PhotoCleaner/ExitCode.cs new file mode 100644 index 0000000..ce2e399 --- /dev/null +++ b/PhotoCleaner/ExitCode.cs @@ -0,0 +1,13 @@ +namespace PhotoCleaner; + +// Callers script against these values, so each code's meaning is part of the contract. +internal static class ExitCode +{ + internal const int Success = 0; + + // The command could not complete, so it reports nothing about the files. + internal const int Error = 1; + + // The command completed, but at least one file failed. + internal const int Failed = 2; +} diff --git a/PhotoCleaner/ImmichVerifyScript.cs b/PhotoCleaner/ImmichVerifyScript.cs new file mode 100644 index 0000000..29ff665 --- /dev/null +++ b/PhotoCleaner/ImmichVerifyScript.cs @@ -0,0 +1,109 @@ +namespace PhotoCleaner; + +// Calls Immich's own compiled modules rather than reimplementing the preview pipeline. +// Behavior therefore tracks Immich across releases. +// The coupling surface is three module paths and their exported names, which the preflight checks. +internal static class ImmichVerifyScript +{ + // Required for module resolution inside the image. + internal const string WorkingDirectory = "/usr/src/app/server"; + + // Loads the same modules the verify script needs, so a moved module fails before any file is judged. + internal const string Preflight = """ + const ROOT = "/usr/src/app/server/dist"; + const { MediaRepository } = require(ROOT + "/repositories/media.repository.js"); + const { defaults } = require(ROOT + "/config.js"); + const { ThumbnailConfig } = require(ROOT + "/utils/media.js"); + if (!MediaRepository || !defaults || !defaults.image || !ThumbnailConfig) { + throw new Error("Immich modules loaded but expected exports are missing"); + } + process.stdout.write("PHOTOCLEANER_PREFLIGHT_OK\n"); + """; + + internal const string PreflightSentinel = "PHOTOCLEANER_PREFLIGHT_OK"; + + // Reads absolute paths on stdin and writes one JSON object per line: { path, ok, via, error }. + internal const string Verify = """ + const path = require("node:path"); + const readline = require("node:readline"); + const ROOT = "/usr/src/app/server/dist"; + const { MediaRepository } = require(ROOT + "/repositories/media.repository.js"); + const { defaults } = require(ROOT + "/config.js"); + const { ThumbnailConfig } = require(ROOT + "/utils/media.js"); + + // MediaRepository calls whatever the logger interface exposes, and that set has grown before. + // Swallowing every call through a Proxy keeps this working across Immich updates. + const logger = new Proxy( + {}, + { get: (_target, prop) => (prop === "isLevelEnabled" ? () => false : () => undefined) }, + ); + const repo = new MediaRepository(logger); + + const RAW = new Set([".arw", ".cr2", ".dng", ".nef", ".orf", ".rw2"]); + const VIDEO = new Set([ + ".3gp", ".asf", ".avi", ".m2t", ".m2ts", ".mkv", ".mov", ".mp4", ".mts", ".wmv", + ]); + const OUT = "/tmp/photocleaner-verify.jpg"; + + async function verifyVideo(file) { + const info = await repo.probe(file); + const videoStream = info.videoStreams[0]; + if (!videoStream) { + throw new Error("No video stream found"); + } + const config = ThumbnailConfig.create({ + ...defaults.ffmpeg, + targetResolution: defaults.image.preview.size.toString(), + }); + const options = config.getCommand(0, videoStream, undefined, info.format); + await repo.transcode(file, OUT, options); + return "transcode"; + } + + async function verifyImage(file, ext) { + let via = "decode"; + if (RAW.has(ext)) { + try { + if (await repo.extract(file)) { + via = "decode+extract"; + } + } catch { + // Immich treats extraction as best effort too; only the decode must succeed + } + } + await repo.generateThumbnail( + file, + { ...defaults.image.preview, colorspace: "srgb", processInvalidImages: false }, + OUT, + ); + return via; + } + + const rl = readline.createInterface({ input: process.stdin, crlfDelay: Infinity }); + (async () => { + for await (const line of rl) { + const file = line.trim(); + if (!file) { + continue; + } + const ext = path.extname(file).toLowerCase(); + try { + const via = VIDEO.has(ext) + ? await verifyVideo(file) + : await verifyImage(file, ext); + process.stdout.write(JSON.stringify({ path: file, ok: true, via }) + "\n"); + } catch (e) { + const error = String((e && e.message) || e) + .replaceAll(/\s+/g, " ") + .slice(0, 300); + process.stdout.write( + JSON.stringify({ path: file, ok: false, via: "", error }) + "\n", + ); + } + } + })().catch((e) => { + process.stderr.write("fatal: " + String((e && e.message) || e) + "\n"); + process.exit(1); + }); + """; +} diff --git a/PhotoCleaner/ImportCommand.cs b/PhotoCleaner/ImportCommand.cs index 01ff1dd..b084368 100644 --- a/PhotoCleaner/ImportCommand.cs +++ b/PhotoCleaner/ImportCommand.cs @@ -25,6 +25,7 @@ await CommandRunner int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await TrashDatabaseScope @@ -81,7 +82,10 @@ await DatabaseScope trashSkipped ); Log.Information("Deleted {DeletedCount} empty directories", deletedDirs); + Log.Information("Invalid {InvalidCount} files", invalid); Log.Information("Failed {FailedCount} files", failed); + + return failed > 0 || invalid > 0 ? ExitCode.Failed : ExitCode.Success; } ) .ConfigureAwait(false); diff --git a/PhotoCleaner/ImportTask.cs b/PhotoCleaner/ImportTask.cs index 4e2d863..ffda7cc 100644 --- a/PhotoCleaner/ImportTask.cs +++ b/PhotoCleaner/ImportTask.cs @@ -40,6 +40,7 @@ internal enum ImportResult Skipped, SkippedBySkipDb, TrashedInImmich, + Invalid, Failed, } @@ -54,6 +55,7 @@ internal enum ImportResult int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs )> ExecuteAsync( @@ -67,6 +69,7 @@ int deletedDirs skipped = 0, skipDbSkipped = 0, trashSkipped = 0, + invalid = 0, failed = 0; Log.Information("Importing {FileCount} files", allFiles.Count); await Parallel @@ -90,6 +93,7 @@ await Parallel ref skipDbSkipped ), ImportResult.TrashedInImmich => Interlocked.Increment(ref trashSkipped), + ImportResult.Invalid => Interlocked.Increment(ref invalid), ImportResult.Failed => Interlocked.Increment(ref failed), _ => throw new NotImplementedException(), }; @@ -100,6 +104,8 @@ ref skipDbSkipped } catch (Exception ex) { + // Every failure counts, including a file missing since it was indexed. + // Import never renames its source, so a vanished file means the tree changed. Log.Error(ex, "Failed to import '{FilePath}'", file); _ = Interlocked.Increment(ref failed); } @@ -110,7 +116,16 @@ ref skipDbSkipped int deletedDirs = options.DeleteEmpty ? DirectoryCleaner.DeleteEmptyDirectories(options.OutPath!, options.DryRun) : 0; - return (imported, ignored, skipped, skipDbSkipped, trashSkipped, failed, deletedDirs); + return ( + imported, + ignored, + skipped, + skipDbSkipped, + trashSkipped, + invalid, + failed, + deletedDirs + ); } private async Task<ImportResult> ImportFileAsync( @@ -133,9 +148,8 @@ private async Task<ImportResult> ImportFileAsync( string? sha1 = null; if (database is not null || skipDatabase is not null || trashDatabase is not null) { - // Source-side hash caching: when the source file is already recorded in the import DB - // and its size/mtime match disk, ResolveHashesAsync returns the cached hashes without - // rehashing. The cache is keyed by source path because import inserts source paths. + // ResolveHashesAsync returns cached hashes when size and mtime still match disk. + // The cache is keyed by source path because import inserts source paths. FileRecord? cached = database is null ? null : await database.GetByPathAsync(file, cancellationToken).ConfigureAwait(false); @@ -147,9 +161,8 @@ private async Task<ImportResult> ImportFileAsync( sha1 = resolvedSha1; Log.Debug("File '{FilePath}' has SHA-256 '{Sha256}'", file, sha256); - // Trash check is mandatory and runs FIRST. The Trash.db is the durable record of - // "user threw this away in Immich" beyond Immich's 30-day trash retention. Skipping - // here is the only thing preventing re-import after Immich purges its own trash. + // Trash.db outlives Immich's 30-day trash retention. + // Skipping here is the only thing preventing re-import after Immich purges its trash. if (trashDatabase is not null) { bool trashed = await trashDatabase @@ -203,6 +216,20 @@ private async Task<ImportResult> ImportFileAsync( ExifToolJson? meta = await GetFileMetaAsync(file, cancellationToken).ConfigureAwait(false); + // Warnings are ignored on purpose, since most healthy files carry them. + (int validateErrors, _) = ExifToolJson.ParseValidate(meta?.Validate); + if (validateErrors > 0) + { + Log.Error( + "Skipping import of '{FilePath}': exiftool validation reported {ErrorCount} error(s): {Validate} {Error}", + file, + validateErrors, + meta!.Validate, + meta.ExifToolError + ); + return ImportResult.Invalid; + } + string inferredDateStr = string.Empty; DateTime? inferredDate = null; if (options.DatePath && !(meta?.IsDateSet() ?? false)) @@ -310,8 +337,8 @@ await MediaUtilities // Restore source mtime last - after any exiftool writes File.SetLastWriteTimeUtc(finalDest, sourceInfo.LastWriteTimeUtc); - // Record the SOURCE in the import DB. The row identifies the source file we - // imported, not the destination. Lookups are by source content hash (sha256). + // The row identifies the source file, not the destination. + // Lookups are by source content hash. if (database is not null && sha256 is not null && sha1 is not null) { Log.Debug("Inserting source '{SourcePath}' with SHA-256 '{Sha256}'", file, sha256); @@ -334,9 +361,8 @@ await database } catch (OperationCanceledException) { - // Clean up partial destination file on cancelled copy (not move - moves are atomic - // on the same filesystem; for cross-device moves .NET leaves the source intact on - // failure, so no orphan risk) + // Only a cancelled copy can leave a partial destination. + // A move is atomic on one filesystem, and cross-device failure leaves the source intact. if (!options.Move && File.Exists(finalDest)) { try @@ -414,9 +440,8 @@ internal static string[] ComputePathTags(string sourceFile, DirectoryInfo source ); } - // Adds each tag to XMP:Subject on destFile without creating duplicates. - // Uses remove-then-add (-= then +=) per value so existing copies are replaced - // rather than doubled. -overwrite_original skips exiftool's _original backup. + // Remove-then-add per value replaces an existing copy instead of doubling it. + // -overwrite_original skips exiftool's _original backup. private static async Task ApplyTagsAsync( string[] tags, string destFile, diff --git a/PhotoCleaner/IndexCommand.cs b/PhotoCleaner/IndexCommand.cs index bef475d..a56ee9c 100644 --- a/PhotoCleaner/IndexCommand.cs +++ b/PhotoCleaner/IndexCommand.cs @@ -38,6 +38,8 @@ await DatabaseScope Log.Information("Ignored {IgnoredCount} non-media files", ignored); _skippedExtensions.LogWarnings(); Log.Information("Failed {FailedCount} files", failed); + + return failed > 0 ? ExitCode.Failed : ExitCode.Success; } ) .ConfigureAwait(false); diff --git a/PhotoCleaner/MediaUtilities.cs b/PhotoCleaner/MediaUtilities.cs index 555d4e7..c85937c 100644 --- a/PhotoCleaner/MediaUtilities.cs +++ b/PhotoCleaner/MediaUtilities.cs @@ -73,17 +73,46 @@ internal static string GetUniqueFileName(string filePath) ) { Log.Debug("exiftool: Getting metadata for '{FilePath}'", filePath); + + // Separates a file this tool cannot open from one it reads but exiftool cannot parse. + // Both come back from exiftool as an error alongside well formed JSON. + // Without this they are indistinguishable, so a permission problem reads as damaged media. + EnsureReadable(filePath); + + // -all is required because -validate alone narrows the output to just that one tag. + // Measured at no cost over the plain call, so validation rides along instead of a second run. + // A file whose verdict reports an error also makes exiftool exit non-zero, JSON and all. + // The parsed output decides, not the exit code. BufferedCommandResult result = await Cli.Wrap("exiftool") - .WithArguments(["-groupNames", "-json", filePath]) + .WithArguments(["-groupNames", "-json", "-validate", "-all", filePath]) + .WithValidation(CommandResultValidation.None) .ExecuteBufferedAsync(cancellationToken); + + ReadOnlySpan<char> json = result.StandardOutput.AsSpan().Trim([' ', '\n', '\r', '[', ']']); + if (json.IsEmpty) + { + throw new InvalidOperationException( + $"exiftool returned no metadata for '{filePath}' (exit {result.ExitCode}): " + + result.StandardError.Trim() + ); + } + ExifToolJson? exifToolJson = JsonSerializer.Deserialize( - result.StandardOutput.AsSpan().Trim([' ', '\n', '\r', '[', ']']), + json, ExifToolJsonContext.Default.ExifToolJson ); ArgumentNullException.ThrowIfNull(exifToolJson); return exifToolJson; } + // Reading attributes needs no read permission on the content. + // Nothing about a file's metadata therefore proves its bytes can be reached. + // A command that hands the file to another process sees its own lack of access nowhere else. + internal static void EnsureReadable(string filePath) + { + using FileStream probe = File.OpenRead(filePath); + } + internal static async Task SetCreateDateAsync( string createdDate, string outputFile, diff --git a/PhotoCleaner/ProcessCommand.cs b/PhotoCleaner/ProcessCommand.cs index ff4a234..5acca0a 100644 --- a/PhotoCleaner/ProcessCommand.cs +++ b/PhotoCleaner/ProcessCommand.cs @@ -10,6 +10,7 @@ CancellationToken cancellationToken private ConcurrentBag<string> _fileNames = []; private readonly SkippedExtensionTracker _skippedExtensions = new(); private int _failedCount; + private int _invalidCount; private int _deletedCount; private int _modifiedCount; private int _skippedCount; @@ -78,7 +79,12 @@ await ExecuteProcessAsync(database, trashDatabase) ); Log.Information("Modified {ModifiedCount} files", _modifiedCount); Log.Information("Deleted {DeletedCount} files", _deletedCount); + Log.Information("Invalid {InvalidCount} files", _invalidCount); Log.Information("Failed {FailedCount} files", _failedCount); + + return _failedCount > 0 || _invalidCount > 0 + ? ExitCode.Failed + : ExitCode.Success; } ) .ConfigureAwait(false); @@ -90,9 +96,8 @@ await ExecuteProcessAsync(database, trashDatabase) )] private async Task ExecuteProcessAsync(Database? database, TrashDatabase? trashDatabase) { - // Separate files that share a stem (different extensions) so they - // are never processed in parallel - prevents one thread from - // deleting/renaming a file that another thread is reading. + // Files sharing a stem are never processed in parallel. + // Otherwise one thread could delete or rename a file another thread is reading. ConcurrentBag<string> deferred = FixExtensionConflicts(); ConcurrentBag<string> reProcessNames = []; @@ -127,6 +132,9 @@ await ProcessTask case ProcessTask.ProcessResult.Failure: _ = Interlocked.Increment(ref _failedCount); break; + case ProcessTask.ProcessResult.Invalid: + _ = Interlocked.Increment(ref _invalidCount); + break; case ProcessTask.ProcessResult.Deleted: _ = Interlocked.Increment(ref _deletedCount); break; @@ -144,10 +152,18 @@ await ProcessTask } } catch (Exception ex) - when (ex is FileNotFoundException || !File.Exists(fileName)) + when (ex is FileNotFoundException or DirectoryNotFoundException) { + // Process is the only command that rewrites the tree it walks as it goes. + // A missing name is usually its own earlier rename rather than a fault. + // It could equally be an external deletion, and the two are not cheaply told apart. + // So this neither fails the run nor claims to know which one happened. + // Import and verify both count a missing file as failed instead. + // Neither modifies its input, so there a vanished file can only be external. + // Matching the exception alone keeps a permission error out of this branch. + // Information rather than debug, so the run records why the file is gone. Log.Information( - "File no longer exists during processing (concurrent rename): '{FilePath}'", + "File no longer exists during processing: '{FilePath}'", fileName ); } @@ -223,12 +239,8 @@ internal static ( ConcurrentBag<string> deferred ) SplitExtensionConflicts(ConcurrentBag<string> fileNames) { - // Group media files by directory + stem (filename without extension). - // When multiple media files share the same stem (e.g. IMG.DNG + IMG.jpg), - // keep only one per group and defer the rest so they are never - // processed in parallel. Non-media files (e.g. PhotoCleaner.Process.db, - // PhotoCleaner.Process.log) bypass conflict detection - they are skipped - // by ProcessTask and cannot race with media-file pipeline steps. + // Media files are grouped by directory and stem, keeping one per group and deferring the rest. + // Non-media files bypass conflict detection, since ProcessTask skips them and they cannot race. ConcurrentBag<string> filtered = []; ConcurrentBag<string> deferred = []; Dictionary<string, List<string>> stemMap = new( diff --git a/PhotoCleaner/ProcessTask.cs b/PhotoCleaner/ProcessTask.cs index 234d0c9..edbd8d6 100644 --- a/PhotoCleaner/ProcessTask.cs +++ b/PhotoCleaner/ProcessTask.cs @@ -23,6 +23,7 @@ public enum ProcessResult Modified, UnknownExtension, Skipped, + Invalid, } private ExifToolJson? _exifToolJson; @@ -114,10 +115,9 @@ private async Task<ProcessResult> ExecuteAsync() preProcessHash ); - // Trash check: delete files whose SHA-1 matches an Immich-trashed asset. - // The user explicitly threw these away in Immich; uploading them again on the - // next immich-cli run would be wasted work, so prune them here while we are - // already touching every file. Persist deletion in Process.db so re-runs are no-ops. + // Deletes files whose SHA-1 matches an asset already trashed in Immich. + // Pruning here costs nothing, since every file is being touched anyway. + // The deletion is persisted so re-runs are no-ops. if ( trashDatabase is not null && await TryDeleteIfTrashedAsync(sha1).ConfigureAwait(false) @@ -142,6 +142,11 @@ trashDatabase is not null .GetExifToolJsonAsync(fileInfo.FullName, cancellationToken) .ConfigureAwait(false); + if (!CheckExifToolValidation()) + { + return ProcessResult.Invalid; + } + // Process files ProcessResult result = !RenameMismatchedMimeExtensions() @@ -635,6 +640,34 @@ await MediaUtilities } } + // Roughly three quarters of healthy files carry validation warnings, so only errors fail a file. + private bool CheckExifToolValidation() + { + (int errors, int warnings) = ExifToolJson.ParseValidate(_exifToolJson!.Validate); + if (errors == 0) + { + if (warnings > 0) + { + Log.Debug( + "exiftool validation reported {WarningCount} warning(s) for '{FilePath}': {Validate}", + warnings, + fileInfo.FullName, + _exifToolJson.Validate + ); + } + return true; + } + + Log.Error( + "exiftool validation reported {ErrorCount} error(s) for '{FilePath}': {Validate} {Error}", + errors, + fileInfo.FullName, + _exifToolJson.Validate, + _exifToolJson.ExifToolError + ); + return false; + } + private bool WarnDngVersion() { if (!fileInfo.Extension.Equals(".dng", StringComparison.OrdinalIgnoreCase)) diff --git a/PhotoCleaner/SkippedExtensionTracker.cs b/PhotoCleaner/SkippedExtensionTracker.cs index 9e6f9fd..e189c33 100644 --- a/PhotoCleaner/SkippedExtensionTracker.cs +++ b/PhotoCleaner/SkippedExtensionTracker.cs @@ -23,6 +23,14 @@ internal void LogWarnings() sorted.Sort(StringComparer.OrdinalIgnoreCase); foreach (string extension in sorted) { + // A file with no extension tracks as an empty string, which reads as a quoted nothing. + // Naming the case keeps the count honest without printing a blank. + if (extension.Length == 0) + { + Log.Warning("Skipped files carrying no extension"); + continue; + } + Log.Warning("Unknown file extension: '{Extension}'", extension); } } diff --git a/PhotoCleaner/TrashCommand.cs b/PhotoCleaner/TrashCommand.cs index deef835..ffe8c99 100644 --- a/PhotoCleaner/TrashCommand.cs +++ b/PhotoCleaner/TrashCommand.cs @@ -14,6 +14,7 @@ await CommandRunner "Trash", async () => { + bool partialSync = false; await TrashDatabaseScope .RunAsync( options.TrashDbFile, @@ -120,11 +121,12 @@ await trashDatabase } else { - Log.Warning( + Log.Error( "Stopping pagination: invalid NextPage value '{NextPage}' after page {Page}", result.NextPage, page ); + partialSync = true; hasMore = false; } } @@ -152,6 +154,13 @@ await trashDatabase cancellationToken: cancellationToken ) .ConfigureAwait(false); + + // Pagination stopping early leaves the database short of the server. + // Callers use it to skip files, so a short one re-imports trashed assets. + // That is a completed run carrying a failure rather than a success. + // A page that throws leaves it short as well, keeping the pages before it. + // That exits Error from the handler above rather than reaching here. + return partialSync ? ExitCode.Failed : ExitCode.Success; } ) .ConfigureAwait(false); diff --git a/PhotoCleaner/UndoCommand.cs b/PhotoCleaner/UndoCommand.cs index b46b224..7554b77 100644 --- a/PhotoCleaner/UndoCommand.cs +++ b/PhotoCleaner/UndoCommand.cs @@ -25,7 +25,7 @@ await CommandRunner Log.Information("Deleted {DeletedCount} files", deleted); Log.Information("Failed {FailedCount} files", failed); - return Task.CompletedTask; + return Task.FromResult(failed > 0 ? ExitCode.Failed : ExitCode.Success); } ) .ConfigureAwait(false); diff --git a/PhotoCleaner/UndoTask.cs b/PhotoCleaner/UndoTask.cs index 1a03d66..95fe1fa 100644 --- a/PhotoCleaner/UndoTask.cs +++ b/PhotoCleaner/UndoTask.cs @@ -59,9 +59,8 @@ internal static int GetBackupSortKey(string path) // Pass 1 - identify derived base paths HashSet<string> derivedBases = new(StringComparer.OrdinalIgnoreCase); - // Among groups that share the same stem+dir but different - // extensions, the one whose base file currently exists on disk is the derived conversion - // output - the original was backed up then removed; the output is still present. + // Among same-stem groups, the one whose base file still exists is the conversion output. + // The original was backed up then removed, so only the output remains on disk. foreach (KeyValuePair<string, List<string>> kvp in groups) { string basePath = kvp.Key; @@ -191,9 +190,8 @@ internal static int GetBackupSortKey(string path) restored++; - // Delete any extra (numbered) backups for this group - the original is now - // restored and intermediate-state backups (.bak1, .bak2, ...) serve no purpose. - // The primaryBackup was already consumed (moved) by TryRestore, so we skip it. + // Once the original is restored the intermediate-state backups serve no purpose. + // TryRestore already consumed primaryBackup, so it is skipped. foreach ( string backup in backups.Where(b => !string.Equals(b, primaryBackup, StringComparison.OrdinalIgnoreCase) @@ -211,10 +209,8 @@ string backup in backups.Where(b => } } - // Locate derived conversion output. - // Prefer the explicit .bak.out companion written by ProcessTask (reliable even - // when GetUniqueFileName appended a counter suffix like img_1.mp4). - // Fall back to the stem-based heuristic for backups created before this feature. + // The .bak.out companion stays reliable when the output name gained a counter suffix. + // The stem-based heuristic is the fallback for backups predating that companion. string companionPath = primaryBackup + ".out"; if (File.Exists(companionPath)) { @@ -253,10 +249,9 @@ string backup in backups.Where(b => ) ) { - // Legacy fallback: look for same-stem .mp4 with no backup of its own. - // Note: a format-agnostic scan (any same-stem different-ext file) is unsafe - // because companion images (e.g. img.heic alongside img.mov) would be falsely - // deleted. We therefore target only the known video output extension. + // Legacy fallback looking for a same-stem .mp4 with no backup of its own. + // Only the known video output extension is targeted. + // A format-agnostic scan would falsely delete companion images. string stem = Path.GetFileNameWithoutExtension(basePath); string? dir = Path.GetDirectoryName(basePath); string derivedOutput = Path.Combine( diff --git a/PhotoCleaner/VerifyCommand.cs b/PhotoCleaner/VerifyCommand.cs new file mode 100644 index 0000000..d8d5eaa --- /dev/null +++ b/PhotoCleaner/VerifyCommand.cs @@ -0,0 +1,52 @@ +namespace PhotoCleaner; + +internal sealed class VerifyCommand( + CommandLine.Options options, + CancellationToken cancellationToken +) +{ + private readonly SkippedExtensionTracker _skippedExtensions = new(); + + internal async Task<int> ExecuteAsync() => + await CommandRunner + .RunAsync( + "Verify", + async () => + { + (IReadOnlyList<string> files, int totalCount) = FileEnumerator.Enumerate( + options.Path, + options.Threads, + cancellationToken + ); + + VerifyTask.Counts counts = await DatabaseScope + .RunAsync( + options.DbFile, + async database => + { + VerifyTask task = new(options, database, _skippedExtensions); + return await task.ExecuteAsync(files, cancellationToken) + .ConfigureAwait(false); + }, + cancellationToken: cancellationToken + ) + .ConfigureAwait(false); + + Log.Information("Total {TotalCount} files", totalCount); + Log.Information("Verified {VerifiedCount} files", counts.Verified); + Log.Information( + "Skipped {SkippedCount} already verified files", + counts.Skipped + ); + Log.Information("Ignored {IgnoredCount} non-media files", counts.Ignored); + _skippedExtensions.LogWarnings(); + Log.Information("Invalid {InvalidCount} files", counts.Invalid); + Log.Information("Failed {FailedCount} files", counts.Failed); + + return counts.Invalid > 0 || counts.Failed > 0 + ? ExitCode.Failed + : ExitCode.Success; + } + ) + .ConfigureAwait(false); +} diff --git a/PhotoCleaner/VerifyResult.cs b/PhotoCleaner/VerifyResult.cs new file mode 100644 index 0000000..f517dc4 --- /dev/null +++ b/PhotoCleaner/VerifyResult.cs @@ -0,0 +1,22 @@ +using System.Text.Json.Serialization; + +namespace PhotoCleaner; + +// One line of the JSON protocol emitted by the in-container verify script. +internal sealed class ImmichVerifyLine +{ + [JsonPropertyName("path")] + public string? Path { get; set; } + + [JsonPropertyName("ok")] + public bool Ok { get; set; } + + [JsonPropertyName("via")] + public string? Via { get; set; } + + [JsonPropertyName("error")] + public string? Error { get; set; } +} + +[JsonSerializable(typeof(ImmichVerifyLine))] +internal partial class ImmichVerifyJsonContext : JsonSerializerContext; diff --git a/PhotoCleaner/VerifyTask.cs b/PhotoCleaner/VerifyTask.cs new file mode 100644 index 0000000..5ec7e75 --- /dev/null +++ b/PhotoCleaner/VerifyTask.cs @@ -0,0 +1,449 @@ +using System.Text.Json; +using CliWrap; +using CliWrap.Buffered; + +namespace PhotoCleaner; + +// Only running Immich's own decoder can catch files that are intact but undecodable. +internal sealed class VerifyTask( + CommandLine.Options options, + Database? database, + SkippedExtensionTracker skippedExtensions +) +{ + // Immich publishes no :latest tag, so :release is the rolling equivalent. + internal const string ImmichImage = "ghcr.io/immich-app/immich-server:release"; + + // Namespaced to this application and purpose so it cannot collide with any other container. + internal const string ContainerLabel = "photocleaner-verify"; + + // Stateless and shared across the parallel loop, rather than rebuilt for every file. + private readonly IndexTask? _indexTask = database is null + ? null + : new IndexTask(options, database, skippedExtensions); + + // A host path is not a valid container path on every platform. + // The tree is mounted here and paths are translated across the boundary. + private const string ContainerMount = "/photocleaner"; + + // Files per container invocation, amortizing a measured four seconds of fixed startup. + // Most of that is Immich's own module graph loading, which no faithful invocation can avoid. + // Per-file cost spans fiftyfold between a thumbnail and a raw frame. + // The size is set for the cheap end, where it still outruns startup sevenfold. + // Going higher risks fewer batches than threads on a small tree, which costs more than it saves. + private const int BatchSize = 1024; + + internal sealed record Counts(int Verified, int Invalid, int Skipped, int Ignored, int Failed); + + private sealed class Tally + { + private int _verified; + private int _invalid; + private int _skipped; + private int _ignored; + private int _failed; + + internal void Verified() => Interlocked.Increment(ref _verified); + + internal void Invalid() => Interlocked.Increment(ref _invalid); + + internal void Skipped() => Interlocked.Increment(ref _skipped); + + internal void Ignored() => Interlocked.Increment(ref _ignored); + + internal void Failed() => Interlocked.Increment(ref _failed); + + internal Counts ToCounts() => new(_verified, _invalid, _skipped, _ignored, _failed); + } + + internal async Task<Counts> ExecuteAsync( + IReadOnlyCollection<string> files, + CancellationToken cancellationToken = default + ) + { + Tally tally = new(); + await RunAsync(files, tally, cancellationToken).ConfigureAwait(false); + return tally.ToCounts(); + } + + // Batched so one container serves many files, since startup dominates the per-file cost. + // Batches run in parallel, each recording its own results, so state lands throughout the run. + [System.Diagnostics.CodeAnalysis.SuppressMessage( + "Design", + "CA1031:Do not catch general exception types", + Justification = "Per-file catch-all logs the file path and continues verifying remaining files" + )] + private async Task RunAsync( + IReadOnlyCollection<string> files, + Tally tally, + CancellationToken cancellationToken + ) + { + // Nothing to decode means nothing needs Docker. + // The preflight therefore waits for a file that actually reaches the decoder. + // A tree of non-media files needs none. + // Neither does a database-backed run whose files are all already verified. + // The first batch to find a candidate pays for it, bounding a Docker failure to one batch. + Lazy<Task> preflight = new( + () => PreflightAsync(cancellationToken), + LazyThreadSafetyMode.ExecutionAndPublication + ); + + string mount = options.Path.FullName; + List<string[]> batches = [.. files.Chunk(BatchSize)]; + Log.Information( + "Verifying {FileCount} files in {BatchCount} batches", + files.Count, + batches.Count + ); + + await Parallel + .ForEachAsync( + batches, + CreateParallelOptions(cancellationToken), + async (batch, token) => + { + List<string> candidates = new(batch.Length); + foreach (string file in batch) + { + try + { + if (!await ShouldVerifyAsync(file, tally, token).ConfigureAwait(false)) + { + continue; + } + candidates.Add(file); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + // One unreadable file must not abort the batch, nor the run. + Log.Error(ex, "Failed to verify '{FilePath}'", file); + tally.Failed(); + } + } + + if (candidates.Count > 0) + { + await preflight.Value.ConfigureAwait(false); + await DecodeBatchAsync(candidates, mount, tally, token) + .ConfigureAwait(false); + } + } + ) + .ConfigureAwait(false); + } + + // Extension filter and database skip check, which is what hashes the file. + private async Task<bool> ShouldVerifyAsync( + string file, + Tally tally, + CancellationToken cancellationToken + ) + { + string extension = Path.GetExtension(file); + if (!MediaUtilities.SupportedExtensions.Contains(extension)) + { + Log.Debug("Skipping non-media file: '{FilePath}'", file); + skippedExtensions.Track(extension); + tally.Ignored(); + return false; + } + + // Verify only reads, so a file gone since enumeration means the tree changed mid-run. + // The run no longer covers what it was asked to, which is a failure rather than damage. + // Import reaches the same verdict through the exception its file access throws. + // Verify needs the check stated, because without a database it never opens the file. + // The path would otherwise reach Immich and come back as unrenderable instead. + if (!File.Exists(file)) + { + Log.Error("Failed to verify '{FilePath}': the file no longer exists", file); + tally.Failed(); + return false; + } + + if (_indexTask is not null) + { + (IndexStatus status, _, _, bool wasVerified) = await _indexTask + .IndexFileAsync(file, cancellationToken) + .ConfigureAwait(false); + if (status == IndexStatus.Unchanged && wasVerified && !options.Reprocess) + { + Log.Debug("Skipping already verified '{FilePath}'", file); + tally.Skipped(); + return false; + } + } + + // Everything reaching here goes to the decoder, so this is where readability has to hold. + // With no database nothing opens the file at all. + // With one the size and mtime cache returns hashes without reading it. + // The per-file guard turns the throw into a failed count. + MediaUtilities.EnsureReadable(file); + return true; + } + + private async Task DecodeBatchAsync( + List<string> batch, + string mount, + Tally tally, + CancellationToken cancellationToken + ) + { + List<(string Host, string Container)> mapped = new(batch.Count); + foreach (string file in batch) + { + // Paths reach the container one per line, so a line break cannot be expressed here. + // Without this the name splits and the container decodes fragments of it. + // The counts land the same either way, because a split path matches no returned verdict. + // So this buys an accurate reason rather than a different tally. + if ( + file.Contains('\n', StringComparison.Ordinal) + || file.Contains('\r', StringComparison.Ordinal) + ) + { + Log.Error("Failed to verify '{FilePath}': the path contains a line break", file); + tally.Failed(); + continue; + } + + if (!TryToContainerPath(mount, file, out string container)) + { + // The container path would leave the mount, so Immich would judge a different file. + Log.Error( + "File '{FilePath}' is not under the verified directory '{MountRoot}'", + file, + mount + ); + tally.Failed(); + continue; + } + mapped.Add((file, container)); + } + + if (mapped.Count == 0) + { + return; + } + + BufferedCommandResult result; + try + { + result = await RunImmichAsync( + ImmichVerifyScript.Verify, + mount, + string.Join('\n', mapped.Select(entry => entry.Container)), + $"decode of {mapped.Count} files", + cancellationToken + ) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // The batch never ran, so none of its files were judged. + // Counted over mapped rather than batch, since anything dropped in mapping already counted. + Log.Error(ex, "Immich decode batch of {FileCount} files failed to run", mapped.Count); + for (int i = 0; i < mapped.Count; i++) + { + tally.Failed(); + } + return; + } + + if (result.ExitCode != 0) + { + Log.Error( + "Immich decode batch exited {ExitCode}: {Error}", + result.ExitCode, + result.StandardError.Trim() + ); + } + + Dictionary<string, ImmichVerifyLine> results = new(StringComparer.Ordinal); + foreach (string line in result.StandardOutput.Split('\n')) + { + ImmichVerifyLine? parsed = ParseLine(line); + if (parsed?.Path is not null) + { + results[parsed.Path] = parsed; + } + } + + foreach ((string host, string container) in mapped) + { + if (!results.TryGetValue(container, out ImmichVerifyLine? line)) + { + // A missing verdict is a tooling gap, so it must never be reported as corruption. + Log.Error("No verification result returned for '{FilePath}'", host); + tally.Failed(); + continue; + } + + if (!line.Ok) + { + // A file removed after the batch was assembled makes the decoder fail too. + // Reporting that as unrenderable would call a vanished file damaged media. + // Deliberately untested, because reproducing it needs a race. + // The file has to vanish between assembling a batch and reading its results. + if (!File.Exists(host)) + { + Log.Error("Failed to verify '{FilePath}': the file no longer exists", host); + tally.Failed(); + continue; + } + + Log.Error("Immich cannot render '{FilePath}': {Error}", host, line.Error); + tally.Invalid(); + continue; + } + + Log.Debug("Verified '{FilePath}' via {Via}", host, line.Via); + tally.Verified(); + if (database is not null) + { + await database.SetProcessedAsync(host, cancellationToken).ConfigureAwait(false); + } + } + } + + internal static ImmichVerifyLine? ParseLine(string line) + { + ReadOnlySpan<char> trimmed = line.AsSpan().Trim(); + if (trimmed.IsEmpty || trimmed[0] != '{') + { + return null; + } + + try + { + return JsonSerializer.Deserialize( + trimmed, + ImmichVerifyJsonContext.Default.ImmichVerifyLine + ); + } + catch (JsonException) + { + Log.Warning("Ignoring unparsable verification output: '{Line}'", line); + return null; + } + } + + // Throws rather than returning, so an unreachable Docker exits Error instead of condemning every file. + private static async Task PreflightAsync(CancellationToken cancellationToken) + { + Log.Information("Verifying Immich image '{Image}' is usable", ImmichImage); + + BufferedCommandResult result; + try + { + result = await RunImmichAsync( + ImmichVerifyScript.Preflight, + mount: null, + standardInput: null, + "preflight", + cancellationToken + ) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw new InvalidOperationException( + "Verification requires the 'docker' command and the Immich image, and has no " + + "offline mode, because Immich's decoder is the whole check. Run verify " + + "somewhere Docker is available.", + ex + ); + } + + if ( + result.ExitCode != 0 + || !result.StandardOutput.Contains( + ImmichVerifyScript.PreflightSentinel, + StringComparison.Ordinal + ) + ) + { + throw new InvalidOperationException( + $"Immich image '{ImmichImage}' could not be prepared for verification " + + $"(exit {result.ExitCode}): {result.StandardError.Trim()}" + ); + } + } + + private static async Task<BufferedCommandResult> RunImmichAsync( + string script, + string? mount, + string? standardInput, + string label, + CancellationToken cancellationToken + ) + { + List<string> arguments = + [ + "run", + "--rm", + "-i", + // The image's healthcheck cannot pass when the entrypoint is node. + // Without this every container reports unhealthy and alerts whatever watches Docker. + "--no-healthcheck", + // Marks the containers this tool starts so they can be found without matching on image. + // Other containers may run the same image, so an image or ancestor filter is not unique. + "--label", + $"{ContainerLabel}=1", + "--entrypoint", + "node", + "-w", + ImmichVerifyScript.WorkingDirectory, + ]; + if (mount is not null) + { + // Not -v, whose spec is colon delimited, so a colon in the host path is rejected. + // --mount is comma delimited and would reject a comma instead. + // A comma is the more common of the two in a photo directory name. + // Quoting the whole source field as CSV covers both, with any quote doubled. + string source = mount.Replace("\"", "\"\"", StringComparison.Ordinal); + arguments.Add("--mount"); + arguments.Add($"type=bind,\"source={source}\",target={ContainerMount},readonly"); + } + arguments.Add(ImmichImage); + arguments.Add("-e"); + arguments.Add(script); + + // Paired start and stop lines, so the log timestamps carry the duration. + Log.Debug("docker: Starting Immich {Label}", label); + BufferedCommandResult result = await Cli.Wrap("docker") + .WithArguments(arguments) + .WithStandardInputPipe( + standardInput is null ? PipeSource.Null : PipeSource.FromString(standardInput) + ) + .WithValidation(CommandResultValidation.None) + .ExecuteBufferedAsync(cancellationToken); + Log.Debug("docker: Stopped Immich {Label}", label); + return result; + } + + // Fails rather than mapping a file outside the mount. + // Such a path resolves to a container path that leaves the mount, naming a different file. + internal static bool TryToContainerPath(string mountRoot, string file, out string containerPath) + { + string relative = Path.GetRelativePath(mountRoot, file); + if ( + Path.IsPathRooted(relative) + || relative.Equals("..", StringComparison.Ordinal) + || relative.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) + ) + { + containerPath = string.Empty; + return false; + } + + containerPath = ContainerMount + "/" + relative.Replace(Path.DirectorySeparatorChar, '/'); + return true; + } + + private ParallelOptions CreateParallelOptions(CancellationToken cancellationToken) => + new() { MaxDegreeOfParallelism = options.Threads, CancellationToken = cancellationToken }; +} diff --git a/PhotoCleanerTests/CommandLineTests.cs b/PhotoCleanerTests/CommandLineTests.cs index c94bd18..e4f1d4f 100644 --- a/PhotoCleanerTests/CommandLineTests.cs +++ b/PhotoCleanerTests/CommandLineTests.cs @@ -6,6 +6,63 @@ public sealed class CommandLineTests { private static string ExistingDir => Directory.GetCurrentDirectory(); + // -- Verify command ------------------------------------------------------- + + [Fact] + public void VerifyCommand_PathOnly_ParsesWithoutErrors() + { + CommandLine cli = new(["verify", "--path", ExistingDir]); + + cli.Result.Errors.Should().BeEmpty(); + } + + // An option the command does not define is an error, never something silently ignored. + [Fact] + public void VerifyCommand_UnknownOption_IsRejected() + { + CommandLine cli = new(["verify", "--path", ExistingDir, "--nosuchoption"]); + + cli.Result.Errors.Should().NotBeEmpty(); + } + + [Fact] + public void VerifyCommand_WithDbAndThreads_ParsesOptions() + { + CommandLine cli = new([ + "verify", + "--path", + ExistingDir, + "--db", + "Verify.db", + "--threads", + "2", + "--reprocess", + ]); + + CommandLine.Options options = cli.CreateOptions(cli.Result); + + cli.Result.Errors.Should().BeEmpty(); + options.DbFile!.Name.Should().Be("Verify.db"); + options.Threads.Should().Be(2); + options.Reprocess.Should().BeTrue(); + } + + [Fact] + public void VerifyCommand_MissingPath_ReportsError() + { + CommandLine cli = new(["verify"]); + + cli.Result.Errors.Should().NotBeEmpty(); + } + + [Fact] + public void VerifyCommand_IsRegisteredAsSubcommand() + { + CommandLine cli = new(["--help"]); + + cli.Root.Subcommands.Select(command => command.Name).Should().Contain("verify"); + } + // -- SkipBackup option ---------------------------------------------------- [Fact] diff --git a/PhotoCleanerTests/DirectoryCleanerTests.cs b/PhotoCleanerTests/DirectoryCleanerTests.cs index 1e45ad8..7759476 100644 --- a/PhotoCleanerTests/DirectoryCleanerTests.cs +++ b/PhotoCleanerTests/DirectoryCleanerTests.cs @@ -156,9 +156,8 @@ public void DeleteEmptyDirectories_DryRun_DoesNotDeleteFilesystem() dryRun: true ); - // Dry-run counts each directory that is empty at scan time but does not delete. - // Cascading deletes (parent that would become empty after child removal) are not - // simulated, so this only reports the leaves. + // Dry-run counts each directory empty at scan time but deletes nothing. + // Cascading deletes are not simulated, so only the leaves are reported. deleted.Should().Be(2); Directory.Exists(emptyA).Should().BeTrue(); Directory.Exists(emptyB).Should().BeTrue(); diff --git a/PhotoCleanerTests/ExifToolJsonTests.cs b/PhotoCleanerTests/ExifToolJsonTests.cs index a99361f..7aa52be 100644 --- a/PhotoCleanerTests/ExifToolJsonTests.cs +++ b/PhotoCleanerTests/ExifToolJsonTests.cs @@ -433,4 +433,48 @@ public void IsDngVersionNewer_WhenMalformed_ReturnsFalse(string version) // Assert result.Should().BeFalse(); } + + // -- Validate verdict parsing --------------------------------------------- + // The sample strings below are verbatim from a sweep of a real collection. + // Roughly three quarters of healthy files carry warnings, so only the error count is acted on. + + [Theory] + [InlineData("OK", 0, 0)] + [InlineData("1 Warning", 0, 1)] + [InlineData("1 Warning (minor)", 0, 1)] + [InlineData("2 Warnings", 0, 2)] + [InlineData("4 Warnings (all minor)", 0, 4)] + [InlineData("5 Warnings (1 minor)", 0, 5)] + [InlineData("12 Warnings (9 minor)", 0, 12)] + [InlineData("1 Error", 1, 0)] + [InlineData("2 Errors, 3 Warnings", 2, 3)] + [InlineData("1 Error, 11 Warnings (3 minor)", 1, 11)] + public void ParseValidate_KnownVerdicts_ReturnsCounts( + string validate, + int expectedErrors, + int expectedWarnings + ) + { + // Act + (int errors, int warnings) = ExifToolJson.ParseValidate(validate); + + // Assert + errors.Should().Be(expectedErrors); + warnings.Should().Be(expectedWarnings); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("something unexpected")] + public void ParseValidate_AbsentOrUnrecognized_ReturnsZeroCounts(string? validate) + { + // An unreadable verdict must not count as an error. + // An exiftool output change would otherwise start failing every file in the collection. + (int errors, int warnings) = ExifToolJson.ParseValidate(validate); + + errors.Should().Be(0); + warnings.Should().Be(0); + } } diff --git a/PhotoCleanerTests/ImportTaskTests.cs b/PhotoCleanerTests/ImportTaskTests.cs index a7d8ddf..f88efd8 100644 --- a/PhotoCleanerTests/ImportTaskTests.cs +++ b/PhotoCleanerTests/ImportTaskTests.cs @@ -79,6 +79,64 @@ await Cli.Wrap("exiftool") .WithValidation(CommandResultValidation.None) .ExecuteAsync(); + // -- Invalid: exiftool reports an error, file is skipped ----------------- + + [Fact] + public async Task ExecuteAsync_ExifToolErrorFile_IsInvalidAndNotImported() + { + string srcDir = TempDir(); + string outDir = TempDir(); + try + { + // A file whose bytes are not a real image makes exiftool report a format error. + string bad = Path.Combine(srcDir, "garbage.jpg"); + await File.WriteAllBytesAsync( + bad, + [.. Enumerable.Range(0, 500).Select(i => (byte)(i % 251))], + TestContext.Current.CancellationToken + ); + + ImportTask task = new( + CreateOptions(outDir), + database: null, + skipDatabase: null, + trashDatabase: null, + new() + ); + ( + int organized, + int ignored, + int skipped, + int skipDbSkipped, + int trashSkipped, + int invalid, + int failed, + int deletedDirs + ) = await task.ExecuteAsync( + [bad], + new DirectoryInfo(srcDir), + TestContext.Current.CancellationToken + ); + + // Invalid is its own outcome, distinct from failed, and nothing is copied out. + invalid.Should().Be(1); + failed.Should().Be(0); + organized.Should().Be(0); + ignored.Should().Be(0); + skipped.Should().Be(0); + skipDbSkipped.Should().Be(0); + trashSkipped.Should().Be(0); + deletedDirs.Should().Be(0); + Directory.GetFiles(outDir, "*", SearchOption.AllDirectories).Should().BeEmpty(); + File.Exists(bad).Should().BeTrue(); + } + finally + { + Directory.Delete(srcDir, true); + Directory.Delete(outDir, true); + } + } + // -- UnsupportedFile: ignored, count incremented ------------------------- [Fact] @@ -104,6 +162,7 @@ public async Task ExecuteAsync_UnsupportedFile_Ignored() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -152,6 +211,7 @@ public async Task ExecuteAsync_DryRun_ReportsCountButLeavesFiles() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -201,6 +261,7 @@ public async Task ExecuteAsync_SupportedFile_CopiedToDateSubdir() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -250,6 +311,7 @@ public async Task ExecuteAsync_NoExifDate_FallsBackToMinValue() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -305,6 +367,7 @@ public async Task ExecuteAsync_SameNameFilesInSameMonth_SecondGetsUniqueName() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -359,6 +422,7 @@ public async Task ExecuteAsync_OrganizedFile_PreservesLastWriteTime() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -407,6 +471,7 @@ public async Task ExecuteAsync_MoveFlag_SourceFileRemoved() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -455,6 +520,7 @@ public async Task ExecuteAsync_CopyDefault_SourceFileRetained() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -505,6 +571,7 @@ public async Task ExecuteAsync_DeleteEmpty_RemovesEmptyTargetSubdirectories() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -575,6 +642,7 @@ await db.InsertAsync( int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -623,6 +691,7 @@ public async Task ExecuteAsync_SubdirectoryFormat_CreatesNestedDirs() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -678,6 +747,7 @@ public async Task ExecuteAsync_WithDatabase_NewFile_RecordedInDb() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -697,9 +767,8 @@ int deletedDirs ); recorded.Should().BeTrue(); - // The DB row identifies the SOURCE file we imported (not the destination), - // so the dedup lookup keeps working even if the destination is later mutated - // (e.g. by tag injection or process re-hash). + // The row identifies the source file, not the destination. + // Dedup therefore survives later mutation of the destination. FileRecord? row = await db.GetByPathAsync( jpg, cancellationToken: TestContext.Current.CancellationToken @@ -727,12 +796,11 @@ int deletedDirs [Fact] public async Task ExecuteAsync_TagPathMutatesDest_SourceRowIsUntouched() { - // Models the central design property: import inserts a row keyed by SOURCE path, - // even though the destination file gets mutated by tag injection. Because no - // command writes to the source path's row through normal use, dedup remains stable. + // Import keys its row by source path even though tag injection mutates the destination. + // No command writes that row in normal use, so dedup stays stable. string srcDir = TempDir(); - // Put the file in a sub-directory so --tagpath actually has tokens to apply - // (ComputePathTags returns [] for files at the root of --path). + // The file needs a sub-directory for --tagpath to have any tokens to apply. + // ComputePathTags returns nothing for files at the root of --path. string srcSubDir = Path.Combine(srcDir, "vacation"); Directory.CreateDirectory(srcSubDir); string outDir = TempDir(); @@ -757,15 +825,14 @@ public async Task ExecuteAsync_TagPathMutatesDest_SourceRowIsUntouched() trashDatabase: null, new() ); - (int organized, _, _, _, _, _, _) = await task.ExecuteAsync( + (int organized, _, _, _, _, _, _, _) = await task.ExecuteAsync( [jpg], new DirectoryInfo(srcDir), TestContext.Current.CancellationToken ); organized.Should().Be(1); - // The dest file on disk has had XMP tags written and now hashes to something - // different from the source. + // Writing XMP tags changed the destination, so it no longer hashes to the source. string dest = Path.Combine(outDir, "2024-06", "photo.jpg"); File.Exists(dest).Should().BeTrue(); (string destSha256, _) = await Database.ComputeHashesAsync( @@ -782,8 +849,8 @@ public async Task ExecuteAsync_TagPathMutatesDest_SourceRowIsUntouched() srcRow.Should().NotBeNull(); srcRow.Sha256.Should().Be(srcSha256); - // No row was inserted for the dest path. Process operating on /Processed with a - // separate Process.db is what tracks the dest state; import never writes there. + // No row is inserted for the destination path. + // A separate Process.db tracks destination state, and import never writes there. FileRecord? destRow = await db.GetByPathAsync( dest, cancellationToken: TestContext.Current.CancellationToken @@ -822,7 +889,7 @@ public async Task ExecuteAsync_ImportTwice_SecondRunSkipsAndCreatesNoDuplicate() trashDatabase: null, new() ); - (int organized1, _, int skipped1, _, _, int failed1, _) = await first.ExecuteAsync( + (int organized1, _, int skipped1, _, _, _, int failed1, _) = await first.ExecuteAsync( [jpg], new DirectoryInfo(srcDir), TestContext.Current.CancellationToken @@ -838,7 +905,7 @@ public async Task ExecuteAsync_ImportTwice_SecondRunSkipsAndCreatesNoDuplicate() trashDatabase: null, new() ); - (int organized2, _, int skipped2, _, _, int failed2, _) = await second.ExecuteAsync( + (int organized2, _, int skipped2, _, _, _, int failed2, _) = await second.ExecuteAsync( [jpg], new DirectoryInfo(srcDir), TestContext.Current.CancellationToken @@ -1095,6 +1162,7 @@ public async Task ExecuteAsync_TagPath_DryRun_NoTagsApplied() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1140,6 +1208,7 @@ public async Task ExecuteAsync_DatePath_DateInferredFromPath_SetsExifDateAndOrga int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1264,6 +1333,7 @@ public async Task ExecuteAsync_DatePath_DryRun_NoDateWritten() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1310,6 +1380,7 @@ public async Task ExecuteAsync_NonMediaFile_TracksExtension() int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1370,6 +1441,7 @@ await trashDb.InsertHashAsync( int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1427,6 +1499,7 @@ await trashDb.InsertHashAsync( int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1494,6 +1567,7 @@ await skipDb.InsertAsync( int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( @@ -1559,6 +1633,7 @@ await skipDb.InsertAsync( int skipped, int skipDbSkipped, int trashSkipped, + int invalid, int failed, int deletedDirs ) = await task.ExecuteAsync( diff --git a/PhotoCleanerTests/IndexTaskTests.cs b/PhotoCleanerTests/IndexTaskTests.cs index ee9996b..7f4817c 100644 --- a/PhotoCleanerTests/IndexTaskTests.cs +++ b/PhotoCleanerTests/IndexTaskTests.cs @@ -229,8 +229,7 @@ public async Task IndexFileAsync_Rehash_AlwaysRecomputesHash() cancellationToken: TestContext.Current.CancellationToken ); - // Rehash task forces recomputation - result should be same hash (file unchanged) - // but the code path goes through ComputeHashesAsync + // The file is unchanged so the hash matches, but the path runs through ComputeHashesAsync. IndexTask rehashTask = new( CreateOptions(rehash: true), db, @@ -322,9 +321,8 @@ public async Task IndexFileAsync_MarkProcessed_InsertsWithIsProcessedTrue() [Fact] public async Task IndexFileAsync_MarkProcessed_PreservesExistingFlagOnUpdate() { - // --processed only affects rows being INSERTED. Existing rows keep their flag - // (they get cleared by UpdateHashesAsync on a hash change, which is the existing - // behavior, but the --processed flag itself never alters rows during update). + // --processed affects only rows being inserted, and existing rows keep their flag. + // UpdateHashesAsync clears that flag on a hash change, but --processed never alters it. string dbPath = TempDb(); string filePath = TempFile("original"); try @@ -339,8 +337,8 @@ await noFlag.IndexFileAsync( cancellationToken: TestContext.Current.CancellationToken ); - // Re-run WITH --processed; the file is unchanged so this should be Unchanged - // and the existing row's is_processed should remain 0 (not flipped to 1). + // The file is unchanged, so the re-run reports Unchanged. + // The existing row keeps is_processed at 0. IndexTask withFlag = new( CreateOptions(markProcessed: true), db, diff --git a/PhotoCleanerTests/ProcessTaskTests.cs b/PhotoCleanerTests/ProcessTaskTests.cs index 288b910..fe277b4 100644 --- a/PhotoCleanerTests/ProcessTaskTests.cs +++ b/PhotoCleanerTests/ProcessTaskTests.cs @@ -33,6 +33,66 @@ private sealed class InMemorySink : ILogEventSink // -- Extension / MIME handling -------------------------------------------- + // A file exiftool cannot open looks like one it cannot parse: an error plus valid JSON. + // The shared metadata call separates them by opening the file first. + // A permission problem is therefore never recorded as damaged media, in any command. + [Fact] + public async Task GetExifToolJsonAsync_UnreadableFile_Throws() + { + if (!OperatingSystem.IsLinux()) + { + Assert.Skip("File mode permissions are only enforced on Linux"); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + string locked = Path.Combine(workDir, "locked.jpg"); + try + { + // Arrange + File.Copy(fixture.SourceFile(TempDirectoryFixture.SmallJpegFile), locked); + File.SetUnixFileMode(locked, UnixFileMode.None); + using (FileStream? readable = TryOpen(locked)) + { + if (readable is not null) + { + Assert.Skip("Running with permission to read anything"); + return; + } + } + + // Act + Func<Task> act = async () => + await MediaUtilities.GetExifToolJsonAsync( + locked, + TestContext.Current.CancellationToken + ); + + // Assert + await act.Should().ThrowAsync<UnauthorizedAccessException>(); + } + finally + { + if (File.Exists(locked)) + { + File.SetUnixFileMode(locked, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + private static FileStream? TryOpen(string path) + { + try + { + return File.OpenRead(path); + } + catch (UnauthorizedAccessException) + { + return null; + } + } + [Fact] public async Task ExecuteAsync_UnknownExtension_ReturnsUnknownExtension() { @@ -62,9 +122,8 @@ public async Task ExecuteAsync_MixedCaseExtension_RenamesFile() string workDir = TempDirectoryFixture.CreateWorkDir(); try { - // On case-insensitive filesystems (Windows NTFS, macOS HFS+) "photo.Jpg" and - // "photo.jpg" are the same file, so the rename is meaningless and File.Exists() - // cannot distinguish them. Skip on those platforms; the test runs fully on Linux. + // On a case-insensitive filesystem the two names are the same file. + // The rename is then meaningless and File.Exists cannot distinguish them. if (!TempDirectoryFixture.IsFileSystemCaseSensitive(workDir)) { Assert.Skip( @@ -365,8 +424,8 @@ public async Task ExecuteAsync_MtsFile_RemuxesToMp4() string filePath = Path.Combine(workDir, TempDirectoryFixture.MtsFile); File.Copy(fixture.SourceFile(TempDirectoryFixture.MtsFile), filePath); - // Act - first pass: exiftool returns "m2t" for MPEG-TS, so .mts is renamed to .m2t - // and queued for reprocess; remux to .mp4 happens on the second pass + // For MPEG-TS, exiftool returns "m2t", so the first pass renames and re-queues. + // The remux to .mp4 happens on the second pass. ProcessTask.ProcessResult result = await CreateContext(filePath); // Assert @@ -390,8 +449,8 @@ public async Task ExecuteAsync_M2tsFile_RemuxesToMp4() string filePath = Path.Combine(workDir, TempDirectoryFixture.M2tsFile); File.Copy(fixture.SourceFile(TempDirectoryFixture.M2tsFile), filePath); - // Act - first pass: exiftool returns "m2t" for MPEG-TS, so .m2ts is renamed to .m2t - // and queued for reprocess; remux to .mp4 happens on the second pass + // For MPEG-TS, exiftool returns "m2t", so the first pass renames and re-queues. + // The remux to .mp4 happens on the second pass. ProcessTask.ProcessResult result = await CreateContext(filePath); // Assert @@ -614,8 +673,8 @@ public async Task ExecuteAsync_MovWithPcmAudio_ReencodesAudio() // Act ProcessTask.ProcessResult result = await CreateContext(filePath); - // Assert - exiftool returns "mov" for QuickTime, which matches the extension, so no - // rename occurs; PCM audio is detected and re-encoded directly in the first pass + // For QuickTime, exiftool returns "mov", which matches the extension, so no rename occurs. + // PCM audio is detected and re-encoded directly in the first pass. result.Should().Be(ProcessTask.ProcessResult.Reprocess); File.Exists(filePath).Should().BeFalse(); File.Exists(filePath + ".bak").Should().BeTrue(); @@ -640,8 +699,8 @@ public async Task ExecuteAsync_MovWithAacAudio_NoConversion() // Act ProcessTask.ProcessResult result = await CreateContext(filePath); - // Assert - exiftool returns "mov" for QuickTime, which matches the extension, so no - // rename occurs; AAC audio requires no conversion, file is left as-is + // For QuickTime, exiftool returns "mov", which matches the extension, so no rename occurs. + // AAC audio needs no conversion, so the file is left as-is. result.Should().Be(ProcessTask.ProcessResult.Success); File.Exists(filePath).Should().BeTrue(); File.Exists(Path.ChangeExtension(filePath, ".mp4")).Should().BeFalse(); @@ -704,8 +763,8 @@ public async Task ExecuteAsync_Mp4WithAacAudio_ReturnsSuccess() [Fact] public async Task ExecuteAsync_PcmMp4WithContentIdentifier_PreservesContentIdentifierAfterConversion() { - // Arrange - live photo MP4 with PCM audio: first pass re-encodes audio (no companion yet), - // then companion is added and second pass deletes the video as a live photo + // The first pass re-encodes the audio, since no companion image exists yet. + // The companion is then added and the second pass deletes the video as a live photo. string workDir = TempDirectoryFixture.CreateWorkDir(); try { @@ -741,6 +800,93 @@ public async Task ExecuteAsync_PcmMp4WithContentIdentifier_PreservesContentIdent } } + // -- exiftool validation --------------------------------------------------- + + [Fact] + public async Task ExecuteAsync_FileWithValidationWarnings_IsNotTreatedAsInvalid() + { + // A truncated JPEG makes exiftool report "1 Warning: JPEG format error". + // Warnings must stay advisory, since roughly three quarters of healthy files carry one. + string workDir = TempDirectoryFixture.CreateWorkDir(); + try + { + // Arrange + string filePath = Path.Combine(workDir, "truncated.jpg"); + byte[] source = await File.ReadAllBytesAsync( + fixture.SourceFile(TempDirectoryFixture.SmallJpegFile), + TestContext.Current.CancellationToken + ); + await File.WriteAllBytesAsync( + filePath, + source[..(source.Length / 2)], + TestContext.Current.CancellationToken + ); + + // Act + ProcessTask.ProcessResult result = await CreateContext(filePath); + + // Assert + result.Should().NotBe(ProcessTask.ProcessResult.Invalid); + File.Exists(filePath).Should().BeTrue(); + } + finally + { + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + [Fact] + public async Task ExecuteAsync_FileExifToolErrorsOn_ReturnsInvalid() + { + // On exactly these files, exiftool exits non-zero while still emitting the JSON verdict. + // Without validation disabled on that call this path throws and is counted as a failure. + string workDir = TempDirectoryFixture.CreateWorkDir(); + try + { + // Arrange + string filePath = Path.Combine(workDir, "garbage.jpg"); + await File.WriteAllBytesAsync( + filePath, + [.. Enumerable.Range(0, 500).Select(i => (byte)(i % 251))], + TestContext.Current.CancellationToken + ); + + // Act + ProcessTask.ProcessResult result = await CreateContext(filePath); + + // Assert + result.Should().Be(ProcessTask.ProcessResult.Invalid); + File.Exists(filePath).Should().BeTrue(); + } + finally + { + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + [Fact] + public async Task ExecuteAsync_HealthyFile_ReturnsSuccessWithValidationEnabled() + { + // Guards the always-on -validate addition: it must not perturb the existing pipeline. + string workDir = TempDirectoryFixture.CreateWorkDir(); + try + { + // Arrange + string filePath = Path.Combine(workDir, "photo.jpg"); + File.Copy(fixture.SourceFile(TempDirectoryFixture.SmallJpegFile), filePath); + + // Act + ProcessTask.ProcessResult result = await CreateContext(filePath); + + // Assert + result.Should().Be(ProcessTask.ProcessResult.Success); + } + finally + { + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + // -- DNG version warning --------------------------------------------------- [Fact] diff --git a/PhotoCleanerTests/TempDirectoryFixture.cs b/PhotoCleanerTests/TempDirectoryFixture.cs index 465e3f6..603f382 100644 --- a/PhotoCleanerTests/TempDirectoryFixture.cs +++ b/PhotoCleanerTests/TempDirectoryFixture.cs @@ -91,10 +91,7 @@ internal static void DeleteWorkDir(string workDir) } } - /// <summary> - /// Returns true when the filesystem hosting <paramref name="directory"/> is case-sensitive. - /// Probes at runtime so it works correctly on Linux (true), Windows NTFS (false), and macOS HFS+ (false). - /// </summary> + // Probed at runtime because Linux is case-sensitive while Windows NTFS and macOS HFS+ are not. internal static bool IsFileSystemCaseSensitive(string directory) { string upper = Path.Combine(directory, "_CASE_PROBE_"); @@ -370,8 +367,8 @@ private static async Task RunFfmpegAsync(string[] arguments) => await Cli.Wrap("ffmpeg").WithArguments(["-nostdin", "-y", .. arguments]).ExecuteAsync(); private static async Task RunExiftoolAsync(string[] arguments) => - // exiftool exits with code 1 for warnings (e.g. writing DNG tags to TIFF); - // the write still succeeds, so we suppress exit code validation here. + // Warnings such as writing DNG tags to TIFF make exiftool exit 1. + // The write still succeeds, so exit code validation is suppressed. await Cli.Wrap("exiftool") .WithArguments(arguments) .WithValidation(CommandResultValidation.None) diff --git a/PhotoCleanerTests/TrashCommandTests.cs b/PhotoCleanerTests/TrashCommandTests.cs index a2fab4b..2d0ea37 100644 --- a/PhotoCleanerTests/TrashCommandTests.cs +++ b/PhotoCleanerTests/TrashCommandTests.cs @@ -111,6 +111,48 @@ public void ConvertChecksum_EmptyString_ReturnsNull() => public void ConvertChecksum_InvalidBase64_ReturnsNull() => TrashCommand.ConvertChecksum("not-valid-base64!!!").Should().BeNull(); + // Pagination stopping early leaves the database short of the server. + // Callers use it to skip files, so a short one silently re-imports trashed assets. + [Fact] + public async Task ExecuteAsync_InvalidNextPage_ExitsFailed() + { + string dbPath = TempDb(); + try + { + // Arrange - a page that hands back a NextPage the loop cannot use + byte[] sha1 = new byte[20]; + sha1[0] = 0x01; + using MockImmichHandler handler = new(); + handler.EnqueueResponse( + MakeResponse([Convert.ToBase64String(sha1)], nextPage: "not-a-page") + ); + + using HttpClient client = new(handler, disposeHandler: false) + { + BaseAddress = new Uri("http://localhost:9999"), + }; + TrashCommand command = new( + CreateOptions(dbPath), + TestContext.Current.CancellationToken, + client + ); + + // Act + int exitCode = await command.ExecuteAsync(); + + // Assert - the page it did read is kept, and the run reports the shortfall + exitCode.Should().Be(2); + await using TrashDatabase db = new(dbPath); + await db.InitializeAsync(TestContext.Current.CancellationToken); + long count = await db.GetCountAsync(TestContext.Current.CancellationToken); + count.Should().Be(1); + } + finally + { + File.Delete(dbPath); + } + } + [Fact] public async Task ExecuteAsync_SinglePage_InsertsHashes() { @@ -273,9 +315,6 @@ public async Task ExecuteAsync_DuplicateHashes_Idempotent() } } - /// <summary> - /// Mock HTTP handler that returns queued ImmichSearchResponse payloads. - /// </summary> internal sealed class MockImmichHandler : DelegatingHandler { private readonly Queue<ImmichSearchResponse> _responses = new(); diff --git a/PhotoCleanerTests/UndoTaskTests.cs b/PhotoCleanerTests/UndoTaskTests.cs index 515857c..0a76831 100644 --- a/PhotoCleanerTests/UndoTaskTests.cs +++ b/PhotoCleanerTests/UndoTaskTests.cs @@ -114,7 +114,7 @@ public void Execute_SingleBackupNoCurrentFile_RestoresOriginal() string dir = TempDir(); try { - // img.mp4 was deleted; img.mp4.bak is the only artefact + // The only remaining artifact is img.mp4.bak, because img.mp4 was deleted string bakPath = Path.Combine(dir, "img.mp4.bak"); WriteContent(bakPath); diff --git a/PhotoCleanerTests/VerifyTaskTests.cs b/PhotoCleanerTests/VerifyTaskTests.cs new file mode 100644 index 0000000..57c4bad --- /dev/null +++ b/PhotoCleanerTests/VerifyTaskTests.cs @@ -0,0 +1,584 @@ +using PhotoCleaner; +using Serilog.Events; + +namespace PhotoCleanerTests; + +// A misread line becomes a false accusation against a real photo, so non-verdict output is ignored. +public sealed class VerifyTaskTests(TempDirectoryFixture fixture) + : IClassFixture<TempDirectoryFixture> +{ + [Fact] + public void ParseLine_SuccessfulVerdict_ParsesPathAndVia() + { + // Act + ImmichVerifyLine? line = VerifyTask.ParseLine( + """{"path":"/photos/IMG_0969.HEIC","ok":true,"via":"decode"}""" + ); + + // Assert + line.Should().NotBeNull(); + line!.Path.Should().Be("/photos/IMG_0969.HEIC"); + line.Ok.Should().BeTrue(); + line.Via.Should().Be("decode"); + } + + [Fact] + public void ParseLine_FailureVerdict_ParsesError() + { + // Arrange: the error Immich actually emits for the reported defect + const string json = """ + {"path":"/photos/bad.heic","ok":false,"via":"","error":"Input file has corrupt header: bad seek to 104941"} + """; + + // Act + ImmichVerifyLine? line = VerifyTask.ParseLine(json); + + // Assert + line.Should().NotBeNull(); + line!.Ok.Should().BeFalse(); + line.Error.Should().Contain("corrupt header"); + } + + [Fact] + public void ParseLine_RawExtractVerdict_ParsesVia() + { + // Act + ImmichVerifyLine? line = VerifyTask.ParseLine( + """{"path":"/photos/IMG_1603.dng","ok":true,"via":"decode+extract"}""" + ); + + // Assert + line!.Via.Should().Be("decode+extract"); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("(node:1) Warning: some deprecation notice")] + [InlineData("npm notice")] + public void ParseLine_NonJsonOutput_ReturnsNull(string line) + { + // Act + ImmichVerifyLine? result = VerifyTask.ParseLine(line); + + // Assert + result.Should().BeNull(); + } + + [Fact] + public void ParseLine_TruncatedJson_ReturnsNullWithoutThrowing() + { + // Arrange: what a killed container leaves behind mid-write + const string partial = """{"path":"/photos/IMG_0001.HEIC","ok":tr"""; + + // Act + Func<ImmichVerifyLine?> act = () => VerifyTask.ParseLine(partial); + + // Assert + act.Should().NotThrow(); + act().Should().BeNull(); + } + + [Fact] + public void ParseLine_UnknownFields_AreIgnored() + { + // Arrange: a future script revision adding a field must not break an older binary + const string json = """ + {"path":"/photos/a.jpg","ok":true,"via":"decode","durationMs":42,"newField":{"x":1}} + """; + + // Act + ImmichVerifyLine? line = VerifyTask.ParseLine(json); + + // Assert + line.Should().NotBeNull(); + line!.Path.Should().Be("/photos/a.jpg"); + line.Ok.Should().BeTrue(); + } + + // A host path is not a valid container path on Windows, so paths are translated at the boundary. + + [Fact] + public void TryToContainerPath_MapsUnderTheFixedMount() + { + string mount = Path.Combine(Path.DirectorySeparatorChar.ToString(), "photos"); + string file = Path.Combine(mount, "2024", "IMG_0001.HEIC"); + + VerifyTask.TryToContainerPath(mount, file, out string container).Should().BeTrue(); + container.Should().Be("/photocleaner/2024/IMG_0001.HEIC"); + } + + [Fact] + public void TryToContainerPath_FileAtMountRoot_HasNoExtraSeparator() + { + string mount = Path.Combine(Path.DirectorySeparatorChar.ToString(), "photos"); + string file = Path.Combine(mount, "IMG_0001.HEIC"); + + VerifyTask.TryToContainerPath(mount, file, out string container).Should().BeTrue(); + container.Should().Be("/photocleaner/IMG_0001.HEIC"); + } + + [Fact] + public void TryToContainerPath_AlwaysUsesForwardSlashes() + { + string mount = Path.Combine(Path.DirectorySeparatorChar.ToString(), "photos"); + string file = Path.Combine(mount, "a", "b", "c.jpg"); + + VerifyTask.TryToContainerPath(mount, file, out string container).Should().BeTrue(); + + container.Should().StartWith("/photocleaner/"); + container.Should().NotContain("\\"); + } + + // A path outside the mount maps to a container path that leaves it, naming a different file. + [Theory] + [InlineData("elsewhere", "secret.jpg")] + [InlineData("photos-other", "IMG_0001.HEIC")] + public void TryToContainerPath_FileOutsideTheMount_Fails(string sibling, string name) + { + string root = Path.DirectorySeparatorChar.ToString(); + string mount = Path.Combine(root, "photos"); + string file = Path.Combine(root, sibling, name); + + VerifyTask.TryToContainerPath(mount, file, out string container).Should().BeFalse(); + container.Should().BeEmpty(); + } + + // A name merely starting with two dots is an ordinary file, not an escape. + [Fact] + public void TryToContainerPath_NameStartingWithDots_IsMapped() + { + string mount = Path.Combine(Path.DirectorySeparatorChar.ToString(), "photos"); + string file = Path.Combine(mount, "..hidden.jpg"); + + VerifyTask.TryToContainerPath(mount, file, out string container).Should().BeTrue(); + container.Should().Be("/photocleaner/..hidden.jpg"); + } + + // A file that cannot be read is a tooling gap, never damaged media. + // The verdict must not depend on whether a database happens to be configured. + [Fact] + public async Task ExecuteAsync_UnreadableFileWithoutDatabase_CountsFailedNotInvalid() + { + if (!OperatingSystem.IsLinux()) + { + Assert.Skip("File mode permissions are only enforced on Linux"); + return; + } + + if (!ImmichImageAvailable()) + { + Assert.Skip(ImageSkipReason); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + string locked = Path.Combine(workDir, "unreadable.jpg"); + try + { + // Arrange - a decodable file the process cannot open + File.Copy(fixture.SourceFile(TempDirectoryFixture.SmallJpegFile), locked); + File.SetUnixFileMode(locked, UnixFileMode.None); + if (CanRead(locked)) + { + Assert.Skip("Running with permission to read anything, so the file stays readable"); + return; + } + + VerifyTask task = new(CreateOptions(workDir), null, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [locked], + TestContext.Current.CancellationToken + ); + + // Assert + counts.Failed.Should().Be(1); + counts.Invalid.Should().Be(0); + counts.Verified.Should().Be(0); + } + finally + { + if (File.Exists(locked)) + { + File.SetUnixFileMode(locked, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + // Nothing to decode means nothing needs Docker. + // Deliberately ungated on the image, since a host without one is the case this protects. + // An unconditional preflight throws here instead of counting. + [Fact] + public async Task ExecuteAsync_OnlyNonMediaFiles_NeedsNoDocker() + { + string workDir = TempDirectoryFixture.CreateWorkDir(); + try + { + // Arrange - a tree holding nothing this command can verify + string notes = Path.Combine(workDir, "notes.txt"); + string readme = Path.Combine(workDir, "readme.md"); + await File.WriteAllTextAsync(notes, "text", TestContext.Current.CancellationToken); + await File.WriteAllTextAsync(readme, "text", TestContext.Current.CancellationToken); + + VerifyTask task = new(CreateOptions(workDir), null, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [notes, readme], + TestContext.Current.CancellationToken + ); + + // Assert + counts.Ignored.Should().Be(2); + counts.Verified.Should().Be(0); + counts.Invalid.Should().Be(0); + counts.Failed.Should().Be(0); + } + finally + { + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + // A database row matching on size and mtime returns cached hashes without opening the file. + // The hash read therefore cannot be relied on to notice that the file is unreadable. + [Fact] + public async Task ExecuteAsync_UnreadableFileWithCachedHashes_CountsFailedNotInvalid() + { + if (!OperatingSystem.IsLinux()) + { + Assert.Skip("File mode permissions are only enforced on Linux"); + return; + } + + if (!ImmichImageAvailable()) + { + Assert.Skip(ImageSkipReason); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + string target = Path.Combine(workDir, "cached.jpg"); + string dbPath = Path.Combine(workDir, "Verify.db"); + try + { + // Arrange - index the file while it is readable, so the row caches its hashes + File.Copy(fixture.SourceFile(TempDirectoryFixture.SmallJpegFile), target); + await using Database database = new(dbPath); + await database.InitializeAsync(TestContext.Current.CancellationToken); + IndexTask index = new(CreateOptions(workDir), database, new SkippedExtensionTracker()); + _ = await index.IndexFileAsync(target, TestContext.Current.CancellationToken); + + // The row is present and unverified, so the run reaches the decoder without rehashing + File.SetUnixFileMode(target, UnixFileMode.None); + if (CanRead(target)) + { + Assert.Skip("Running with permission to read anything, so the file stays readable"); + return; + } + + VerifyTask task = new(CreateOptions(workDir), database, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [target], + TestContext.Current.CancellationToken + ); + + // Assert + counts.Failed.Should().Be(1); + counts.Invalid.Should().Be(0); + counts.Verified.Should().Be(0); + } + finally + { + if (File.Exists(target)) + { + File.SetUnixFileMode(target, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + [Fact] + public async Task ExecuteAsync_UnreadableDirectory_CountsFailedRatherThanSkipping() + { + if (!OperatingSystem.IsLinux()) + { + Assert.Skip("File mode permissions are only enforced on Linux"); + return; + } + + if (!ImmichImageAvailable()) + { + Assert.Skip(ImageSkipReason); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + string lockedDir = Path.Combine(workDir, "locked"); + string filePath = Path.Combine(lockedDir, "photo.heic"); + try + { + // Arrange + Directory.CreateDirectory(lockedDir); + await File.WriteAllBytesAsync( + filePath, + [.. Enumerable.Range(0, 64).Select(i => (byte)i)], + TestContext.Current.CancellationToken + ); + File.SetUnixFileMode(lockedDir, UnixFileMode.None); + if (File.Exists(filePath)) + { + Assert.Skip("Running with permission to read anything, so the path stays visible"); + return; + } + + VerifyTask task = new(CreateOptions(workDir), null, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [filePath], + TestContext.Current.CancellationToken + ); + + // Assert + counts.Failed.Should().Be(1); + counts.Verified.Should().Be(0); + } + finally + { + if (Directory.Exists(lockedDir)) + { + File.SetUnixFileMode( + lockedDir, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + ); + } + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + // Verify only reads, so a file that vanished mid-run means the tree changed underneath it. + [Fact] + public async Task ExecuteAsync_FileMissingSinceIndexing_CountsFailed() + { + if (!ImmichImageAvailable()) + { + Assert.Skip(ImageSkipReason); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + try + { + // Arrange: a path that was enumerated but is gone by the time it is verified + string missing = Path.Combine(workDir, "gone.heic"); + + VerifyTask task = new(CreateOptions(workDir), null, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [missing], + TestContext.Current.CancellationToken + ); + + // Assert + counts.Failed.Should().Be(1); + counts.Verified.Should().Be(0); + counts.Invalid.Should().Be(0); + } + finally + { + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + // Hashing runs before the structural check when a database is in use. + // An exception there must not abort a run that may span a whole library. + [Fact] + public async Task ExecuteAsync_UnreadableFileWithDatabase_DoesNotAbortTheRun() + { + if (!OperatingSystem.IsLinux()) + { + Assert.Skip("File mode permissions are only enforced on Linux"); + return; + } + + if (!ImmichImageAvailable()) + { + Assert.Skip(ImageSkipReason); + return; + } + + string workDir = TempDirectoryFixture.CreateWorkDir(); + string locked = Path.Combine(workDir, "unreadable.jpg"); + string dbPath = Path.Combine(workDir, "Verify.db"); + try + { + // Arrange - two decodable files either side of one that cannot be opened + string source = fixture.SourceFile(TempDirectoryFixture.SmallJpegFile); + string first = Path.Combine(workDir, "a.jpg"); + string last = Path.Combine(workDir, "b.jpg"); + File.Copy(source, first); + File.Copy(source, last); + File.Copy(source, locked); + File.SetUnixFileMode(locked, UnixFileMode.None); + if (CanRead(locked)) + { + Assert.Skip("Running with permission to read anything, so the file stays readable"); + return; + } + + await using Database database = new(dbPath); + await database.InitializeAsync(TestContext.Current.CancellationToken); + VerifyTask task = new(CreateOptions(workDir), database, new SkippedExtensionTracker()); + + // Act + VerifyTask.Counts counts = await task.ExecuteAsync( + [first, locked, last], + TestContext.Current.CancellationToken + ); + + // Assert - the unreadable file is counted, and the others are still verified + counts.Failed.Should().Be(1); + counts.Verified.Should().Be(2); + } + finally + { + if (File.Exists(locked)) + { + File.SetUnixFileMode(locked, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + TempDirectoryFixture.DeleteWorkDir(workDir); + } + } + + private static bool CanRead(string filePath) + { + try + { + using FileStream stream = File.OpenRead(filePath); + return true; + } + catch (UnauthorizedAccessException) + { + return false; + } + } + + private const string ImageSkipReason = + "Docker or the Immich image is unavailable, and verify has no offline mode"; + + // Verify runs Immich's decoder for every file, so these cases cannot run without the image. + private static bool ImmichImageAvailable() + { + try + { + using System.Diagnostics.Process? probe = System.Diagnostics.Process.Start( + new System.Diagnostics.ProcessStartInfo("docker") + { + // Only the exit code is wanted, so render nothing rather than the manifest. + ArgumentList = { "image", "inspect", "--format", "", VerifyTask.ImmichImage }, + RedirectStandardOutput = true, + RedirectStandardError = true, + } + ); + if (probe is null) + { + return false; + } + + // Redirected pipes are drained before waiting. + // A child that fills one blocks on the write while the wait blocks on the child. + Task<string> output = probe.StandardOutput.ReadToEndAsync(); + Task<string> error = probe.StandardError.ReadToEndAsync(); + if (!probe.WaitForExit(60_000)) + { + // Disposing the probe frees its handles without ending the child. + // A wedged docker would otherwise outlive the test that started it. + try + { + probe.Kill(entireProcessTree: true); + + // Kill only signals, so the child is still alive when it returns. + probe.WaitForExit(5_000); + } + catch (InvalidOperationException) + { + // It exited between the timeout expiring and the kill. + } + + // The kill closes the pipes, so the drains finish rather than being left unobserved. + try + { + Task.WaitAll([output, error], 5_000); + } + catch (AggregateException) + { + // A drain that faulted on the closing pipe is observed here and discarded. + } + + return false; + } + + Task.WaitAll(output, error); + return probe.ExitCode == 0; + } + catch (System.ComponentModel.Win32Exception) + { + return false; + } + } + + private static CommandLine.Options CreateOptions(string path) => + new() + { + Path = new DirectoryInfo(path), + Threads = 1, + DryRun = false, + DatePath = false, + SkipBackup = false, + OutPath = null, + Format = "yyyy/MM/dd", + DeleteEmpty = false, + Move = false, + TagPath = false, + Tags = null, + DbFile = null, + Rehash = false, + ShortVideoDuration = MediaUtilities.ShortVideoDuration, + Reprocess = false, + MarkProcessed = false, + ImmichUrl = null, + ImmichApiKey = null, + TrashDbFile = null, + SkipDbFile = null, + LogOptions = new LoggerFactory.Options + { + Level = LogEventLevel.Information, + File = null, + FileClear = false, + }, + }; + + [Fact] + public void ImmichImage_UsesReleaseTag() + { + // Immich publishes no :latest tag, so a wrong tag here fails preflight on every run. + VerifyTask.ImmichImage.Should().Be("ghcr.io/immich-app/immich-server:release"); + } + + [Fact] + public void VerifyScript_ReferencesImmichOwnModules() + { + // Replacing these requires with a reimplementation would silently lose the fidelity. + ImmichVerifyScript.Verify.Should().Contain("media.repository.js"); + ImmichVerifyScript.Verify.Should().Contain("generateThumbnail"); + ImmichVerifyScript.Verify.Should().Contain("defaults.image.preview"); + ImmichVerifyScript.Preflight.Should().Contain(ImmichVerifyScript.PreflightSentinel); + } +} diff --git a/README.md b/README.md index 81b9575..a9c5722 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,51 @@ # PhotoCleaner <!-- omit from toc --> -An application that prepares photos and videos for import into photo managers. +Utility to prepare photos and videos for import into photo managers. ## Build and Distribution <!-- omit from toc --> -- **Source Code**: [GitHub][photocleaner-link], holding the source, the issues, and the CI/CD pipelines. -- **Versioned Releases**: [GitHub Releases][releases-link], attaching the Linux and Windows executables as a 7z archive. -- **Docker Images**: [Docker Hub][docker-link], multi-arch `linux/amd64` and `linux/arm64`. +- **Source Code**: [GitHub][github-link] for source, issues, and the CI/CD pipelines. +- **Versioned Releases**: [GitHub Releases][releases-link] for pre-compiled executables for Windows, Linux, and macOS. +- **Docker Images**: [Docker Hub][docker-link] for container images with all tools pre-installed. ### Build Status <!-- omit from toc --> -[![Docker Image Size][docker-size-shield]][docker-link]\ -[![License][license-shield]][license-link] +[![Release Status][release-status-shield]][actions-link]\ +[![Docker Status][docker-status-shield]][actions-link]\ +[![Last Commit][last-commit-shield]][commit-link] ### Releases <!-- omit from toc --> -[![Docker Latest][docker-latest-shield]][docker-link] +[![GitHub Release][release-version-shield]][releases-link]\ +[![GitHub Pre-Release][pre-release-version-shield]][releases-link]\ +[![Docker Latest][docker-latest-version-shield]][docker-link]\ +[![Docker Develop][docker-develop-version-shield]][docker-link] ### Release Notes <!-- omit from toc --> -**Version: 1.0**: +**Version 1.1**: **Summary**: -- First published release, carrying the multi-arch Docker image and the GitHub release with the Linux and Windows executables attached. +- Added `verify` command to detect possibly corrupt images, e.g. Immich fails to generate a preview image. +- Added `-validate` to exiftool to report metadata warnings and errors. -See [Release History][history-link] for the full history. +> **Breaking**: commands now exit `2` when they complete with per-file failures. + +See [Release History][history-link] for complete release notes and older versions. ## Table of Contents <!-- omit from toc --> - [Overview](#overview) - [Usage](#usage) - [Command Line Syntax](#command-line-syntax) + - [Exit Codes](#exit-codes) - [Examples](#examples) - [Processing Flow](#processing-flow) - [Undo Flow](#undo-flow) - [Import Flow](#import-flow) - [Trash Flow](#trash-flow) +- [Verify Flow](#verify-flow) - [Supported File Types](#supported-file-types) - [Docker](#docker) - [Workflow Example](#workflow-example) @@ -44,7 +53,6 @@ See [Release History][history-link] for the full history. - [Development Environment Setup](#development-environment-setup) - [Install](#install) - [Update](#update) -- [License](#license) ## Overview @@ -55,12 +63,12 @@ PhotoCleaner analyzes and transforms media files through a validation pipeline t compound extensions (e.g. `photo.heic.jpg` -> `photo.jpg`). - **Renames mixed-case extensions**: Converts uppercase or mixed-case extensions to lowercase (e.g. `.JPG` -> `.jpg`). -- **Handles Live Photos**: Removes Apple Live Photo video components - videos <= 1s are always - removed; videos <= 4s with a candidate companion image (same basename, or basename with `_hevc` - suffix stripped) are removed when both files share the same `ContentIdentifier` EXIF tag; - longer videos with a matching image trigger a warning but are kept. -- **Converts video formats**: Remuxes MTS, M2TS, and MKV to MP4; re-encodes WMV, AVI, 3GP, and - GIF to MP4 (H.264/AAC); re-encodes PCM audio to AAC in MOV and MP4 files while preserving +- **Handles Live Photos**: Removes Apple Live Photo video components. Videos <= 1s are always + removed. Videos <= 4s with a candidate companion image (same basename, or basename with `_hevc` + suffix stripped) are removed when both files share the same `ContentIdentifier` EXIF tag. + Longer videos with a matching image trigger a warning but are kept. +- **Converts video formats**: Remuxes MTS, M2TS, and MKV to MP4, re-encodes WMV, AVI, 3GP, and + GIF to MP4 (H.264/AAC), and re-encodes PCM audio to AAC in MOV and MP4 files while preserving the video stream. All source metadata (including `ContentIdentifier` and other QuickTime tags) is copied to the converted file using `exiftool -TagsFromFile`. - **Imports into date folders** (via `import` command): Copies (default) or moves supported @@ -79,6 +87,12 @@ PhotoCleaner analyzes and transforms media files through a validation pipeline t database. This trash DB can then be used with `import --trashdb` to skip files that were already imported and trashed in Immich, and with `process --trashdb` to delete files trashed in Immich after upload, preventing re-import of known duplicates. +- **Verifies files render in Immich** (via `verify` command): Answers whether a file will survive + Immich's preview generation, which metadata checks cannot predict. Immich's own thumbnail + pipeline is run inside the `immich-server` image, so the verdict is the one Immich will reach + rather than an approximation of it. That decoder is the sole authority: PhotoCleaner does not + parse container formats itself, because a format it did not recognize would be indistinguishable + from a damaged one. Docker is required. Nothing is modified, and `verify` only ever reads. - **Warns on DNG version**: Flags DNG files with a format version newer than v1.4 that may not render correctly in older applications. @@ -94,7 +108,7 @@ detailed logging of all operations. ```text $> PhotoCleaner --help Description: - PhotoCleaner - An application that prepares photos and videos for import into photo managers. + PhotoCleaner - Utility to prepare photos and videos for import into photo managers. Usage: PhotoCleaner [command] [options] @@ -105,6 +119,7 @@ Commands: import Import media files into date-based subdirectories index Index files into the database for deduplication tracking trash Sync trashed asset hashes from Immich + verify Verify that media files can be rendered by Immich Options: --loglevel <Debug|Error|Fatal|Information|Verbose|Warning> Set the log level [default: Information] @@ -120,7 +135,7 @@ Description: Process media files Options: - --path <path> (REQUIRED) The directory path to process + --path <path> (REQUIRED) The media directory path --dryrun Perform a dry run without making changes --threads <threads> Number of parallel threads [default: 4] --skipbackup Skip creating backup files (disables undo) @@ -128,8 +143,8 @@ Options: --db <db> SQLite database file for file state tracking --rehash Force rehashing of all files, ignoring size/mtime cache --duration <duration> Maximum duration in seconds below which a video is considered a short clip and deleted [default: 1] - --reprocess Re-process files even if already marked as processed in the database - --trashdb <trashdb> SQLite database with Immich trash hashes (read-only). Files matching are deleted from disk and the DB + --reprocess Re-run every file even if the database marks it done + --trashdb <trashdb> SQLite database with Immich trash hashes (read-only) ``` ```text @@ -138,7 +153,7 @@ Description: Undo media file processing Options: - --path <path> (REQUIRED) The directory path to process + --path <path> (REQUIRED) The media directory path --dryrun Perform a dry run without making changes ``` @@ -148,11 +163,11 @@ Description: Import media files into date-based subdirectories Options: - --path <path> (REQUIRED) The directory path to process + --path <path> (REQUIRED) The media directory path --dryrun Perform a dry run without making changes --threads <threads> Number of parallel threads [default: 4] --outpath <outpath> (REQUIRED) Output directory for organized files - --format <format> Date format for output subdirectory names [default: yyyy/MM/dd] + --format <format> Date format for output subdirectory names; use '/' to create nested subdirectories (e.g. yyyy/MM/dd) [default: yyyy/MM/dd] --deleteempty Delete empty subdirectories under the target directory after the command completes --move Move files instead of copying (default: copy) --tagpath Apply path sub-directory components as XMP Subject tags to the organized file @@ -170,11 +185,11 @@ Description: Index files into the database for deduplication tracking Options: - --path <path> (REQUIRED) The directory path to index + --path <path> (REQUIRED) The media directory path --threads <threads> Number of parallel threads [default: 4] --db <db> (REQUIRED) SQLite database file for file state tracking --rehash Force rehashing of all files, ignoring size/mtime cache - --processed Mark newly inserted rows as already processed (use when seeding a Process.db) + --processed Mark newly inserted rows as already processed (use when seeding a Process.db from existing files) ``` ```text @@ -189,41 +204,54 @@ Options: --trashdb <trashdb> (REQUIRED) SQLite database to store Immich trash hashes ``` +```text +$> PhotoCleaner verify --help +Description: + Verify that media files can be rendered by Immich + +Options: + --path <path> (REQUIRED) The media directory path + --threads <threads> Number of parallel threads [default: 4] + --db <db> SQLite database file for file state tracking + --rehash Force rehashing of all files, ignoring size/mtime cache + --reprocess Re-run every file even if the database marks it done +``` + **Option notes:** -- `--path` - must point to an existing directory; accepts exactly one directory per command +- `--path`: must point to an existing directory. Accepts exactly one directory per command invocation. -- `--threads` - defaults to `min(CPU count, 4)`; must be `> 0` and `<= CPU count`. -- `--skipbackup` - opt-in (`process` only); skips all `.bak` file creation. The `undo` +- `--threads`: defaults to `min(CPU count, 4)`. Must be `> 0` and `<= CPU count`. +- `--skipbackup`: opt-in (`process` only). Skips all `.bak` file creation. The `undo` command cannot reverse a run made with this flag. -- `--outpath` - required for `import`; target directory (created on demand). -- `--format` - optional (`import` only); a C# date format string used to name date - subdirectories (default `"yyyy/MM/dd"`). Must be date-only - time components are rejected. +- `--outpath`: required for `import`. Target directory (created on demand). +- `--format`: optional (`import` only). A C# date format string used to name date + subdirectories (default `"yyyy/MM/dd"`). Must be date-only, so time components are rejected. Files with no EXIF date land in a `"0001/01/01"` fallback bucket. -- `--deleteempty` - optional (`import`, `process`); after the command completes, deletes +- `--deleteempty`: optional (`import`, `process`). After the command completes, deletes empty child subdirectories from the target directory (deepest first). For `import` the - target is `--outpath`; for `process` it is `--path` (which is operated on in-place). The + target is `--outpath`, and for `process` it is `--path` (which is operated on in-place). The target root itself is never deleted. Useful for cleaning up directory trees left behind after `process` deletes files (live photos, originals when `--skipbackup`) or after pruning organized output. -- `--move` - optional (`import` only); moves files instead of copying. Default behavior is +- `--move`: optional (`import` only). Moves files instead of copying. Default behavior is to copy, which preserves the source files. Use `--move` when the source directory is temporary. -- `--tagpath` - optional (`import` only); splits the source sub-directory path relative to +- `--tagpath`: optional (`import` only). Splits the source sub-directory path relative to `--path` into tokens and writes each token as an `XMP:Subject` tag on the destination file using exiftool. Files at the root of `--path` receive no tags. Tags are applied with a remove-then-add pattern (`-XMP:Subject-= / -XMP:Subject+=`) so existing tags are preserved and duplicates are not created. Only file types that support XMP writes are tagged. -- `--tags` - optional (`import` only); a comma-separated list of `XMP:Subject` tags applied to +- `--tags`: optional (`import` only). A comma-separated list of `XMP:Subject` tags applied to every organized file (e.g. `--tags "vacation,family trip,2018"`). Tags are applied using the - same remove-then-add pattern as `--tagpath`. Can be combined with `--tagpath` - both sets of + same remove-then-add pattern as `--tagpath`. Can be combined with `--tagpath`, and both sets of tags are merged. Only file types that support XMP writes are tagged. -- `--datepath` - optional (`import` only); when a file has no embedded creation date, infers +- `--datepath`: optional (`import` only). When a file has no embedded creation date, infers one from the filename or directory path structure (via `DateFromPath`) and writes it to the destination file before restoring mtime. Opt-in because writing to files is destructive and the source path context is only available during `import` (before files move to date-based directories). -- `--db <path>` - optional for `import` and `process`, **required** for `index`; path to a +- `--db <path>`: optional for `import` and `process`, **required** for `index`. Path to a SQLite database file. Uses a single `files` table (`path` PRIMARY KEY, `sha256`, `sha1`, `file_size`, `mtime_ticks`, `is_processed`). The schema is the same for every command, but the **meaning of the `path` column depends on which command writes the row**, so each @@ -237,14 +265,14 @@ Options: The DB file is created automatically on first use. **The two stages MUST use separate DB files**: if `import` and `process` shared one DB, `process` would overwrite the source-content hashes that `import` wrote (because `import` mutates the dest file via XMP tag injection, so the dest - hash diverges from the source hash; `process` then re-hashes the dest and clobbers the row). - This was a real bug; the per-stage layout is the fix. -- `--trashdb <path>` - **required** for `trash`, optional for `import` and `process`; path + hash diverges from the source hash, and `process` then re-hashes the dest and clobbers the row). + This was a real bug, and the per-stage layout is the fix. +- `--trashdb <path>`: **required** for `trash`, optional for `import` and `process`. Path to a SQLite database with Immich trash hashes. - For `trash`: hashes are fetched from the Immich API and written to the database. - For `import`: files matching a trash hash are skipped (read-only). This is the durable "do not re-import" record beyond Immich's own ~30-day trash retention. Without it, a file - the user trashed > 30 days ago can come back the next time icloudpd re-downloads it, because + the user trashed > 30 days ago can come back the next time the downloader re-fetches it, because Immich no longer has the hash to deduplicate against on re-upload. Limitation: `import` compares the **source-file** SHA-1 against the trash DB. When `import` rewrites the destination via `--tags`, `--tagpath`, or `--datepath`, the dest file's @@ -256,30 +284,53 @@ Options: cleans up files trashed in Immich after upload, before the next `immich-cli` upload would re-upload them. Also acts as the safety net for files that `import --trashdb` could not match because of the source-vs-dest SHA-1 drift described above. -- `--skipdb <path>` - optional for `import`; path to a SQLite database with indexed files +- `--skipdb <path>`: optional for `import`. Path to a SQLite database with indexed files to be skipped (read-only). Files matching a record in this DB are skipped without being recorded. Use to skip files already present in another collection. -- `--url <url>` - **required** for `trash`; the Immich server URL (e.g. `http://immich:2283`). -- `--apikey <key>` / `--apikey-file <path>` - the Immich API key for `trash`; supply it via +- `--url <url>`: **required** for `trash`. The Immich server URL (e.g. `http://immich:2283`). +- `--apikey <key>` / `--apikey-file <path>`: the Immich API key for `trash`. Supply it via exactly one of these two mutually exclusive options (one is required). Create the key in Immich - under Account Settings > API Keys. `--apikey` passes the key inline; `--apikey-file` points to a + under Account Settings > API Keys. `--apikey` passes the key inline, while `--apikey-file` points to a file whose trimmed contents are the key, keeping the secret out of shell history and process arguments. The file must exist and be non-empty. -- `--rehash` - optional (`process`, `import`, `index`); forces SHA-256 recomputation for +- `--rehash`: optional (`process`, `import`, `index`). Forces SHA-256 recomputation for every file, bypassing the size/mtime cache. SHA-1 is also recomputed when `--trashdb` is in use. Use when file content may have changed without the modification timestamp being updated. -- `--duration` - optional (`process` only); overrides the short-video deletion threshold +- `--duration`: optional (`process` only). Overrides the short-video deletion threshold (default `1.0` seconds). Videos in a live-photo-compatible format whose duration is <= this value are always deleted. Must be `> 0`. -- `--reprocess` - optional (`process` only); when set, ignores the `is_processed` flag in +- `--reprocess`: optional (`process`, `verify`). When set, ignores the `is_processed` flag in the database and processes every file regardless of prior run history. Useful after changing pipeline settings (e.g. `--duration`) without wiping the database. -- `--processed` - optional (`index` only); marks newly inserted rows with `is_processed = 1`. +- `--processed`: optional (`index` only). Marks newly inserted rows with `is_processed = 1`. Use this when seeding a Process.db from existing files so a subsequent `process` run treats them as already processed and only touches new arrivals. The flag does not flip the bit on rows that already exist in the DB. +### Exit Codes + +Every command uses the same three codes, so a pipeline can branch on the result without parsing +logs: + +| Code | Meaning | +| ---- | ------- | +| `0` | Success. The command completed and every file succeeded. | +| `1` | Error. The command could not complete: unhandled exception, fatal configuration error, cancellation, or a failed `verify` preflight. | +| `2` | Completed with failures. The command ran to completion, but one or more files failed or failed verification, or `trash` synced only part of the server. | + +The distinction between `1` and `2` matters most for `verify`. A `1` means the check itself could +not run at all, because Docker was unreachable or the Immich image could not be prepared, and it +says nothing about any file. A `2` means the run completed and one or more files were invalid **or could +not be verified**, the latter covering a file that could not be read or that no verdict came back +for. A script must never treat an infrastructure failure as a verdict on the collection, and should +read the `Invalid` and `Failed` counts to tell a bad file from a gap in coverage. + +`trash` exits `2` when pagination stops early, which leaves the trash database holding fewer +hashes than the server. That database is used by `import --trashdb` and `process --trashdb` to skip +files, so a short one silently re-imports assets that were trashed, and a pipeline gating on the +exit code should not go on to upload against it. + ### Examples ```bash @@ -346,21 +397,32 @@ PhotoCleaner import --path /home/user/Photos --outpath /home/user/Organized --db # Import and skip files already in another collection (read-only reference) PhotoCleaner import --path /home/user/Photos --outpath /home/user/Organized --skipdb /data/existing-collection.db +# Verify that Immich can render every file, before uploading +PhotoCleaner verify --path /home/user/Intermediate --db /data/verify.db + # Full workflow with Immich trash integration PhotoCleaner trash --url http://immich:2283 --apikey $IMMICH_KEY --trashdb /data/trash.db PhotoCleaner import --path /home/user/iCloud --outpath /home/user/Intermediate --db /data/photos.db --trashdb /data/trash.db PhotoCleaner process --path /home/user/Intermediate --db /data/process.db +PhotoCleaner verify --path /home/user/Intermediate --db /data/verify.db ``` ## Processing Flow 1. **File enumeration**: Recursively scans all specified directories. 2. **Case conflict detection**: Identifies files with the same name but different casing that - would collide on case-insensitive file systems; renames conflicting files before processing. + would collide on case-insensitive file systems, then renames them before processing. 3. **Per-file validation pipeline** (runs in parallel, stops on first action per file): - 1. Rename to canonical MIME extension - corrects mismatches and strips compound extensions. + 0. Act on the exiftool `-validate` verdict, which rides along with the metadata read that + `process` already performs and so costs nothing extra. A file exiftool reports **errors** + on is marked invalid and no further step touches it. **Warnings are logged at debug level + and nothing more**: measured across a real collection, roughly three quarters of perfectly + healthy files carry at least one (odd IFD offsets, non-standard maker note tags, short + IPTC fields), so failing on warnings would condemn most of a library. This is a cheap net + for a rare case, not a substitute for the `verify` command. + 1. Rename to canonical MIME extension, correcting mismatches and stripping compound extensions. 2. Rename mixed-case extension to lowercase. - 3. Delete short or Live Photo video clips: videos <= 1s are always deleted; videos <= 4s + 3. Delete short or Live Photo video clips. Videos <= 1s are always deleted, and videos <= 4s with a candidate companion image (direct name match or `_hevc`-suffix match) are deleted when both files share a matching `ContentIdentifier` tag. 4. Convert legacy or incompatible video formats to MP4: @@ -370,13 +432,16 @@ PhotoCleaner process --path /home/user/Intermediate --db /data/process.db - After every conversion: all source metadata copied to output via `exiftool -TagsFromFile` 5. Warn on DNG version > v1.4. 4. **Reprocess loop**: Any file that was renamed or converted is re-queued until stable. -5. **Results summary**: Reports counts of failed, modified, and successfully processed files; - lists any unrecognized file extensions. +5. **Results summary**: Reports counts of failed, invalid, modified, and successfully processed + files, and lists any unrecognized file extensions. Exits `2` if anything failed or was invalid. + +The `import` command applies the same exiftool validation, skipping any file that reports errors +rather than pulling it into the collection. ## Undo Flow Every file modification or deletion made by `process` creates a `.bak` backup alongside the -original: the first backup is `X.bak`; if that already exists (from a prior run) the next is +original: the first backup is `X.bak`, and if that already exists (from a prior run) the next is `X.bak1`, then `X.bak2`, etc. The `undo` command reverses all processing by scanning the given directories for backup files and applying a two-pass algorithm: @@ -389,7 +454,7 @@ directories for backup files and applying a two-pass algorithm: - *Non-derived* base: delete the current file if it exists (overwritten in-place), rename `X.bak` -> `X` to restore the original. The converted output is located via the `X.bak.out` companion file written at conversion time (handles uniquified names like - `stem_1.mp4`); if no companion exists, falls back to deleting `stem.mp4` when present + `stem_1.mp4`). If no companion exists, falls back to deleting `stem.mp4` when present and untracked (legacy single-run heuristic). **Known limitation**: extension renames that target a filename that did not previously exist @@ -412,7 +477,7 @@ directories to `outpath/date/filename`: - `--trashdb`: if the file matches a hash in the Immich trash DB, the file is skipped (counted as "trashed in Immich"). - `--skipdb`: if the file matches a record in the reference DB, the file is skipped - (counted as "skipped by reference"). This is a read-only check - no records are written. + (counted as "skipped by reference"). This is a read-only check, so no records are written. - `--db` (Import.db): if the source SHA-256 is already present (from a previous import run), the file is skipped (counted as "skipped"). Otherwise, the file is copied/moved and a record is inserted **keyed by the SOURCE path** (not the dest path) with the source @@ -421,15 +486,15 @@ directories to `outpath/date/filename`: Process.db cannot clobber the dedup key. The DB file is created automatically on first use. 3. **Date resolution**: reads EXIF metadata via `exiftool`. Uses `EXIF:DateTimeOriginal` or `QuickTime:CreateDate` (whichever is set). Falls back to `DateTime.MinValue` when no date - is found - those files land in a `"0001/01/01"` bucket (with the default `yyyy/MM/dd` format), + is found. Those files land in a `"0001/01/01"` bucket (with the default `yyyy/MM/dd` format), making undated files easy to locate and handle manually. 4. **Subdirectory naming**: the date is formatted using `--format` (default `"yyyy/MM/dd"`). - The format is validated at startup - time components are rejected. + The format is validated at startup, and time components are rejected. 5. **Copy or move**: by default files are copied and the source is preserved. Pass `--move` to remove the source file after a successful copy. 6. **Tagging**: `--tagpath` splits the source sub-directory path relative to `--path` into tokens and writes each as an `XMP:Subject` tag. `--tags` applies explicit comma-separated - tags to every file. Both can be combined - tags are merged and deduplicated. Applied after + tags to every file. Both can be combined, and tags are merged and deduplicated. Applied after copy/move, before mtime restore. Files at the root of `--path` receive no path tags. 7. **Collision handling**: if a file with the same name already exists in the destination, `_1`, `_2`, ... suffixes are appended (e.g. `photo_1.jpg`). A warning is logged. @@ -459,6 +524,50 @@ were already imported and trashed in Immich. The trash database is append-only. If an asset is restored (un-trashed) in Immich, its hash remains in the database. Delete the database file and re-run `trash` to rebuild from scratch. +## Verify Flow + +The `verify` command answers one question: will Immich be able to generate a preview for this +file? It exists because a file can be byte-complete, pass every other check, upload successfully, +and then fail thumbnail generation forever. Metadata inspection cannot see this, because the file +reports as a perfectly clean HEIC or DNG, so the only reliable answer comes from running the decoder +Immich runs. + +`verify` is a standalone pipeline step rather than an option on `process`, so the calling script +chooses where to run it and whether a failure should stop the pipeline or merely be recorded. + +1. **Partition**: non-media files are ignored. With `--db`, files already verified and unchanged + since are skipped, so a repeat run over a large collection is cheap. + + **Give `verify` its own database file.** It records the verified state in the same + `is_processed` column that `process` writes, so pointing `--db` at a `Process.db` makes + `verify` skip every file as "already verified" when they were only processed. Nothing detects + this, so use a separate `Verify.db`, as the examples below do. +2. **Decode pass** (requires Docker). Every file is handed to + Immich's own `MediaRepository` running inside the `immich-server` image, using the same + `generateThumbnail`, the same libvips build, the same libheif and libraw versions, and for RAW + the same embedded-preview extraction. Paths are streamed in batches over stdin so container + startup is paid once per batch rather than once per file. The media directory is mounted + read-only at the fixed container path `/photocleaner`, and every path is translated onto it + before being sent in, so a host path that is not a valid container path still works. + + The decoder is the only judge of a file's health. PhotoCleaner deliberately carries no + container parser of its own, because such a parser condemns whatever it fails to understand, + and an unfamiliar but valid format is indistinguishable from a damaged one from the inside. + A file that cannot be read, or that vanishes mid-run, counts as failed rather than invalid, + since neither is evidence of damage. +3. **Report**: logs each rejection with the decoder's own message, then a summary. Exits `2` if + any file is invalid or any file failed. + +Nothing is modified, moved, or deleted. `verify` only ever reads. + +Because the decode pass calls Immich's own compiled code rather than reimplementing its pipeline, +it tracks Immich's behavior across releases automatically. It runs `docker` directly, so it must +be run somewhere Docker is available, and is not supported from inside PhotoCleaner's own +container. There is no offline mode, because the decoder is the whole check. + +Before any file is judged, the command runs a preflight against the image. If Docker is +unreachable or the image cannot be prepared, it exits `1` without marking a single file invalid. + ## Supported File Types - **Images**: ARW, CR2, DNG, HEIC, HEIF, JPEG, JPG, NEF, ORF, PNG, PSD, RW2, TIF, TIFF @@ -525,29 +634,89 @@ docker run --rm \ ## Workflow Example -**Run [icloudpd][icloudpd-link] to download photos from iCloud**: +**Run [kei][kei-link] to download photos from iCloud**: + +kei keeps its settings in a TOML file rather than on the command line, so the same configuration +serves the one-shot and the long-running forms. A minimal `config.toml`: + +```toml +[auth] +username = "your@icloud.email" + +[download] +directory = "/photos" +folder_structure = "%Y/%m/%Y-%m-%d" + +[filters] +media = ["photos", "videos", "live-photos"] + +[photos] +raw_policy = "prefer-raw" + +[watch] +interval = 86400 +``` + +Authenticate once, interactively, so the session is stored in the data directory: ```shell #!/bin/bash set -Eeuo pipefail -docker run -it --rm --name icloudpd \ - -v $(pwd)/.icloudpd:/cookies \ - -v /data/media/Test:/data \ - -e TZ=America/Los_Angeles \ - docker.io/icloudpd/icloudpd:latest \ - icloudpd \ - --cookie-directory /cookies \ - --username your@icloud.email \ - --directory /data \ - --set-exif-datetime \ - --folder-structure "{:%Y/%m}" \ - --recent 1000 - # --skip-created-before 2025-01-01 +docker run -it --rm --name kei \ + -v /data/appdata/kei/config:/config \ + -v /data/media/icloud:/photos \ + -e KEI_DATA_DIR=/config \ + ghcr.io/rhoopr/kei:latest \ + kei login ``` -**Run [PhotoCleaner][photocleaner-link] to sync Immich trash hashes** (optional, prevents re-importing trashed files): +Then sync on demand: + +```shell +#!/bin/bash + +set -Eeuo pipefail + +docker run -it --rm --name kei \ + -v /data/appdata/kei/config:/config \ + -v /data/media/icloud:/photos \ + -e KEI_DATA_DIR=/config \ + ghcr.io/rhoopr/kei:latest \ + kei sync \ + --config /config/config.toml \ + --recent 30d + # --dry-run to preview without writing +``` + +Or run it as a service that keeps mirroring on the `[watch]` interval: + +```yaml +services: + kei: + image: ghcr.io/rhoopr/kei:latest + container_name: kei + restart: unless-stopped + environment: + - TZ=America/Los_Angeles + - KEI_DATA_DIR=/config + volumes: + - /data/media/icloud:/photos + - /data/appdata/kei/config:/config + secrets: + - icloud_password + command: + - kei + - service + - run + - --config + - /config/config.toml + - --password-file + - /run/secrets/icloud_password +``` + +**Run PhotoCleaner to sync [Immich][immich-link] trash hashes** (optional, prevents re-importing trashed files): ```shell #!/bin/bash @@ -563,10 +732,10 @@ docker run --rm \ --trashdb /db/trash.db ``` -**Run [PhotoCleaner][photocleaner-link] to import new photos**: +**Run PhotoCleaner to import new photos**: -Copy only new files (not already in the DB and not trashed in Immich) from the icloudpd -directory to an intermediate directory, without touching the icloudpd originals: +Copy only new files (not already in the DB and not trashed in Immich) from the download +directory to an intermediate directory, without touching the downloaded originals: ```shell #!/bin/bash @@ -586,7 +755,7 @@ docker run --rm \ --threads 4 ``` -**Run [PhotoCleaner][photocleaner-link] to process the intermediate photos**: +**Run PhotoCleaner to process the intermediate photos**: ```shell #!/bin/bash @@ -601,6 +770,31 @@ docker run --rm \ --threads 4 ``` +**Run PhotoCleaner to verify Immich can render the photos**: + +Run this on the host rather than inside the PhotoCleaner container: the decode pass invokes +`docker` to run Immich's own decoder, and Docker-in-Docker is not supported. Capture the exit +code instead of letting `set -e` abort, so the upload can be skipped while the run still reports +cleanly: + +```shell +#!/bin/bash + +set -Eeuo pipefail + +rc=0 +PhotoCleaner verify \ + --path /data/media/intermediate \ + --db /data/media/verify.db \ + --threads 4 || rc=$? + +case $rc in + 0) echo "All files verified" ;; + 2) echo "Invalid files found - skipping upload, review the log" >&2; exit 2 ;; + *) echo "Verification could not run (exit $rc) - this says nothing about the files" >&2; exit "$rc" ;; +esac +``` + **Run [Immich CLI][immich-cli-link] to import photos into Immich**: ```shell @@ -649,7 +843,7 @@ immich-go upload from-folder --server=https://your.immich.server \ ## Questions or Issues -Report a bug, request a feature, or ask a question in [GitHub Issues][issues-link]. +Ask questions in the [Discussions][discussions-link] forum and report bugs in [GitHub Issues][issues-link]. ## Development Environment Setup @@ -706,28 +900,40 @@ dotnet tool update --all dotnet outdated --upgrade:prompt ``` -## License +## License <!-- omit from toc --> Licensed under the [MIT License][license-link]\ ![GitHub License][license-shield] <!-- Shields --> -[docker-latest-shield]: https://img.shields.io/docker/v/ptr727/photocleaner/latest?logo=docker&label=Docker%20Latest -[docker-size-shield]: https://img.shields.io/docker/image-size/ptr727/photocleaner/latest?logo=docker&label=Image%20Size +[docker-develop-version-shield]: https://img.shields.io/docker/v/ptr727/photocleaner/develop?label=Docker%20Develop&logo=docker&color=orange +[docker-latest-version-shield]: https://img.shields.io/docker/v/ptr727/photocleaner/latest?label=Docker%20Latest&logo=docker +[docker-status-shield]: https://img.shields.io/github/actions/workflow/status/ptr727/PhotoCleaner/publish-release.yml?event=schedule&logo=github&label=Docker%20Build +[last-commit-shield]: https://img.shields.io/github/last-commit/ptr727/PhotoCleaner?logo=github&label=Last%20Commit [license-shield]: https://img.shields.io/github/license/ptr727/PhotoCleaner?label=License +[pre-release-version-shield]: https://img.shields.io/github/v/release/ptr727/PhotoCleaner?include_prereleases&label=GitHub%20Pre-Release&logo=github +[release-status-shield]: https://img.shields.io/github/actions/workflow/status/ptr727/PhotoCleaner/publish-release.yml?event=schedule&logo=github&label=Releases%20Build +[release-version-shield]: https://img.shields.io/github/v/release/ptr727/PhotoCleaner?logo=github&label=GitHub%20Release <!-- Repo --> [history-link]: ./HISTORY.md [license-link]: ./LICENSE -<!-- External --> +<!-- Internal --> +[actions-link]: https://github.com/ptr727/PhotoCleaner/actions +[commit-link]: https://github.com/ptr727/PhotoCleaner/commits/main +[discussions-link]: https://github.com/ptr727/PhotoCleaner/discussions [docker-link]: https://hub.docker.com/r/ptr727/photocleaner -[icloudpd-link]: https://icloud-photos-downloader.github.io -[immich-cli-link]: https://docs.immich.app/features/command-line-interface/ -[immich-go-link]: https://github.com/simulot/immich-go +[github-link]: https://github.com/ptr727/PhotoCleaner [issues-link]: https://github.com/ptr727/PhotoCleaner/issues -[photocleaner-link]: https://github.com/ptr727/PhotoCleaner [releases-link]: https://github.com/ptr727/PhotoCleaner/releases + +<!-- External --> + +[kei-link]: https://github.com/rhoopr/kei +[immich-link]: https://immich.app +[immich-cli-link]: https://docs.immich.app/features/command-line-interface +[immich-go-link]: https://github.com/simulot/immich-go diff --git a/WORKFLOW.md b/WORKFLOW.md index d1d9889..6e6aed6 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -2,13 +2,13 @@ The guide for CI/CD **workflows** (GitHub Actions): a deliberate mixture of code style, architecture, a **behavioral contract** (expected inputs and outputs), and a **test methodology**. Code style lives in [`CODESTYLE.md`][codestyle]. This file is its sibling for everything under [`.github/workflows/`][workflows]. -Its defining principle: **it describes required outcomes, not a required implementation.** Two repos may implement the same guarantee with different YAML. A workflow is correct when it **satisfies the contract** in section 4 and is **defect-free against the expected inputs and outputs** - not when it matches a reference implementation byte for byte. The conventions in section 2 keep workflows legible. The contract in section 4 is what they must *do*. +Its defining principle: **it describes required outcomes, not a required implementation.** Two repos may implement the same guarantee with different YAML. A workflow is correct when it **satisfies the contract** in section 4 and is **defect-free against the expected inputs and outputs**, not when it matches a reference implementation byte for byte. The conventions in section 2 keep workflows legible. The contract in section 4 is what they must *do*. Given this document, an agent must be able to do three things to any project: 1. **Audit** - statically check the workflows against the conventions (section 2) and the structural facts each guarantee implies (section 5A). 2. **Test** - trace the expected inputs/outputs (section 5B) and, where warranted, drive a live probe (section 5C). -3. **Assess** - render a verdict: **operational** (every *applicable* guarantee holds and every *applicable* scenario's observed output equals the expected) or **not operational** (any mismatch - which is a *defect*, not a style nit). +3. **Assess** - render a verdict: **operational** (every *applicable* guarantee holds and every *applicable* scenario's observed output equals the expected) or **not operational** (any mismatch, which is a *defect*, not a style nit). > **Canonical scope.** This document is authoritative for the workflow contract and test methodology (sections 3 to 6). The conventions in section 2 and the release policy also live in `GOVERNANCE.md` ("Workflow YAML Conventions" and "Release Model"), which is authoritative where the two overlap. Section 2 restates them so this file reads on its own. On any conflict in that overlap, `GOVERNANCE.md` wins. @@ -17,7 +17,7 @@ The guarantees are distilled from failures observed in practice and stated as th ## 1. Purpose and How to Use This Document - **Contract, not implementation.** Conform to the *outcomes* in section 4. Shape, job names, and file layout may differ between repos, but the input/output behavior may not. -- **Applicability.** A guarantee (or a 5A check, or a 5B scenario) is **applicable** only if the repo contains the construct it governs - a given target, a transfer artifact, a registry push, a wrapper-version source. An item that governs an absent construct is **N/A**: record it as N/A and **exclude it from the verdict**. N/A is never a defect. Section 6 names which items go N/A per project type. A near-empty pipeline (source-only) is mostly N/A and that is fine. +- **Applicability.** A guarantee (or a 5A check, or a 5B scenario) is **applicable** only if the repo contains the construct it governs: a given target, a transfer artifact, a registry push, a wrapper-version source. An item that governs an absent construct is **N/A**: record it as N/A and **exclude it from the verdict**. N/A is never a defect. Section 6 names which items go N/A per project type. A near-empty pipeline (source-only) is mostly N/A and that is fine. - **Operational is binary.** A workflow is operational only if every *applicable* guarantee holds. A single applicable input/output mismatch is a defect and makes the workflow non-operational, regardless of how clean the YAML looks. - **Default branch.** Guarantees say "default branch" portably. It is implemented as the literal `main` in several places (the validate gate, the `prerelease` expression, and `version.json`'s `publicReleaseRefSpec`). These MUST all reference the repo's *actual* default branch. A divergence is a defect (section 5A). - **Two layers when auditing.** The pipeline splits into an **orchestrator** layer (the PR entry workflow, the publisher, and the version/release/badge jobs) and a **build-leaf** layer (`build-<target>-task.yml`). Inputs like `github`/`nuget`/`dockerhub`/`expect_release_assets` live on the orchestrator. A leaf only ever receives `ref`/`branch`/`smoke` (and a derived `push`). When a check names an input, assert it in the layer that declares it. @@ -30,9 +30,9 @@ Prescriptive style/legibility rules. Cheap to check, necessary but not sufficien - **Action pinning.** Pin **every** action to a commit SHA with a trailing `# vX.Y.Z` comment. Use `# vX` only when the upstream floating major tag has no specific patch SHA. The single documented no-pin exception is a tool whose tag stream lags `master` such that tag-tracking would downgrade (here, `dotnet/nbgv@master`). Invent no others. - **Filename.** Reusable workflows (`on: workflow_call`) end in `-task.yml`. Entry-point workflows do not (`-pull-request.yml`, `-release.yml`). Lowercase, hyphen-separated. - **Workflow `name:`.** Reusable names end in **"task"**. Entry-point names end in **"action"**. -- **Job and step `name:`.** Every job ends in **"job"**, every step in **"step"** - including a ruleset-bound required-check job, whose `name:` and the ruleset `context:` are one string renamed together (never independently). +- **Job and step `name:`.** Every job ends in **"job"**, every step in **"step"**, including a ruleset-bound required-check job, whose `name:` and the ruleset `context:` are one string renamed together (never independently). - **Concurrency.** Top-level workflows declare `concurrency: { group: '${{ github.workflow }}-${{ github.ref }}', cancel-in-progress: true }`. Document exceptions inline (D7). -- **Shells.** Every multi-line bash `run:` - and every committed `.sh` script - starts `set -Eeuo pipefail`. +- **Shells.** Every multi-line bash `run:` (and every committed `.sh` script) starts `set -Eeuo pipefail`. - **Conditionals.** Multi-line `if:` uses the folded scalar `if: >-`. - **Boolean inputs.** A boolean used by both `workflow_call` and `workflow_dispatch` is declared in **both** trigger blocks, and `workflow_dispatch` delivers the **string** `"true"`/`"false"`, so any `if:` compares both forms: `${{ inputs.foo == true || inputs.foo == 'true' }}`. - **Reusable-workflow permissions.** Job-level `permissions:` are validated **before** `if:`, so even a skipped job needs valid permissions. Grant least privilege. A reusable callee's extra scope (e.g. `actions: write` for cleanup) is granted by the **caller**. @@ -53,7 +53,7 @@ flowchart LR main -.->|no back-merge| develop ``` -`operational` repos (live-service config; `workflowModel: operational`) commit directly to `develop` and promote a known-good snapshot to `main` via an occasional PR: +`operational` repos (live-service config, `workflowModel: operational`) commit directly to `develop` and promote a known-good snapshot to `main` via an occasional PR: ```mermaid flowchart LR @@ -61,13 +61,13 @@ flowchart LR develop -->|merge commit, enforced lint CI| main ``` -Their CI is lint/validation only (editorconfig/EOL plus domain linters - Home Assistant or ESPHome config validation, a firmware build - **no unit tests**), so the D-guarantees below that assume a build/test pipeline are **N/A** exactly as for `source-only` (Section 6). What binds: the promotion gate - the `develop -> main` PR must pass the required `Check pull request workflow status job` - and the source-only release on manual dispatch (`releaseTrigger: dispatch-only`; tag + source zip). Branch-model rulesets are specified in [GOVERNANCE.md "Branching Model"][governance-branching-model] and [repo-config/README.md][repo-config-readme], not here. +Their CI is lint/validation only (editorconfig/EOL plus domain linters such as Home Assistant or ESPHome config validation or a firmware build, but **no unit tests**), so the D-guarantees below that assume a build/test pipeline are **N/A** exactly as for `source-only` (Section 6). What binds: the promotion gate, where the `develop -> main` PR must pass the required `Check pull request workflow status job`, and the source-only release on manual dispatch (`releaseTrigger: dispatch-only`; tag + source zip). Branch-model rulesets are specified in [GOVERNANCE.md "Branching Model"][governance-branching-model] and [repo-config/README.md][repo-config-readme], not here. ### Two Layers: Orchestration vs Build - **Orchestration** is generic and forms the standardization baseline **at the job level**: the single-branch publisher, the `get-version`, `validate-release`, and `github-release` jobs, the date-badge job, and the `changes -> smoke-build -> aggregator` shape of the PR workflow. These job *bodies* should not need per-repo edits. - **Build** is repo-owned: the `build-<target>-task.yml` leaf tasks. -- **What the repo curates** (by design, not a leak): the *list* of targets. This is **not** a byte-for-byte file carry. Adding or dropping a target edits the orchestrator's surface - the `enable_<target>` inputs and the `build-<target>` job + its `github-release` `needs:` entry in the release task, **and** the `changes` paths-filter entry + output + the `smoke-build` enable-forward in the PR workflow. "Verbatim" applies to the `github-release` job and the version/publish-plan logic, not to the release task's job list or the paths-filter. Subsetting is symmetric: the same surface you trim to drop a target you extend to add a new one (e.g. a `release-asset-<branch>-library` producer needs a new `enable_library` input, a `build-library` job, a `needs:` entry, and a `library` paths-filter). +- **What the repo curates** (by design, not a leak): the *list* of targets. This is **not** a byte-for-byte file carry. Adding or dropping a target edits the orchestrator's surface: the `enable_<target>` inputs and the `build-<target>` job + its `github-release` `needs:` entry in the release task, **and** the `changes` paths-filter entry + output + the `smoke-build` enable-forward in the PR workflow. "Verbatim" applies to the `github-release` job and the version/publish-plan logic, not to the release task's job list or the paths-filter. Subsetting is symmetric: the same surface you trim to drop a target you extend to add a new one (e.g. a `release-asset-<branch>-library` producer needs a new `enable_library` input, a `build-library` job, a `needs:` entry, and a `library` paths-filter). ### The Seam Contract @@ -87,7 +87,7 @@ Every leaf and the release task take `ref`, `branch` (the **logical** branch tha ### Versioning -NBGV versions the branch being published. Each run builds a single branch (the trigger ref), so `GITHUB_REF` already names it and NBGV classifies it directly - no `IGNORE_GITHUB_REF` override is required. The default branch is the public-release ref, so it builds clean `X.Y.Z`. Every other branch builds a prerelease `X.Y.Z-g<sha>`. `version.json`'s `version` is the major.minor floor. NBGV appends the git height as the patch. **NBGV and `version.json` are retained even by a repo with no compiled code** - they are the source of the release tag (`SemVer2`) and `target_commitish` (`GitCommitId`) and the prerelease classification. The .NET SDK is pulled in only as the versioning toolchain. A package build derives its registry version from the same NBGV outputs, but **not always from `SemVer2`**: the PyPI version is built from `AssemblyFileVersion` (four-part `M.N.P.B`) with a PEP 440 `.dev0` appended on the `develop` branch. A wrapper repo may drive its build/image version from an external committed `name -> version` state file while NBGV still tags the release. +NBGV versions the branch being published. Each run builds a single branch (the trigger ref), so `GITHUB_REF` already names it and NBGV classifies it directly, and no `IGNORE_GITHUB_REF` override is required. The default branch is the public-release ref, so it builds clean `X.Y.Z`. Every other branch builds a prerelease `X.Y.Z-g<sha>`. `version.json`'s `version` is the major.minor floor. NBGV appends the git height as the patch. **NBGV and `version.json` are retained even by a repo with no compiled code**, since they are the source of the release tag (`SemVer2`) and `target_commitish` (`GitCommitId`) and the prerelease classification. The .NET SDK is pulled in only as the versioning toolchain. A package build derives its registry version from the same NBGV outputs, but **not always from `SemVer2`**: the PyPI version is built from `AssemblyFileVersion` (four-part `M.N.P.B`) with a PEP 440 `.dev0` appended on the `develop` branch. A wrapper repo may drive its build/image version from an external committed `name -> version` state file while NBGV still tags the release. ### Validate-at-Entry @@ -95,7 +95,7 @@ When a workflow's inputs carry a cross-input or input-versus-derived-state invar ### Resource Lifecycle -Workflow artifacts are an **intra-run handoff** only. Durable copies live on the release/registry. The rule: a transfer artifact handed **between jobs** is deleted by exact name/pattern **at its point of consumption**, the delete is **gated to the same condition as the consumer**, and it is **best-effort**. **Every** `upload-artifact` sets `retention-days: 1` as the universal failure-path backstop, so no terminal blanket-delete job is needed - and an intermediate consumed only within the same run (e.g. an executable's per-runtime outputs feeding an aggregation step) may rely on the retention backstop alone. The run is **never** blanket-deleted (`.artifacts[].id`). See D5. +Workflow artifacts are an **intra-run handoff** only. Durable copies live on the release/registry. The rule: a transfer artifact handed **between jobs** is deleted by exact name/pattern **at its point of consumption**, the delete is **gated to the same condition as the consumer**, and it is **best-effort**. **Every** `upload-artifact` sets `retention-days: 1` as the universal failure-path backstop, so no terminal blanket-delete job is needed, and an intermediate consumed only within the same run (e.g. an executable's per-runtime outputs feeding an aggregation step) may rely on the retention backstop alone. The run is **never** blanket-deleted (`.artifacts[].id`). See D5. ### Fast PR Feedback @@ -114,7 +114,7 @@ flowchart TD ### Release Model -Each publish builds a **single branch** - the trigger ref (`main` a release, `develop` a prerelease) - so there is no branch matrix and `github.ref` always names the built branch. A **human merge never auto-publishes**: a first `plan` job (`publish-plan-task.yml`) decides once and every job gates on it. A run publishes on a **code-affecting bot push to `main`** (the App merges every Dependabot/codegen PR, so `github.actor` gates it, and a shared paths filter also drops a non-substantive change like an Actions bump), a **manual dispatch** of `main`/`develop`, or a **main-only weekly schedule** (Docker, to refresh the base image). The `push` is main-only, so a develop bot merge publishes nothing (its prerelease comes via dispatch). A **source-only** repo publishes on **dispatch only**. Every release is a tag on the built commit plus a source archive, README, and LICENSE. Targets amend it with `release-asset-*` files or push to their own registry. An unchanged version re-pushes nothing (no-op republish). Docker re-pushes by design. +Each publish builds a **single branch**, the trigger ref (`main` a release, `develop` a prerelease), so there is no branch matrix and `github.ref` always names the built branch. A **human merge never auto-publishes**: a first `plan` job (`publish-plan-task.yml`) decides once and every job gates on it. A run publishes on a **code-affecting bot push to `main`** (the App merges every Dependabot/codegen PR, so `github.actor` gates it, and a shared paths filter also drops a non-substantive change like an Actions bump), a **manual dispatch** of `main`/`develop`, or a **main-only weekly schedule** (Docker, to refresh the base image). The `push` is main-only, so a develop bot merge publishes nothing (its prerelease comes via dispatch). A **source-only** repo publishes on **dispatch only**. Every release is a tag on the built commit plus a source archive, README, and LICENSE. Targets amend it with `release-asset-*` files or push to their own registry. An unchanged version re-pushes nothing (no-op republish). Docker re-pushes by design. ```mermaid flowchart TD @@ -130,27 +130,27 @@ flowchart TD Pick each output's path by **where the artifact goes**: - **File on the GitHub release** (zip, binary, packaged library): one leaf per output uploading `release-asset-<branch>-<name>`. The repo keeps `expect_release_assets: true` (its default). -- **Package-registry push** (NuGet, PyPI): the leaf builds and publishes to its registry. NuGet pushes from the leaf *and* uploads a `release-asset-*`. PyPI is **split** - the leaf only builds + uploads its build artifact, a separate publish job does the OIDC upload (so `id-token: write` is granted at one entry point, behind an environment gate) and contributes **no** `release-asset-*`. -- **Image-registry push** (Docker): the leaf pushes the default branch multi-arch (amd64+arm64) and any other branch `amd64`-only (arm64 emulation is reserved for the released image) - contributes no `release-asset-*`. -- **No file target via the release task** (Docker-only, PyPI-only): the release is tag + source zip + README + LICENSE. The repo's **caller MUST pass `expect_release_assets: false`** to the release task (the input is never set by a publisher that ships file targets, which keeps the default `true`). This is the one case where the otherwise-verbatim publisher is edited. With the default `true` and no assets, the release-create step fails on `fail_on_unmatched_files`. A **source-only** repo has no release task at all - its standalone `publish-release.yml` inlines `action-gh-release`, so `expect_release_assets` does not apply (see Section 6). +- **Package-registry push** (NuGet, PyPI): the leaf builds and publishes to its registry. NuGet pushes from the leaf *and* uploads a `release-asset-*`. PyPI is **split**: the leaf only builds + uploads its build artifact, a separate publish job does the OIDC upload (so `id-token: write` is granted at one entry point, behind an environment gate) and contributes **no** `release-asset-*`. +- **Image-registry push** (Docker): the leaf pushes the default branch multi-arch (amd64+arm64) and any other branch `amd64`-only (arm64 emulation is reserved for the released image), and contributes no `release-asset-*`. +- **No file target via the release task** (Docker-only, PyPI-only): the release is tag + source zip + README + LICENSE. The repo's **caller MUST pass `expect_release_assets: false`** to the release task (the input is never set by a publisher that ships file targets, which keeps the default `true`). This is the one case where the otherwise-verbatim publisher is edited. With the default `true` and no assets, the release-create step fails on `fail_on_unmatched_files`. A **source-only** repo has no release task at all. Its standalone `publish-release.yml` inlines `action-gh-release`, so `expect_release_assets` does not apply (see Section 6). -## 4. Behavioral Contract - Expected Outcomes +## 4. Behavioral Contract: Expected Outcomes The required behaviors, organized by domain. Each is a **MUST**, stated as input -> output plus the failure-mode it prevents. A workflow that violates any *applicable* guarantee is **not operational**. ### D1 - PR Fast-Feedback (Smoke) -- **D1.1 Only changed targets build.** Input: a PR touching some targets. Output: the paths-filter marks exactly those targets and only their smoke builds run. Unchanged targets skip. A repo's own targets MUST each have a filter entry (so a touched target is never silently skipped). *Prevents: rebuilding everything, and a changed target slipping through unbuilt.* -- **D1.2 A validation job always runs.** Input: any PR. Output: a type-appropriate validation job runs unconditionally and the aggregator `needs:` it. In a .NET repo this is the `unit-test` job (format/style/test). A non-.NET repo **replaces** it (not deletes) with its own validator (lint, schema-check) and re-points **every** `needs:` on it - both the aggregator and `smoke-build` (which `needs:` the validation job by name) - to the replacement. *Prevents: a PR merging with no validation, or a dangling `needs:` that fails the whole workflow to load.* +- **D1.1 Only changed targets build.** Input: a PR touching some targets. Output: the paths-filter marks exactly those targets and only their smoke builds run. Unchanged targets skip. A repo's own targets MUST each have a filter entry (so a touched target is never silently skipped). *Prevents: rebuilding everything, and a changed target slipping through unbuilt.* **This repo runs no paths filter**, so every push builds both targets. That is stricter than the guarantee and prevents the same failure by never letting a changed target go unbuilt, at the cost of building an unchanged one. +- **D1.2 A validation job always runs.** Input: any PR. Output: a type-appropriate validation job runs unconditionally and the aggregator `needs:` it. In a .NET repo this is the `unit-test` job (format/style/test). A non-.NET repo **replaces** it (not deletes) with its own validator (lint, schema-check) and re-points **every** `needs:` on it to the replacement. *Prevents: a PR merging with no validation, or a dangling `needs:` that fails the whole workflow to load.* Here `validate` and `smoke-build` are siblings rather than a chain: both run unconditionally and the aggregator `needs:` both, so validation cannot be skipped and neither waits on the other. - **D1.3 Smoke never publishes and never uploads.** Input: `smoke: true`. Output: full compile/lint/test, but no registry/image push, no release, and **no** artifact uploads (every `upload-artifact`, including any aggregation job, is gated `!smoke`). *Prevents: a PR publishing; orphaned artifacts churning the storage quota.* -- **D1.4 Workflow-file changes are not smoke-built.** Input: a PR changing only `.github/workflows/**`. Output: the paths-filter excludes workflow files, so smoke-build skips. *Implication: a workflow-only change is not smoke-built, but actionlint still validates it in CI.* -- **D1.5 One required aggregator gates merge.** Input: any PR. Output: a single aggregator job must **succeed**, `needs:` the changes job and the validation job, treat a **skipped** smoke build as pass, and **block** on `failure`/`cancelled`. Its name is ruleset-bound: the job `name:` and the ruleset `context:` are the same string and MUST be renamed together, never independently. *Prevents: a paths-filter error letting a target-changing PR merge unbuilt.* -- **D1.6 Coverage is reported to Codecov (C# and Python).** Input: a C# or Python repo's validation/test job. Output: tests run with coverage collection (`dotnet test --collect:"XPlat Code Coverage"` or `pytest --cov-report=xml`) and a `codecov/codecov-action` step uploads it, **best-effort** (`continue-on-error` and/or `fail_ci_if_error: false`, so a Codecov outage or an absent token never reds the gate). `CODECOV_TOKEN` lives in the repo's **actions** secret store and reaches the reusable validator via `secrets: inherit`. Required for **every** C# and Python repo that has tests (see `spec/secrets.json` `typeMechanisms`). The repo also ships a **`codecov.yml`** that sets the project and patch statuses to **`informational: true`** so a coverage delta never gates a PR - a distinct knob from `fail_ci_if_error` (which only guards the upload step) - and excludes intentionally-untested, non-shipped code (an example/demo or benchmark project) from the coverage denominator via `ignore`. A repo may override this to enforce a coverage threshold where its quality bar requires it. Coverage output is a build artifact - `.gitignore` excludes it (e.g. `coverage/`, `*.cobertura.xml`; `.gitignore` is the full source of truth) so a blanket `git add -A` won't stage the untracked output. *Prevents: coverage silently going unreported; a stale, unused token; a coverage regression blocking an unrelated PR; a coverage artifact committed by a blanket add.* +- **D1.4 Workflow-file changes are not smoke-built.** Input: a PR changing only `.github/workflows/**`. Output: the paths-filter excludes workflow files, so smoke-build skips. *Implication: a workflow-only change is not smoke-built, but actionlint still validates it in CI.* **N/A here**, since this repo runs no paths filter: a workflow-only change is smoke-built like any other. `test-pull-request.yml` runs on push so the reusable tasks resolve from the pushed head, which means such a change tests its own copy rather than the base branch's. +- **D1.5 One required aggregator gates merge.** Input: any PR. Output: a single aggregator job must **succeed**, `needs:` the changes job and the validation job, treat a **skipped** smoke build as pass, and **block** on `failure`/`cancelled`. Its name is ruleset-bound: the job `name:` and the ruleset `context:` are the same string and MUST be renamed together, never independently. *Prevents: a paths-filter error letting a target-changing PR merge unbuilt.* Here the aggregator is `check-workflow-status`, named `Check pull request workflow status job`, and it `needs: [validate, smoke-build]`. There is no changes job to need, and it demands **success** from both rather than treating a skip as pass, since neither is ever skipped. +- **D1.6 Coverage is reported to Codecov (C# and Python).** Input: a C# or Python repo's validation/test job. Output: tests run with coverage collection (`dotnet test --collect:"XPlat Code Coverage"` or `pytest --cov-report=xml`) and a `codecov/codecov-action` step uploads it, **best-effort** (`continue-on-error` and/or `fail_ci_if_error: false`, so a Codecov outage or an absent token never reds the gate). `CODECOV_TOKEN` lives in the repo's **actions** secret store and reaches the reusable validator via `secrets: inherit`. Required for **every** C# and Python repo that has tests (see `spec/secrets.json` `typeMechanisms`). The repo also ships a **`codecov.yml`** that sets the project and patch statuses to **`informational: true`** so a coverage delta never gates a PR (a distinct knob from `fail_ci_if_error`, which only guards the upload step) and excludes intentionally-untested, non-shipped code (an example/demo or benchmark project) from the coverage denominator via `ignore`. A repo may override this to enforce a coverage threshold where its quality bar requires it. Coverage output is a build artifact, so `.gitignore` excludes it (e.g. `coverage/`, `*.cobertura.xml`; `.gitignore` is the full source of truth) so a blanket `git add -A` won't stage the untracked output. *Prevents: coverage silently going unreported; a stale, unused token; a coverage regression blocking an unrelated PR; a coverage artifact committed by a blanket add.* ### D2 - Input/State Validation at Entry - **D2.1 Validate before expensive work.** Output: a dedicated entry job/step asserts each cross-input/derived-state invariant and fails fast before builds. Downstream jobs `needs:` it. -- **D2.2 Release branch matches version classification.** Input: a real (non-smoke) release build. Output: the gate fails loudly if the default branch carries a prerelease suffix **or** a non-default branch carries none. It strips `+buildmetadata` before testing for the prerelease `-` (only a core/prerelease `-` counts), and it is **skipped on smoke** (a detached PR head always versions as prerelease). *Prevents: a non-default leg published as stable; a build-metadata false-positive; the gate blocking every default-base promotion PR.* +- **D2.2 Release branch matches version classification.** Input: a real (non-smoke) release build. Output: the gate fails loudly if the default branch carries a prerelease suffix **or** a non-default branch carries none. It strips `+buildmetadata` before testing for the prerelease `-` (only a core/prerelease `-` counts), and on a smoke build the **check exits early while the job still reports success** (a detached PR head always versions as prerelease). Read that as the validation being skipped rather than the job, because a job-level `if:` would skip the job itself, and a dependent skips with it unless that dependent opts out with `if: always()` and reads the result explicitly, the way the PR aggregator does. `github-release` carries `validate-release` in `needs:` and does **not** opt out, so a job-level skip there would couple the release to smoke through a second path on top of the `if:` it already carries. *Prevents: a non-default leg published as stable; a build-metadata false-positive; the gate blocking every default-base promotion PR.* - **D2.3 Publish only from main or develop.** Input: a dispatch publish. Output: a dispatch from any ref other than `main` or `develop` fails fast. *Prevents: cutting a release from an unintended branch.* - **D2.4 Mutually-exclusive / paired inputs are validated.** Input: a workflow with either/or or must-pair inputs (e.g. the docker-readme task's `repositories` XOR `manifest`+`manifest-jq`). Output: a half-filled or conflicting combination fails fast. *Prevents: a silent fall-through.* @@ -160,19 +160,20 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as input - **D3.2 Default = public, others = prerelease.** Output: default branch -> `X.Y.Z`; any other -> `X.Y.Z-g<sha>`. The default-branch literal in the gate, the `prerelease` expression, and `version.json` MUST all name the repo's real default branch. - **D3.3 Version floor + git height.** Output: `version.json` sets the major.minor floor. NBGV appends the git height as the patch, bumped only for a functional change by the maintainer. NBGV and `version.json` are retained even by a no-compiler repo (they own the tag). - **D3.4 Registry versions follow the classification, per registry.** Output: NuGet default = stable, others = prerelease (derived by NuGet.org from the SemVer2 `-g<sha>` suffix on `PackageVersion`, not a flag the workflow sets). PyPI builds from `AssemblyFileVersion` (`M.N.P.B`) and appends `.dev0` on the `develop` branch only (a two-branch literal, not a generic N-branch rule). The develop `.dev0` build must remain `pip install --pre`-selectable and sort above the default release (NBGV git height in the release segment keeps develop ahead). *Prevents: a non-default leg published as a release; a renamed/extra branch silently getting a plain version.* -- **D3.5 Wrapper repos may use an external version.** Output: a repo wrapping an upstream release drives its build/image version from a committed `name -> version` state file, while NBGV still tags the release. *Note: the tracker (the writer) ships without consumer wiring - a wrapper must wire the leaf to read the state file (e.g. `jq` into the image tag) instead of `SemVer2`. If the leaf still tags off NBGV, the wrapper is not actually pinned to upstream.* +- **D3.5 Wrapper repos may use an external version.** Output: a repo wrapping an upstream release drives its build/image version from a committed `name -> version` state file, while NBGV still tags the release. *Note: the tracker (the writer) ships without consumer wiring, so a wrapper must wire the leaf to read the state file (e.g. `jq` into the image tag) instead of `SemVer2`. If the leaf still tags off NBGV, the wrapper is not actually pinned to upstream.* ### D4 - Release / Publish -- **D4.1 Gated single-branch publish.** Output: PRs smoke-test and publish nothing. A **human merge never auto-publishes**. A first `plan` job (`publish-plan-task.yml`) decides once and every job gates on it: publish on a **code-affecting bot push to `main`** (gated to the codegen App / Dependabot `github.actor`; an Actions-only bump matches no release path and publishes nothing), a **dispatch** of `main`/`develop`, or a **main-only weekly schedule** (Docker). A source-only repo publishes on dispatch only. Each run builds one branch. -- **D4.2 Tag the built commit.** Output: the release `target_commitish` is the built commit's SHA (NBGV's commit id), never `github.sha` or a moving branch ref. *Prevents: the tag landing on the default branch instead of the built tree.* +- **D4.1 Gated single-branch publish.** Output: PRs smoke-test and publish nothing. A **human merge never auto-publishes**. The fleet shape puts a first `plan` job (`publish-plan-task.yml`) in front and gates every job on it, covering a **code-affecting bot push to `main`**, a **dispatch** of `main`/`develop`, and a **main-only weekly schedule**. **This repo carries no plan job and no push trigger at all**: `publish-release.yml` runs on `workflow_dispatch` or the weekly `main`-only schedule, and its single job gates on `github.ref_name` being `main` or `develop`, so a dispatch from a feature branch is a no-op. Nothing publishes on a push, bot or human, and accumulated changes ship in the next scheduled or dispatched run. Each run builds one branch. +- **D4.2 Tag the built commit.** Output: the release `target_commitish` is the built commit's SHA (NBGV's `GitCommitId`), never a branch name or a separately re-resolved ref. *Prevents: the tag landing on the default branch instead of the built tree.* - **D4.3 Release contents.** Output: every release is a tag on the built commit plus the auto source zip, README, and LICENSE; file-producing targets attach `release-asset-*`; `prerelease` equals `branch != default`. A no-file-target repo that uses the release task (Docker-only, PyPI-only) reaches the tag-only shape **only** with `expect_release_assets: false` set by the caller (which relaxes `fail_on_unmatched_files` and skips the asset download). With the default `true` and no assets the release-create step fails. A source-only repo reaches the same shape through its inlined `action-gh-release` instead, with no release task or `expect_release_assets`. -- **D4.4 No-op republish.** Input: a re-run whose version is unchanged. Output: nothing is re-pushed - the release-create step is skipped when the tag exists (refreshed only on `workflow_dispatch`), and the paired asset-delete is skipped with it. Registry pushes are no-ops. The NuGet/PyPI publish steps are **not** statically gated on existence - they run and the **server** dedupes (`dotnet nuget push --skip-duplicate` turns a 409 into success; PyPI `skip-existing: true`). **Docker always re-pushes** the image (base-image refresh), independently of the release-create skip, within the same run. *Prevents: duplicate releases and wasted pushes.* +- **D4.4 No-op republish.** Input: a re-run whose version is unchanged. Output: nothing is re-pushed, because the release-create step is skipped when the tag exists (refreshed only on `workflow_dispatch`), and the paired asset-delete is skipped with it. Registry pushes are no-ops. The NuGet/PyPI publish steps are **not** statically gated on existence. They run and the **server** dedupes (`dotnet nuget push --skip-duplicate` turns a 409 into success; PyPI `skip-existing: true`). **Docker always re-pushes** the image (base-image refresh), independently of the release-create skip, within the same run. *Prevents: duplicate releases and wasted pushes.* +- **D4.5 A build failure blocks every publish target.** Input: a real publish where one enabled build fails. Output: nothing publishes. `github-release` needs every build, so a failed build skips it (no tag, no release), and the terminal registry pusher (Docker) needs every other build and guards its `if` with `!failure() && !cancelled()`, so a failed build skips docker too (no image push) while a disabled or unchanged target (skipped, not failed) still lets docker build on smoke. *Prevents: a partial publish, e.g. a Docker image pushed while the executable build failed and no release was cut.* A repo pushing two registry targets at once would need a build/publish split behind an all-builds gate, which none does today. ### D5 - Resource Cleanup - **D5.1 Delete at the point of consumption.** Output: the job that downloads a **cross-job** transfer artifact deletes it (by exact name/pattern) right after consuming it. An intermediate consumed only within the same run (e.g. an executable's per-runtime outputs feeding an in-run aggregation) MAY instead rely on the `retention-days: 1` backstop. *Prevents: transfer artifacts accumulating against the storage quota.* -- **D5.2 Gate the delete to the consumer's condition.** Output: the delete runs under the **same** condition as its consuming step. Where the consumer is conditional (the GitHub release create), the delete is conditional too. Where the consumer always runs when its job runs (the PyPI publish step), the delete always runs - so on a no-op re-run the `release-asset-*` delete is **skipped** while the PyPI build-artifact delete still **runs** (its publish ran). *Prevents: deleting freshly built assets on a no-op re-run.* +- **D5.2 Gate the delete to the consumer's condition.** Output: the delete runs under the **same** condition as its consuming step. Where the consumer is conditional (the GitHub release create), the delete is conditional too. Where the consumer always runs when its job runs (the PyPI publish step), the delete always runs, so on a no-op re-run the `release-asset-*` delete is **skipped** while the PyPI build-artifact delete still **runs** (its publish ran). *Prevents: deleting freshly built assets on a no-op re-run.* - **D5.3 Best-effort.** Output: cleanup is `continue-on-error`, tolerates a failed listing, and deletes **all** matching ids. *Prevents: a cleanup hiccup reddening a job whose publish succeeded.* - **D5.4 Retention backstop.** Output: **every** `upload-artifact` sets `retention-days: 1`. - **D5.5 Never blanket-delete.** Output: cleanup MUST NOT enumerate and delete the run's whole artifact set. *Prevents: destroying diagnostic/log artifacts and auto-emitted build-records.* @@ -195,8 +196,8 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as input - **D8.1 Merge-bot.** Output: enables auto-merge on `opened`/`reopened` for **every** Dependabot tier including semver-major (the required checks are the gate, not the bump magnitude); dispatches `--squash`/`--merge` by the PR's base ref; disables on a maintainer-pushed `synchronize`; concurrency keyed on the **PR number**, not `github.ref`. *Prevents: two PRs colliding in auto-merge.* - **D8.2 CodeGen and Dependabot.** Output: codegen runs as a matrix over both branches and is deterministic from an external source. Dependabot targets both branches, security PRs to default. -- **D8.3 Upstream-version tracker.** Output: a scheduled resolver prints a JSON `name -> version` object to a committed state file, opens a rolling per-branch bump PR naming only the moved keys, the merge-bot auto-merges it. The `main` pin push publishes via the release gate, while a `develop` pin does not auto-publish - it ships via a `develop` dispatch (prerelease) or the next promotion to `main`. The tracker's `bump-branch-prefix` + `branches` MUST match the merge-bot's hard-coded `<prefix>-<base>` head/base pairs, or auto-merge silently never fires. -- **D8.4 An identity allowlist used as a gate fails loud.** Where a gate compares `github.actor` (or a PR author) against hard-coded bot identities, the non-matching branch on an otherwise-legitimate trigger **emits a `::warning::`** rather than falling through silently. Output: a run that declines to act on an unrecognized identity is visibly annotated. *Prevents: the App being renamed, replaced, or reinstalled under a new slug, after which the comparison quietly evaluates false and the gate stops firing - a green, silent run that looks identical to a healthy one.* The masking matters most where a second path hides the loss: a weekly schedule keeps publishing, so the only symptom is release *timeliness*, easily missed for months. Where the failure is self-announcing instead (the merge-bot simply stops merging, so bot PRs visibly pile up) an annotation is optional. Resolving the identity at run time (mint an App token, read `GET /app`) removes the hard-coded string entirely and is the escalation if an allowlist proves fragile in practice. +- **D8.3 Upstream-version tracker.** Output: a scheduled resolver prints a JSON `name -> version` object to a committed state file, opens a rolling per-branch bump PR naming only the moved keys, the merge-bot auto-merges it. The `main` pin push publishes via the release gate, while a `develop` pin does not auto-publish. It ships via a `develop` dispatch (prerelease) or the next promotion to `main`. The tracker's `bump-branch-prefix` + `branches` MUST match the merge-bot's hard-coded `<prefix>-<base>` head/base pairs, or auto-merge silently never fires. +- **D8.4 An identity allowlist used as a gate fails loud.** Where a gate compares `github.actor` (or a PR author) against hard-coded bot identities, the non-matching branch on an otherwise-legitimate trigger **emits a `::warning::`** rather than falling through silently. Output: a run that declines to act on an unrecognized identity is visibly annotated. *Prevents: the App being renamed, replaced, or reinstalled under a new slug, after which the comparison quietly evaluates false and the gate stops firing, a green and silent run that looks identical to a healthy one.* The masking matters most where a second path hides the loss: a weekly schedule keeps publishing, so the only symptom is release *timeliness*, easily missed for months. Where the failure is self-announcing instead (the merge-bot simply stops merging, so bot PRs visibly pile up) an annotation is optional. Resolving the identity at run time (mint an App token, read `GET /app`) removes the hard-coded string entirely and is the escalation if an allowlist proves fragile in practice. ### D9 - Style / Static (See Section 2) @@ -219,7 +220,7 @@ Read the workflow files plus `version.json` and assert the structural fact behin - **D1:** a `changes` paths-filter job exists, covers each of the repo's targets, and **excludes** `.github/workflows/**`; the PR entry workflow's smoke call sets `github/nuget/dockerhub: false` on the release task; the leaf receives `smoke: true` and a derived `push` (false on smoke); every build-task `upload-artifact` (and any aggregation job) is gated `!smoke`; the aggregator `needs:` the `changes` and validation jobs, blocks on `failure`/`cancelled`, passes on `skipped`; a validation job runs unconditionally. - **D2:** an entry validation job/step exists per complex-input workflow; the release gate checks both directions, strips `+buildmetadata`, and skips on smoke; the publisher rejects a dispatch from a ref other than `main` or `develop`. - **D3:** each run builds one branch, so NBGV classifies `github.ref` directly (no `IGNORE_GITHUB_REF`); the default-branch literal in the gate (`== 'main'`), the `prerelease` expression (`!= 'main'`), and `version.json`'s `publicReleaseRefSpec` all name the repo's actual default branch. -- **D4:** `target_commitish` is the NBGV commit id; `prerelease` equals `branch != default`; the release-create step is gated `exists == 'false' || github.event_name == 'workflow_dispatch'` (the step output is the string `'false'`, not a boolean); the asset-delete step is gated identically. A dispatch-only publisher (`releaseTrigger: dispatch-only`) may omit the gate and the exists-check entirely - every run is a dispatch, so the skip leg can never fire and create-or-refresh is unconditional. Record the gate N/A there, not missing. +- **D4:** `target_commitish` is the NBGV commit id; `prerelease` equals `branch != default`; the release-create step is gated `exists == 'false' || github.event_name == 'workflow_dispatch'` (the step output is the string `'false'`, not a boolean); the asset-delete step is gated identically. A dispatch-only publisher (`releaseTrigger: dispatch-only`) may omit the gate and the exists-check entirely: every run is a dispatch, so the skip leg can never fire and create-or-refresh is unconditional. Record the gate N/A there, not missing. - **D5:** each cross-job transfer artifact has a delete step at its consumer, gated to the consumer's condition, `continue-on-error: true`, looping all ids; **every** upload sets `retention-days: 1`; **no** `.artifacts[].id` blanket delete exists anywhere. - **D6:** the release download uses `pattern:`/`merge-multiple:` (no `artifact-ids:`). Branch-derived config reads `inputs.branch` (a `github.ref_name` in such config is a finding). Artifact names are branch-suffixed. The target set is consistent across the release task and the paths-filter. - **D7:** the publisher concurrency group is ref-independent with `cancel-in-progress: false`. Reusable jobs declare permissions. Boolean `if:` uses both forms. @@ -230,7 +231,7 @@ Read the workflow files plus `version.json` and assert the structural fact behin - **Console/executable:** the smoke runtime matrix is a strict non-empty subset of the full matrix. The per-runtime outputs (`publish-<branch>-<runtime>`) are aggregated by `pattern:` + `merge-multiple:` into one `release-asset-<branch>-<target>` and the aggregation job is gated `!smoke`. The per-runtime intermediates rely on the retention backstop (no explicit delete is required for an in-run intermediate). - **NuGet:** the publish step is gated `if: inputs.push` only (not on an existence check) and uses `--skip-duplicate`. `*.nupkg` push also carries the paired `.snupkg` to the symbol server where symbols are enabled. The `release-asset` zip carries the package(s). - **PyPI:** `publish-pypi` declares `environment: { name: pypi }`. `id-token: write` appears only on that job (absent from the build/PR path). `skip-existing: true` is set on the publish action. The build artifact is deleted after publish. The `pypi` environment has a deployment-branch rule. -- **Docker:** a Docker-only repo's caller passes `expect_release_assets: false`. The leaf reads the external state file for the tag instead of `SemVer2` (wrapper repos only - a plain Docker repo correctly tags off `SemVer2` and records this N/A). The readme/date-badge jobs are gated main-only. The docker-readme task validates `repositories` XOR `manifest`+`manifest-jq`. The buildcache follows D9.4. +- **Docker:** a Docker-only repo's caller passes `expect_release_assets: false`. The leaf reads the external state file for the tag instead of `SemVer2` (wrapper repos only, since a plain Docker repo correctly tags off `SemVer2` and records this N/A). The readme/date-badge jobs are gated main-only. The docker-readme task validates `repositories` XOR `manifest`+`manifest-jq`. The buildcache follows D9.4. ### 5B. End-to-End Trace Scenarios (No Execution, Deterministic from the YAML) @@ -238,24 +239,24 @@ For each *applicable* scenario, evaluate every job's `if:`/`needs:` against the | # | Input | Expected output | Exercises | | --- | --- | --- | --- | -| S1 | PR touching a build target | `changes` flags it; validation runs; that target's smoke build runs; no push, **no uploads**; validate-release **skipped (smoke), succeeds**; release **skipped**; aggregator **success**; version = prerelease; no release; no dangling artifacts | D1, D2.2, D3 | -| S2 | PR changing only docs | smoke-build **skipped**; validation runs; aggregator **success** | D1.1, D1.5 | -| S3 | PR changing only `.github/workflows/**` | filter excludes -> smoke-build **skipped**; aggregator **success** | D1.4 | -| S4 | PR base = default branch, carrying a build target | smoke versions as prerelease; validate-release **skipped (smoke)** so the default-branch arm does **not** fire; aggregator **success**; promotion not blocked | D1.3, D2.2 | -| S5 | bot push to `main` not touching a release path (e.g. an Actions bump) | the paths filter excludes it; nothing publishes | D4.1 | -| S6 | code-affecting **bot** push to `main` (a human push/promotion, or any develop push, does not) | the `plan` job gates it to the App/Dependabot actor; `main` publishes a release | D3, D4 | -| S7 | publish run (schedule, a bot push to main, or a dispatch) | builds the **one** trigger branch: `main` -> `X.Y.Z`, `prerelease=false`, registry stable, badge/readme run; `develop` -> `X.Y.Z-g<sha>`, `prerelease=true`, registry prerelease; `release-asset-*` consumed-then-deleted; PyPI build-artifact deleted after its publish; **no dangling artifacts** | D3, D4, D5, D6, D7 | +| S1 | PR touching a build target | validation runs; both smoke builds run; no push, **no uploads**; validate-release **skipped (smoke), succeeds**; release **skipped**; aggregator **success**; version = prerelease; no release; no dangling artifacts | D1, D2.2, D3 | +| S2 | PR changing only docs | validation and both smoke builds run (no filter to skip them), aggregator **success** | D1.1, D1.5 | +| S3 | PR changing only `.github/workflows/**` | validation and both smoke builds run against the pushed head, so the change tests its own copy, and the aggregator reports **success** | D1.4 | +| S4 | PR base = default branch, carrying a build target | smoke versions as prerelease, validate-release **skipped (smoke)** so the default-branch arm does **not** fire, aggregator **success**, promotion not blocked | D1.3, D2.2 | +| S5 | bot push to `main` (e.g. an Actions bump) | `publish-release.yml` has no push trigger, so nothing publishes | D4.1 | +| S6 | code-affecting push to `main`, bot or human | still nothing publishes, and the change ships in the next scheduled or dispatched run | D3, D4 | +| S7 | publish run (the weekly `main` schedule, or a dispatch) | builds the **one** trigger branch: `main` -> `X.Y.Z`, `prerelease=false`, registry stable, badge/readme run; `develop` -> `X.Y.Z-g<sha>`, `prerelease=true`, registry prerelease; `release-asset-*` consumed-then-deleted; PyPI build-artifact deleted after its publish; **no dangling artifacts** | D3, D4, D5, D6, D7 | | S8 | dispatch from a ref other than `main` or `develop` | **fails fast** | D2.3 | | S9 | re-run publish, version unchanged | release-create **skipped**, `release-asset-*` delete **skipped**; NuGet/PyPI pushes no-op (server dedupe); **PyPI build-artifact still deleted** (its publish ran); **Docker still re-pushes** the image; no duplicate release | D4.4, D5.2 | -| S10 | branch/version classification disagree | validate-release **fails loud**; build/publish skip | D2.2 | -| S11 | scheduled upstream-version bump (wrapper) | resolver detects a change -> commits the state file -> opens a `<prefix>-<branch>` PR -> merge-bot auto-merges -> the `main` pin publishes via the gate (a develop pin does not auto-publish; it ships via a develop dispatch or promotion) | D8.3, D3.5 | +| S10 | branch/version classification disagree | validate-release **fails loud**, build/publish skip | D2.2 | +| S11 | scheduled upstream-version bump (wrapper) | resolver detects a change -> commits the state file -> opens a `<prefix>-<branch>` PR -> merge-bot auto-merges -> the `main` pin publishes via the gate (a develop pin does not auto-publish, shipping instead via a develop dispatch or promotion) | D8.3, D3.5 | ### 5C. Live Probe (Where Warranted) - Open a trivial-change PR touching one target and confirm S1. -- Drive a `smoke: true` push-probe of the build task for **both** the default and a non-default branch and assert the version classification (clean vs prerelease) and that the gate passes - **without publishing**. *Caveat: the Docker leg logs in to the registry even on smoke and reads the buildcache, so it needs `DOCKER_HUB_*` secrets and cannot run on a fork PR (same-repo only).* +- Drive a `smoke: true` push-probe of the build task for **both** the default and a non-default branch and assert the version classification (clean vs prerelease) and that the gate passes, **without publishing**. *Caveat: the Docker leg logs in to the registry even on smoke and reads the buildcache, so it needs `DOCKER_HUB_*` secrets and cannot run on a fork PR (same-repo only).* - Per registry: after a real publish, query NuGet.org for the expected version + prerelease classification (and the `.snupkg` on the symbol server), and confirm a re-run added no duplicate. For PyPI inspect the `Compute PyPI version step` log and the built `dist/*` filenames for `.dev0` off `develop` vs a plain version on the default branch. -- Inspect the latest real publish's logs for `PublicRelease`/`SemVer2` per leg and confirm the artifact lifecycle (uploaded, consumed, deleted; none left behind). +- Inspect the latest real publish's logs for `PublicRelease`/`SemVer2` per leg and confirm the artifact lifecycle (uploaded, consumed, deleted, with none left behind). ### Assessment @@ -270,13 +271,13 @@ The workflow is **operational** iff every *applicable* 5A item passes and every Each type maps the *applicable* S-scenarios onto its targets. The differences are which leaf tasks exist and what each produces, which 5A addenda apply, and which scenarios are N/A. Walking these is the self-check that the contract holds for each shape. -- **Console / executable application.** Target produces `release-asset-<branch>-executable` (a 7z archive, `Console.7z`) by building a per-runtime `dotnet publish` matrix, then an aggregation job downloads the per-runtime `publish-<branch>-<runtime>` intermediates (`pattern:` + `merge-multiple:`), zips them, and uploads the single asset. Smoke builds a strict subset of runtimes. The per-runtime upload **and** the aggregation job are both gated `!smoke`, so smoke uploads nothing. The per-runtime intermediates rely on `retention-days: 1` (no explicit delete). Test: S1 with a console change smoke-builds the subset and uploads nothing; S7 attaches the 7z, `prerelease=true` on the non-default leg and `prerelease=false` on the default leg (GitHub auto-marks the stable default release "Latest" - the workflow does not set it). -- **NuGet library.** The leaf both pushes (`dotnet nuget push *.nupkg --skip-duplicate`, gated `if: push` only) and uploads `release-asset-<branch>-nugetlibrary`. Configuration is Release on the default branch, Debug otherwise. Where symbols are enabled (`snupkg`), the push auto-carries the paired `.snupkg` to NuGet.org's symbol server and the asset zip also contains it - a triple surface. NuGet.org derives `isPrerelease` from the SemVer2 `-g<sha>` suffix (the workflow sets no such flag). Test: S7 non-default leg publishes a prerelease package + asset, default a stable; S9 re-run is a server-side `--skip-duplicate` no-op. 5C: query NuGet.org for both versions and the symbol package. -- **PyPI library.** The leaf builds + uploads `pypilibrary-build-<branch>`. A **separate** `publish-pypi` job (with `environment: pypi`, `id-token: write`, `actions: write`) does the OIDC Trusted-Publishing upload with `skip-existing: true`, then **consume-then-deletes** the build artifact - **unconditionally on consume**, so on S9 it is deleted even though the `release-asset-*` delete is skipped. The version is `AssemblyFileVersion` with `.dev0` appended on `develop` only, and must stay `--pre`-selectable and sorted above the default release. PyPI contributes no `release-asset-*`. A PyPI-only repo sets `expect_release_assets: false` at the caller. Test: S7 default leg publishes a release, non-default a `.dev0`; S9 is a `skip-existing` no-op; 5C inspects the `dist/*` filenames and the compute-version log. +- **Console / executable application.** Target produces `release-asset-<branch>-executable` (a 7z archive, `Console.7z`) by building a per-runtime `dotnet publish` matrix, then an aggregation job downloads the per-runtime `publish-<branch>-<runtime>` intermediates (`pattern:` + `merge-multiple:`), zips them, and uploads the single asset. Smoke builds a strict subset of runtimes. The per-runtime upload **and** the aggregation job are both gated `!smoke`, so smoke uploads nothing. The per-runtime intermediates rely on `retention-days: 1` (no explicit delete). Test: S1 with a console change smoke-builds the subset and uploads nothing; S7 attaches the 7z, `prerelease=true` on the non-default leg and `prerelease=false` on the default leg (GitHub auto-marks the stable default release "Latest", and the workflow does not set it). +- **NuGet library.** The leaf both pushes (`dotnet nuget push *.nupkg --skip-duplicate`, gated `if: push` only) and uploads `release-asset-<branch>-nugetlibrary`. Configuration is Release on the default branch, Debug otherwise. Where symbols are enabled (`snupkg`), the push auto-carries the paired `.snupkg` to NuGet.org's symbol server and the asset zip also contains it, a triple surface. NuGet.org derives `isPrerelease` from the SemVer2 `-g<sha>` suffix (the workflow sets no such flag). Test: S7 non-default leg publishes a prerelease package + asset, default a stable; S9 re-run is a server-side `--skip-duplicate` no-op. 5C: query NuGet.org for both versions and the symbol package. +- **PyPI library.** The leaf builds + uploads `pypilibrary-build-<branch>`. A **separate** `publish-pypi` job (with `environment: pypi`, `id-token: write`, `actions: write`) does the OIDC Trusted-Publishing upload with `skip-existing: true`, then **consume-then-deletes** the build artifact, **unconditionally on consume**, so on S9 it is deleted even though the `release-asset-*` delete is skipped. The version is `AssemblyFileVersion` with `.dev0` appended on `develop` only, and must stay `--pre`-selectable and sorted above the default release. PyPI contributes no `release-asset-*`. A PyPI-only repo sets `expect_release_assets: false` at the caller. Test: S7 default leg publishes a release, non-default a `.dev0`; S9 is a `skip-existing` no-op; 5C inspects the `dist/*` filenames and the compute-version log. - **Docker image.** The leaf pushes the default branch multi-arch (amd64+arm64) and any other branch `amd64`-only, with a per-branch registry buildcache (`buildcache-<branch>`; a multi-image repo adds a per-image tag) (`cache-to` only the built branch and only on push, `cache-from` both branches); no `release-asset-*`, so a Docker-only repo's caller passes `expect_release_assets: false`; the readme (`peter-evans/dockerhub-description`, `DOCKER_HUB_ACCESS_TOKEN`) and date-badge jobs run **only** when the default branch publishes; the docker-readme task validates `repositories` XOR `manifest`+`manifest-jq` and a multi-image repo derives its publish matrix from the manifest. Docker **always re-pushes** the image, independently of a skipped release-create (S9). A **wrapper** repo tracks an upstream release: the upstream tracker writes a `name -> version` state file and the merge-bot auto-merges the bump PR (S11), and the leaf MUST read that file for the immutable tag instead of `SemVer2` (the tracker ships without this consumer wiring). Test: S7 default leg pushes `latest` + the version tag and updates readme/badge. Non-default pushes the develop tag (amd64 only). S9 still re-pushes. S11 ships the bumped upstream version next publish. 5C Docker probe needs `DOCKER_HUB_*` secrets and same-repo (not fork) runs. -- **Data / asset library.** A single new leaf: validate -> zip -> upload `release-asset-<branch>-library` (`retention-days: 1`, upload gated `!smoke` - mirror the nugetlibrary leaf's shape). Because no such leaf ships, you **add a target** (D6.4): a new `enable_library` input + `build-library` job + `github-release` `needs:` entry in the release task, and a `library` paths-filter entry + `changes` output + `smoke-build` enable-forward in the PR workflow (without it, D1.1 never smoke-builds the library). Keep `expect_release_assets: true` (it has a file target, unlike Docker). The .NET `unit-test` job is replaced by a type-appropriate validator with the aggregator **and** `smoke-build` both re-pointed to it (D1.2/D1.5). `version.json` + the NBGV `get-version` step are retained (they own the tag). Test: S1 smoke runs validate+zip and uploads nothing; S7 attaches the zip, prerelease on the non-default leg; S9 on a *scheduled* re-run release-create + asset-delete skip (the existing zip is untouched, no registry push), while a `workflow_dispatch` re-run **refreshes** the release and re-runs the asset-delete (the asset is re-uploaded then re-deleted). N/A: the nuget/pypi/docker/executable 5A addenda and their scenario clauses. -- **Source-only / no build.** There is no `build-release-task.yml` (its `appliesTo` excludes source-only) and no package/image leaf, so nothing is edited down. The release is a standalone dispatch-only `publish-release.yml` that inlines NBGV for the tag and `action-gh-release` for the release - tag + source zip + README + LICENSE, no reusable release task and no asset download. With no target the paths-filter matches nothing, so `smoke-build` is **structurally always skipped** - validation is carried solely by the (replaced, non-.NET) validation job that the aggregator and `smoke-build`'s own `needs:` must both point at (D1.2; or drop the never-running `smoke-build` job). NBGV and `version.json` are still retained (they own the tag). Its publish job gates on the repo's reusable validation task (`needs:` the same `workflow_call` job the PR workflow runs), so a dispatch cannot release a ref that fails validation. Applicable scenarios: S1 (validation only), S5/S6 (publish gating), S7 (tag-only release), S8 (dispatch guard), S9 (no-op republish), S10 (classification gate). N/A: S2-S4 (assume a smoke-built target), the artifact-lifecycle and registry clauses of S7/S9, the D5/D6 artifact items, and all per-type 5A addenda - recorded N/A, not failed. -- **Operational (workflow model, not a build target).** A `workflowModel: operational` repo layers a direct-commit `develop` onto the **source-only** release shape (above). Two workflows: (1) a **lint/validation** PR workflow feeding the required `Check pull request workflow status job` - the generic linters (editorconfig/EOL, markdownlint, cspell, actionlint) plus a domain validator (Home Assistant `hass --script check_config`, `esphome config`, a firmware build), **no unit tests**; its triggers differ from the `release` model - `push` to `develop` (advisory feedback on direct commits) plus `pull_request` to `main` (the enforced promotion gate) plus `workflow_dispatch`. (2) the standard **source-only publisher** on `workflow_dispatch` only (`releaseTrigger: dispatch-only`): NBGV + `version.json` own the tag, and a manual dispatch cuts a GitHub release (tag + source zip + README + LICENSE, via the standalone publisher's inlined `action-gh-release`). Applicable scenarios: S1 (validation) on the promotion PR, plus the source-only release set - S7 (tag-only release), S8 (dispatch guard), S9 (no-op republish), S10 (classification). N/A: the auto-publish paths (S5/S6 bot-push and schedule - operational repos have neither) and every build/registry scenario. See the branch-model note in Section 3 and [GOVERNANCE.md "Branching Model"][governance-branching-model]. +- **Data / asset library.** A single new leaf: validate -> zip -> upload `release-asset-<branch>-library` (`retention-days: 1`, upload gated `!smoke`, mirroring the nugetlibrary leaf's shape). Because no such leaf ships, you **add a target** (D6.4): a new `enable_library` input + `build-library` job + `github-release` `needs:` entry in the release task, and a `library` paths-filter entry + `changes` output + `smoke-build` enable-forward in the PR workflow (without it, D1.1 never smoke-builds the library). Keep `expect_release_assets: true` (it has a file target, unlike Docker). The .NET `unit-test` job is replaced by a type-appropriate validator with the aggregator **and** `smoke-build` both re-pointed to it (D1.2/D1.5). `version.json` + the NBGV `get-version` step are retained (they own the tag). Test: S1 smoke runs validate+zip and uploads nothing; S7 attaches the zip, prerelease on the non-default leg; S9 on a *scheduled* re-run release-create + asset-delete skip (the existing zip is untouched, no registry push), while a `workflow_dispatch` re-run **refreshes** the release and re-runs the asset-delete (the asset is re-uploaded then re-deleted). N/A: the nuget/pypi/docker/executable 5A addenda and their scenario clauses. +- **Source-only / no build.** There is no `build-release-task.yml` (its `appliesTo` excludes source-only) and no package/image leaf, so nothing is edited down. The release is a standalone dispatch-only `publish-release.yml` that inlines NBGV for the tag and `action-gh-release` for the release: tag + source zip + README + LICENSE, with no reusable release task and no asset download. With no target the paths-filter matches nothing, so `smoke-build` is **structurally always skipped**, and validation is carried solely by the (replaced, non-.NET) validation job that the aggregator and `smoke-build`'s own `needs:` must both point at (D1.2; or drop the never-running `smoke-build` job). NBGV and `version.json` are still retained (they own the tag). Its publish job gates on the repo's reusable validation task (`needs:` the same `workflow_call` job the PR workflow runs), so a dispatch cannot release a ref that fails validation. Applicable scenarios: S1 (validation only), S5/S6 (publish gating), S7 (tag-only release), S8 (dispatch guard), S9 (no-op republish), S10 (classification gate). N/A: S2-S4 (assume a smoke-built target), the artifact-lifecycle and registry clauses of S7/S9, the D5/D6 artifact items, and all per-type 5A addenda, all recorded N/A, not failed. +- **Operational (workflow model, not a build target).** A `workflowModel: operational` repo layers a direct-commit `develop` onto the **source-only** release shape (above). Two workflows: (1) a **lint/validation** PR workflow feeding the required `Check pull request workflow status job`, built from the generic linters (editorconfig/EOL, markdownlint, cspell, actionlint) plus a domain validator (Home Assistant `hass --script check_config`, `esphome config`, a firmware build), with **no unit tests**; its triggers differ from the `release` model: `push` to `develop` (advisory feedback on direct commits) plus `pull_request` to `main` (the enforced promotion gate) plus `workflow_dispatch`. (2) the standard **source-only publisher** on `workflow_dispatch` only (`releaseTrigger: dispatch-only`): NBGV + `version.json` own the tag, and a manual dispatch cuts a GitHub release (tag + source zip + README + LICENSE, via the standalone publisher's inlined `action-gh-release`). Applicable scenarios: S1 (validation) on the promotion PR, plus the source-only release set: S7 (tag-only release), S8 (dispatch guard), S9 (no-op republish), S10 (classification). N/A: the auto-publish paths (S5/S6 bot-push and schedule, neither of which an operational repo has) and every build/registry scenario. See the branch-model note in Section 3 and [GOVERNANCE.md "Branching Model"][governance-branching-model]. <!-- Workflow --> diff --git a/cspell.json b/cspell.json index adc66a5..a3a694a 100644 --- a/cspell.json +++ b/cspell.json @@ -17,6 +17,7 @@ "astral", "autoremove", "betz", + "BMFF", "buildcache", "buildmetadata", "buildtransitive", @@ -54,6 +55,7 @@ "eeuo", "Emby", "envsubst", + "esac", "exif", "exiftool", "extensionless", @@ -67,6 +69,7 @@ "gyan", "HACS", "hatchling", + "heif", "heredocs", "homeassistant", "icloudpd", @@ -74,14 +77,17 @@ "idempotently", "immich", "immichcli", - "ISOBMFF", + "IPTC", "isort", "Jellyfin", "Keychain", "kicad", "lastbuild", "lavfi", + "libheif", + "libraw", "libsecret", + "libvips", "libx", "lightroom", "LINQ", @@ -136,6 +142,7 @@ "remuxes", "Remuxing", "resharper", + "rhoopr", "rhysd", "Rubba", "ruff", @@ -156,6 +163,7 @@ "subdirs", "subsetting", "tagpath", + "thumbhash", "timonwong", "trashdb", "Triaging", @@ -167,6 +175,7 @@ "USERPROFILE", "venv", "Viljoen", + "vips", "winget", "Xsession", "xunit", diff --git a/repo-config/README.md b/repo-config/README.md index 2ccbbd7..71fed17 100644 --- a/repo-config/README.md +++ b/repo-config/README.md @@ -1,35 +1,35 @@ # repo-config -Repository and branch configuration held as committed files, kept out of `.github/` (which holds the GitHub-consumed configuration - workflows, Dependabot). +Repository and branch configuration held as committed files, kept out of `.github/` (which holds the GitHub-consumed configuration: workflows, Dependabot). -- `main.json` plus one `develop` variant - the branch rulesets as the writable API subset (`name`, `target`, `enforcement`, `bypass_actors`, `conditions`, `rules`). The `develop` payload is `develop.json` (`release` repos) or `operational/develop.json` (`operational` repos). These are the canonical expected payloads that the self-audit (`AUDIT.md`) diffs the live rulesets against. -- `operational/develop.json` - the `develop` ruleset for **operational** repos (registry `workflowModel: operational`): direct signed pushes, no PR gate. Present in operational repos only - a `release` repo does not have it. See "Rulesets" below. -- `configure.sh` - two modes over the GitHub API. `configure.sh apply [owner/repo] [release|operational]` creates-or-updates the settings, the Dependabot security features, and the rulesets idempotently (a full-payload update). `configure.sh check [owner/repo] [release|operational]` is the read-only inverse and exits non-zero on any drift, with the ruleset and settings assertions driven by the committed payloads so they stay repo-agnostic (rule presence, merge methods, and required checks, not a byte diff - so a GitHub-normalized stored ruleset does not false-positive). The command defaults to `apply`, the repo to the current one, and the model to the registry `workflowModel` lookup (or, absent a registry, inference from the carried `develop` payload - an ambiguous layout aborts rather than guesses). The model may be passed as the sole positional (`configure.sh check operational`). +- `main.json` plus one `develop` variant: the branch rulesets as the writable API subset (`name`, `target`, `enforcement`, `bypass_actors`, `conditions`, `rules`). The `develop` payload is `develop.json` (`release` repos) or `operational/develop.json` (`operational` repos). These are the canonical expected payloads that the self-audit (`AUDIT.md`) diffs the live rulesets against. +- `operational/develop.json`: the `develop` ruleset for **operational** repos (registry `workflowModel: operational`), taking direct signed pushes with no PR gate. Present in operational repos only, since a `release` repo does not have it. See "Rulesets" below. +- `configure.sh`: two modes over the GitHub API. `configure.sh apply [owner/repo] [release|operational]` creates-or-updates the settings, the Dependabot security features, and the rulesets idempotently (a full-payload update). `configure.sh check [owner/repo] [release|operational]` is the read-only inverse and exits non-zero on any drift, with the ruleset and settings assertions driven by the committed payloads so they stay repo-agnostic (rule presence, merge methods, and required checks, not a byte diff, so a GitHub-normalized stored ruleset does not false-positive). The command defaults to `apply`, the repo to the current one, and the model to the registry `workflowModel` lookup (or, absent a registry, inference from the carried `develop` payload, where an ambiguous layout aborts rather than guesses). The model may be passed as the sole positional (`configure.sh check operational`). ## Rulesets Two workflow models share `main.json` but differ on `develop` (registry `workflowModel`, default `release`): -- **`release`** (`develop.json`): `develop` requires squash merges with linear history and a PR - the feature-branch pipeline. -- **`operational`** (`operational/develop.json`): `develop` takes **direct signed pushes** - only `deletion`, `non_fast_forward`, and `required_signatures`; no PR, no status-check, no Copilot-on-push. CI runs on the push as advisory feedback. This is for live-service config repos that edit `develop` directly and promote a known-good snapshot to `main` via an occasional PR (see [GOVERNANCE.md "Branching Model"][governance-branching-model]). +- **`release`** (`develop.json`): `develop` requires squash merges with linear history and a PR, the feature-branch pipeline. +- **`operational`** (`operational/develop.json`): `develop` takes **direct signed pushes**, carrying only `deletion`, `non_fast_forward`, and `required_signatures`; no PR, no status-check, no Copilot-on-push. CI runs on the push as advisory feedback. This is for live-service config repos that edit `develop` directly and promote a known-good snapshot to `main` via an occasional PR (see [GOVERNANCE.md "Branching Model"][governance-branching-model]). -`main` (both models) requires merge-commit merges (no linear-history rule), signed commits, a passing `Check pull request workflow status job`, resolved review threads, and Copilot review, and blocks force-pushes and deletion - so a `develop -> main` promotion is always gated even when `develop` takes direct commits. Every ruleset intentionally leaves "Require branches to be up to date before merging" **off** - see [GOVERNANCE.md "Branching Model"][governance-branching-model]. +`main` (both models) requires merge-commit merges (no linear-history rule), signed commits, a passing `Check pull request workflow status job`, resolved review threads, and Copilot review, and blocks force-pushes and deletion, so a `develop -> main` promotion is always gated even when `develop` takes direct commits. Every ruleset intentionally leaves "Require branches to be up to date before merging" **off**, per [GOVERNANCE.md "Branching Model"][governance-branching-model]. -The result is **exactly two rulesets named `develop` and `main`** - the names are load-bearing (`GOVERNANCE.md` and the workflows reference them); only the `develop` *content* varies by model. The required check binds by name and only turns green after the repo's PR workflow runs once. +The result is **exactly two rulesets named `develop` and `main`**, and the names are load-bearing (`GOVERNANCE.md` and the workflows reference them). Only the `develop` *content* varies by model. The required check binds by name and only turns green after the repo's PR workflow runs once. ## Secrets -Publish credentials required per mechanism are enumerated in `spec/secrets.json`. A repo needs only the mechanisms its own publish targets use - a source-only repo needs none of the publish credentials below. NuGet and PyPI use keyless OIDC Trusted Publishing (no stored key; the publish job needs `id-token: write`, and PyPI additionally an `environment: pypi` gate). Docker Hub has no OIDC equivalent and uses a stored `DOCKER_HUB_USERNAME` + `DOCKER_HUB_ACCESS_TOKEN` in both the Actions and Dependabot secret stores. Codegen and merge-bot repos add a GitHub App (`CODEGEN_APP_CLIENT_ID` + `CODEGEN_APP_PRIVATE_KEY` in both stores; the app must be installed, not just created). App-token call sites use `client-id`, never the deprecated `app-id`. +Publish credentials required per mechanism are enumerated in `spec/secrets.json`. A repo needs only the mechanisms its own publish targets use, so a source-only repo needs none of the publish credentials below. NuGet and PyPI use keyless OIDC Trusted Publishing (no stored key; the publish job needs `id-token: write`, and PyPI additionally an `environment: pypi` gate). Docker Hub has no OIDC equivalent and uses a stored `DOCKER_HUB_USERNAME` + `DOCKER_HUB_ACCESS_TOKEN` in both the Actions and Dependabot secret stores. Codegen and merge-bot repos add a GitHub App (`CODEGEN_APP_CLIENT_ID` + `CODEGEN_APP_PRIVATE_KEY` in both stores; the app must be installed, not just created). App-token call sites use `client-id`, never the deprecated `app-id`. ## Repo Settings -The fleet-standard general settings live in [`settings.json`][settings-json] and are applied idempotently by `configure.sh apply` alongside the rulesets (`gh api PATCH /repos/{owner}/{repo}`). The two settings that depend on per-repo state - `has_discussions` (visibility) and `default_branch` (main-must-exist) - are computed by the script, not stored in the file. `configure.sh apply` also enables Dependabot vulnerability alerts and automated security updates - fleet policy applied via the API, not a `settings.json` key. `configure.sh check` validates all of these and exits non-zero on drift. +The fleet-standard general settings live in [`settings.json`][settings-json] and are applied idempotently by `configure.sh apply` alongside the rulesets (`gh api PATCH /repos/{owner}/{repo}`). The two settings that depend on per-repo state, `has_discussions` (visibility) and `default_branch` (main-must-exist), are computed by the script, not stored in the file. `configure.sh apply` also enables Dependabot vulnerability alerts and automated security updates, fleet policy applied via the API rather than a `settings.json` key. `configure.sh check` validates all of these and exits non-zero on drift. - **Default branch `main`** (the script sets it only when a `main` branch exists, never pointing the default at a missing branch). -- **Merge methods**: `Allow merge commits` and `Allow squash merging` on, **rebase off** - each branch ruleset then picks its method (merge on `main`, squash on `develop`). +- **Merge methods**: `Allow merge commits` and `Allow squash merging` on, **rebase off**, and each branch ruleset then picks its method (merge on `main`, squash on `develop`). - **Auto-merge on** (the merge-bot needs it) and **`Always suggest updating pull request branches` on**. -- **`Automatically delete head branches` OFF - deliberately.** With it on, a `develop -> main` promotion (whose PR head is `develop`) would delete `develop`. There is no per-branch exemption, so the repo-wide toggle stays off to protect `develop`. **The CLI has the same trap: never `gh pr merge --delete-branch` a promotion PR whose head is `develop`** - the explicit flag deletes `develop` regardless of this setting (see [GOVERNANCE.md "Branching Model"][governance-branching-model]). -- **Wikis and Projects off. Discussions on public repos only** (off on private). **Sponsorships off** - the button is driven by `.github/FUNDING.yml`, not a REST toggle, and the fleet ships none. +- **`Automatically delete head branches` is OFF, deliberately.** With it on, a `develop -> main` promotion (whose PR head is `develop`) would delete `develop`. There is no per-branch exemption, so the repo-wide toggle stays off to protect `develop`. **The CLI has the same trap: never `gh pr merge --delete-branch` a promotion PR whose head is `develop`**, since the explicit flag deletes `develop` regardless of this setting (see [GOVERNANCE.md "Branching Model"][governance-branching-model]). +- **Wikis and Projects off. Discussions on public repos only** (off on private). **Sponsorships off**, since the button is driven by `.github/FUNDING.yml` rather than a REST toggle, and the fleet ships none. - **Actions / General**: allow GitHub Actions to create and approve pull requests (for the bots). <!-- Repo --> diff --git a/version.json b/version.json index 7cfe403..cd8a018 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", - "version": "1.0", + "version": "1.1", "publicReleaseRefSpec": [ "^refs/heads/main$" ]