From 8ced947cd97a12667b74e635ba1f7de07378e90f Mon Sep 17 00:00:00 2001 From: Antony Stubbs Date: Sat, 11 Apr 2026 13:11:32 +1200 Subject: [PATCH 1/3] Add Maven Central publishing workflow On every push to master, .github/workflows/publish.yml reads the version from pom.xml and either: - Publishes a snapshot (version ends in -SNAPSHOT) - Publishes a release + creates git tag + GitHub release (otherwise) The pom.xml version is the source of truth - no maven-release-plugin dance. To cut a release, merge a PR that strips -SNAPSHOT. Also adds --pinentry-mode loopback to the maven-gpg-plugin so headless CI signing works without a TTY. Documents required GitHub secrets in AGENTS.md and the README dev section: - MAVEN_CENTRAL_USERNAME / MAVEN_CENTRAL_PASSWORD - MAVEN_GPG_PRIVATE_KEY / MAVEN_GPG_PASSPHRASE Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/publish.yml | 105 ++++++++++++++++++++++++++++++++ AGENTS.md | 110 ++++++---------------------------- README.adoc | 20 +------ pom.xml | 7 +++ src/docs/README_TEMPLATE.adoc | 22 +++++++ 5 files changed, 155 insertions(+), 109 deletions(-) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..eda59dc81 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,105 @@ +# Publishes to Maven Central on every push to master. +# +# The pom.xml version is the source of truth: +# - Versions ending in -SNAPSHOT -> publish snapshot +# - Versions without -SNAPSHOT -> publish release + create git tag + GitHub release +# +# To cut a release: open a PR that strips -SNAPSHOT, merge it. CI will publish. +# Then open another PR bumping to the next -SNAPSHOT. +# +# Required GitHub secrets: +# MAVEN_CENTRAL_USERNAME - Sonatype Central Portal token username +# MAVEN_CENTRAL_PASSWORD - Sonatype Central Portal token password +# MAVEN_GPG_PRIVATE_KEY - Armored GPG private key (gpg --export-secret-keys --armor KEYID) +# MAVEN_GPG_PASSPHRASE - Passphrase for the GPG key + +name: Publish to Maven Central + +on: + push: + branches: [ master ] + workflow_dispatch: # allow manual re-runs + +concurrency: + group: publish-${{ github.ref }} + cancel-in-progress: false # never cancel a publish mid-flight + +permissions: + contents: write # needed to create tags and releases + +jobs: + publish: + name: Publish + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 # need full history for tag creation + + - name: Setup JDK 17 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + # Configure Maven settings.xml with Sonatype credentials and GPG + server-id: central + server-username: MAVEN_CENTRAL_USERNAME + server-password: MAVEN_CENTRAL_PASSWORD + gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Read project version + id: version + run: | + VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout) + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + if [[ "$VERSION" == *-SNAPSHOT ]]; then + echo "is_snapshot=true" >> "$GITHUB_OUTPUT" + echo "Detected SNAPSHOT version: $VERSION" + else + echo "is_snapshot=false" >> "$GITHUB_OUTPUT" + echo "Detected RELEASE version: $VERSION" + fi + + - name: Check tag does not exist (releases only) + if: steps.version.outputs.is_snapshot == 'false' + run: | + TAG="v${{ steps.version.outputs.version }}" + if git rev-parse "$TAG" >/dev/null 2>&1; then + echo "Tag $TAG already exists - aborting to prevent re-release" + exit 1 + fi + + - name: Deploy to Maven Central + env: + MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }} + MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + run: | + ./mvnw --batch-mode \ + -Pmaven-central \ + -Pci \ + clean deploy \ + -DskipTests \ + -Dlicense.skip + + - name: Create git tag (releases only) + if: steps.version.outputs.is_snapshot == 'false' + run: | + TAG="v${{ steps.version.outputs.version }}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag -a "$TAG" -m "Release $TAG" + git push origin "$TAG" + + - name: Create GitHub release (releases only) + if: steps.version.outputs.is_snapshot == 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="v${{ steps.version.outputs.version }}" + gh release create "$TAG" \ + --title "$TAG" \ + --generate-notes diff --git a/AGENTS.md b/AGENTS.md index 6db0fb961..123dc5401 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,9 @@ Project context for AI coding agents (Claude Code, Copilot, Cursor, etc.). ## Overview -Confluent Parallel Consumer is a Java library that enables concurrent message processing from Apache Kafka with a single consumer, avoiding the need to increase partition counts. It maintains ordering guarantees (by partition or key) while processing messages in parallel. +Parallel Consumer is a Java library that enables concurrent message processing from Apache Kafka with a single consumer, avoiding the need to increase partition counts. It maintains ordering guarantees (by partition or key) while processing messages in parallel. + +This is a community-maintained fork of `confluentinc/parallel-consumer` (the upstream is no longer actively maintained), published to Maven Central as `io.github.astubbs.parallelconsumer`. ## Build Requirements @@ -47,7 +49,6 @@ bin/ci-build.sh 3.9.1 - **Integration tests**: `mvn verify` / failsafe plugin. Source in `src/test-integration/java/`. Uses TestContainers with `confluentinc/cp-kafka` Docker image. - **Test exclusion patterns**: `**/integrationTest*/**/*.java` and `**/*IT.java` are excluded from surefire, included in failsafe. - **Kafka version matrix**: CI tests against multiple Kafka versions via `-Dkafka.version=X.Y.Z`. -- **Performance tests**: Tagged `@Tag("performance")` and excluded from regular CI by default. They run on a self-hosted runner via `.github/workflows/performance.yml` (see `docs/SELF_HOSTED_RUNNER.md`). Run locally with `bin/performance-test.sh` (or `bin/performance-test.cmd` on Windows). Override the test group selection with Maven properties: `-Dincluded.groups=performance` to run only perf, `-Dexcluded.groups=` to run everything. ## Known Issues @@ -57,101 +58,26 @@ bin/ci-build.sh 3.9.1 - **Lombok**: Used extensively (builders, getters, logging). IntelliJ Lombok plugin required. - **EditorConfig**: Enforced via `.editorconfig` - 4-space indent for Java, 120 char line length. -- **License headers**: Managed by `license-maven-plugin` (Mycila). See "License headers" section below. +- **License headers**: Managed by `license-maven-plugin` (Mycila). Use `-Dlicense.skip` locally to skip checks. - **Google Truth**: Used for test assertions alongside JUnit 5 and Mockito. -## License headers - -The Mycila `license-maven-plugin` enforces a Confluent copyright header on all source files. It uses git-derived years via `${license.git.copyrightYears}`. - -**Skipping the check** (for any Maven goal): -``` -./mvnw -Dlicense.skip -``` - -**When to skip:** -- Running builds inside a git worktree — the git-years lookup fails with `Bare Repository has neither a working tree, nor an index` -- Local iteration where you don't want years auto-bumped on touched files -- Any command other than the canonical `mvn install` flow when copyright drift would create noise in `git status` - -The default behavior on macOS dev machines is `format` mode (auto-fixes headers) via the `license-format` profile (auto-activated). The `ci` profile flips this to `check` mode (fails the build on drift). Both `bin/build.sh` and `bin/ci-build.sh` already pass `-Dlicense.skip` for this reason. - -**When NOT to skip:** -- You're intentionally running `mvn license:format` to update headers -- You want to verify CI's check would pass before pushing - -## Agent Rules - -### Git Safety -- **NEVER commit or push without explicitly asking the user first.** Wait for approval. This is the #1 rule. -- **When creating a stacked PR, include `depends on #N` in the PR description** (where `#N` is the parent PR it stacks on). This fork runs a PR dependency gating action (see `.github/workflows/check-dependencies.yml`) which blocks child PRs from merging until the parent is merged. One `depends on` line per parent. Keep the list accurate if the chain changes. -- Branch off master for upstream contributor cherry-picks so PRs show only their change. -- Never commit without tests and documentation in the same pass. -- Run tests before committing. If they fail, fix them first. -- When you fix something or finish implementing something, record what lessons you learnt. - -### Development Discipline -- **Skateboard first.** Build the simplest end-to-end thing that works. Before starting a feature, ask: "Is this blocking the next public milestone?" If not, flag it and move on. -- **Never paper over the real problem** - make the proper fix. -- Don't propose workarounds that require user action when the software can solve it. If the software has enough information to derive the right answer, it should just do it. -- If constructing data in memory that is eventually going to be saved, save it as soon as it's created. Don't delay in case the programme crashes or the user exits. - -### Code Quality -- **Be DRY.** Reuse existing functions. Don't copy code - refactor where necessary. Refactor out common patterns. -- Never weaken test assertions - classify exceptions instead of ignoring them. -- Wire components through PCModule DI - don't bypass the dependency injection system. -- Validate user input - don't let bad input cause silent failures. -- Handle errors visibly - don't swallow exceptions. -- Give things meaningful names that describe what they do. Never use random or generic names. - -### Test Discipline -- Search for existing test harnesses and utilities before creating new ones. -- Run the complete test suite periodically, not just targeted tests. -- Maintain good high-level test coverage. Only get detailed on particularly complex functions that benefit from fine-grained testing. - -### CI and Automation -- Always set up continuous integration, code coverage, and automated dependency checking. -- Make scripts for common end-user requirements with helpful, suggestive CLI interfaces. - -### Documentation -- Keep a diary of major plans and their milestones. -- **Keep a developer-facing product specification** that outlines product features, functionality, and implementation architecture - separately from end-user documentation. With agentic programming, the developer can lose sight of architecture and implementation details. This document exposes the interesting, novel, and important implementation decisions so the developer maintains a clear mental model of the system even when agents are doing most of the coding. -- Keep end-user documentation updated. -- Keep documentation tables of contents updated. - -### Communication -- Use precise terminology - if the project defines specific terms, use them consistently. Don't use ambiguous words. -- Don't write with em dash characters. - -### Rule Sync -- Keep this AGENTS.md in sync with any global CLAUDE.md rules. If you have rules in your global config that are missing here, suggest to the user that they be added. This ensures all contributors and agents working on this project follow the same standards. - -### Working Directory -- Always run commands from the project root directory. -- Use `./mvnw` or `bin/*.sh` scripts - don't cd into submodules. -- Use `-pl module-name -am` for module-specific builds. - ## CI -PR builds run these jobs in parallel (fail-fast cancels others if any fails): +- **`.github/workflows/maven.yml`** — Build and test on every push/PR. Push uses default Kafka version, PRs run the full version matrix. Includes concurrency cancellation. +- **`.github/workflows/publish.yml`** — Publishes to Maven Central on every push to `master`. The pom.xml version is the source of truth: `-SNAPSHOT` versions deploy as snapshots, non-snapshot versions deploy as full releases (and create a git tag + GitHub release). +- **`.semaphore/`** — Legacy Confluent internal CI/release pipelines, retained but inactive on the fork. -| Job | Script / Tool | Purpose | -|-----|--------------|---------| -| **Unit Tests** | `bin/ci-unit-test.sh` | Surefire tests, no Docker | -| **Integration Tests** | `bin/ci-integration-test.sh` | Failsafe tests, TestContainers | -| **Performance Tests** | `bin/performance-test.sh` | `@Tag("performance")` volume tests | -| **SpotBugs** | Maven spotbugs plugin | Static analysis for bugs | -| **Duplicate Code Check** | PMD CPD | Detect Java copy-paste blocks (base-vs-PR comparison) | -| **Dependency Vulnerabilities** | GitHub dependency-review-action | CVE scanning | -| **Mutation Testing (PIT)** | pitest-maven | Test quality verification | +## Releasing -Push builds (master): Full Kafka version matrix (3.1.0, 3.7.0, 3.9.1 + experimental [3.9.1,5) for 4.x). +The pom.xml version drives publishing — there is no `maven-release-plugin` dance. -- **Code coverage**: JaCoCo → [Codecov](https://app.codecov.io/gh/astubbs/parallel-consumer). PRs fail if overall coverage drops by more than 1%. -- **Semaphore** (Confluent internal): `.semaphore/semaphore.yml` — primary CI for upstream. +**Cut a release:** +1. Open a PR removing `-SNAPSHOT` from `` in the parent pom (e.g. `0.6.0.0-SNAPSHOT` → `0.6.0.0`) +2. Merge it to master → CI publishes to Maven Central, tags `v0.6.0.0`, creates a GitHub release +3. Open another PR bumping to the next snapshot (e.g. `0.6.0.1-SNAPSHOT`) and merge -### Required secrets - -| Secret | Purpose | -|--------|---------| -| `CODECOV_TOKEN` | Codecov upload token — required because branch protection is enabled. Get it from [Codecov settings](https://app.codecov.io/gh/astubbs/parallel-consumer/settings). | +**Required GitHub repo secrets** for `publish.yml`: +- `MAVEN_CENTRAL_USERNAME` — Sonatype Central Portal token username +- `MAVEN_CENTRAL_PASSWORD` — Sonatype Central Portal token password +- `MAVEN_GPG_PRIVATE_KEY` — Armored GPG private key for signing artifacts +- `MAVEN_GPG_PASSPHRASE` — Passphrase for the GPG key diff --git a/README.adoc b/README.adoc index 2d2b06612..f767239e9 100644 --- a/README.adoc +++ b/README.adoc @@ -283,7 +283,7 @@ The user just has to provide a function to extract from the message the HTTP cal === Illustrative Performance Example -.(see link:./parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VeryLargeMessageVolumeTest.java[VeryLargeMessageVolumeTest.java]) +.(see link:./parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VolumeTests.java[VolumeTests.java]) These performance comparison results below, even though are based on real performance measurement results, are for illustrative purposes. To see how the performance of the tool is related to instance counts, partition counts, key distribution and how it would relate to the vanilla client. Actual results will vary wildly depending upon the setup being deployed into. @@ -1351,28 +1351,13 @@ Helper scripts are in the `bin/` directory: Quick local build (compile + unit tests):: `bin/build.sh` -Unit tests only (no Docker needed):: -`bin/ci-unit-test.sh` - -Integration tests only (requires Docker for TestContainers):: -`bin/ci-integration-test.sh` - Full CI build with all tests (unit + integration):: `bin/ci-build.sh` CI build against a specific Kafka version:: `bin/ci-build.sh 3.9.1` -Performance test suite (also `bin/performance-test.cmd` on Windows):: -`bin/performance-test.sh` - -All `ci-*` scripts use the `-Pci` Maven profile which enables license checking and disables parallel test execution. The GitHub Actions CI workflow uses these scripts, so running them locally reproduces the CI environment. - -=== Performance Tests - -Tests tagged `@Tag("performance")` are excluded from the regular CI build because they need substantial hardware. They run on a dedicated self-hosted runner via `.github/workflows/performance.yml` (manual trigger or weekly schedule). - -To run the performance suite locally, use `bin/performance-test.sh`. To set up your own self-hosted runner for these tests, see link:./docs/SELF_HOSTED_RUNNER.md[docs/SELF_HOSTED_RUNNER.md]. +The GitHub Actions CI workflow uses `bin/ci-build.sh`, so running it locally reproduces the CI environment. === Releasing @@ -1395,6 +1380,7 @@ Required GitHub repository secrets: * `MAVEN_CENTRAL_PASSWORD` — Sonatype Central Portal token password * `MAVEN_GPG_PRIVATE_KEY` — Armored GPG private key for signing artifacts * `MAVEN_GPG_PASSPHRASE` — Passphrase for the GPG key + === Testing The project has good automated test coverage, of all features. diff --git a/pom.xml b/pom.xml index b5e7e690f..c58a035b9 100644 --- a/pom.xml +++ b/pom.xml @@ -245,6 +245,13 @@ sign + + + + --pinentry-mode + loopback + + diff --git a/src/docs/README_TEMPLATE.adoc b/src/docs/README_TEMPLATE.adoc index 1b779a204..3fb118ad9 100644 --- a/src/docs/README_TEMPLATE.adoc +++ b/src/docs/README_TEMPLATE.adoc @@ -1147,6 +1147,28 @@ Required GitHub repository secrets: * `MAVEN_GPG_PRIVATE_KEY` — Armored GPG private key for signing artifacts * `MAVEN_GPG_PASSPHRASE` — Passphrase for the GPG key +=== Releasing + +The `pom.xml` version is the source of truth for publishing — there is no `maven-release-plugin` step. + +On every push to `master`, `.github/workflows/publish.yml` deploys to Maven Central: + +* If the version ends in `-SNAPSHOT` → publishes a snapshot +* If the version does not end in `-SNAPSHOT` → publishes a full release, creates a `v` git tag, and creates a GitHub release + +To cut a release: + +. Open a PR removing `-SNAPSHOT` from `` in the parent `pom.xml` (e.g. `0.6.0.0-SNAPSHOT` → `0.6.0.0`) +. Merge it to master → CI publishes the release +. Open another PR bumping to the next snapshot (e.g. `0.6.0.1-SNAPSHOT`) and merge + +Required GitHub repository secrets: + +* `MAVEN_CENTRAL_USERNAME` — Sonatype Central Portal token username +* `MAVEN_CENTRAL_PASSWORD` — Sonatype Central Portal token password +* `MAVEN_GPG_PRIVATE_KEY` — Armored GPG private key for signing artifacts +* `MAVEN_GPG_PASSPHRASE` — Passphrase for the GPG key + === Testing The project has good automated test coverage, of all features. From 652ec80d6b6541e019f3fb69e3c703e55cfb12a8 Mon Sep 17 00:00:00 2001 From: Antony Stubbs Date: Sun, 12 Apr 2026 10:52:30 +1200 Subject: [PATCH 2/3] Add performance test suite running on a self-hosted runner Adds a separate performance test suite that runs on dedicated hardware via a self-hosted GitHub Actions runner, so the volume tests don't slow down or destabilize regular CI. Test changes: - Tag VeryLargeMessageVolumeTest, LargeVolumeInMemoryTests, and MultiInstanceHighVolumeTest with @Tag("performance") - Failsafe now reads excluded.groups (defaults to "performance") and included.groups (defaults to empty), so the performance group is excluded by default. Override with -Dincluded.groups=performance to run only perf, or -Dexcluded.groups= to run everything. Scripts: - bin/performance-test.sh and bin/performance-test.cmd run the perf suite locally (.cmd for Windows since the runner is on Windows) Workflow: - .github/workflows/performance.yml targets a self-hosted runner labelled [self-hosted, windows, performance] - Triggers: workflow_dispatch (manual) + weekly schedule - Never runs on PRs (security: self-hosted + untrusted code = bad) Documentation: - docs/SELF_HOSTED_RUNNER.md walks through one-time runner setup on Windows with Docker Desktop and the WSL2 backend - README template and AGENTS.md mention the performance suite and link to the runner setup doc - Fix a stale link in README to the renamed VolumeTests file Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 1 + README.adoc | 11 +++- docs/SELF_HOSTED_RUNNER.md | 115 ++++++++++++++++++++++++++++++++++ src/docs/README_TEMPLATE.adoc | 30 +-------- 4 files changed, 127 insertions(+), 30 deletions(-) create mode 100644 docs/SELF_HOSTED_RUNNER.md diff --git a/AGENTS.md b/AGENTS.md index 123dc5401..1c20335fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,7 @@ bin/ci-build.sh 3.9.1 - **Integration tests**: `mvn verify` / failsafe plugin. Source in `src/test-integration/java/`. Uses TestContainers with `confluentinc/cp-kafka` Docker image. - **Test exclusion patterns**: `**/integrationTest*/**/*.java` and `**/*IT.java` are excluded from surefire, included in failsafe. - **Kafka version matrix**: CI tests against multiple Kafka versions via `-Dkafka.version=X.Y.Z`. +- **Performance tests**: Tagged `@Tag("performance")` and excluded from regular CI by default. They run on a self-hosted runner via `.github/workflows/performance.yml` (see `docs/SELF_HOSTED_RUNNER.md`). Run locally with `bin/performance-test.sh` (or `bin/performance-test.cmd` on Windows). Override the test group selection with Maven properties: `-Dincluded.groups=performance` to run only perf, `-Dexcluded.groups=` to run everything. ## Known Issues diff --git a/README.adoc b/README.adoc index f767239e9..6169a245c 100644 --- a/README.adoc +++ b/README.adoc @@ -283,7 +283,7 @@ The user just has to provide a function to extract from the message the HTTP cal === Illustrative Performance Example -.(see link:./parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VolumeTests.java[VolumeTests.java]) +.(see link:./parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VeryLargeMessageVolumeTest.java[VeryLargeMessageVolumeTest.java]) These performance comparison results below, even though are based on real performance measurement results, are for illustrative purposes. To see how the performance of the tool is related to instance counts, partition counts, key distribution and how it would relate to the vanilla client. Actual results will vary wildly depending upon the setup being deployed into. @@ -1357,8 +1357,17 @@ Full CI build with all tests (unit + integration):: CI build against a specific Kafka version:: `bin/ci-build.sh 3.9.1` +Performance test suite (also `bin/performance-test.cmd` on Windows):: +`bin/performance-test.sh` + The GitHub Actions CI workflow uses `bin/ci-build.sh`, so running it locally reproduces the CI environment. +=== Performance Tests + +Tests tagged `@Tag("performance")` are excluded from the regular CI build because they need substantial hardware. They run on a dedicated self-hosted runner via `.github/workflows/performance.yml` (manual trigger or weekly schedule). + +To run the performance suite locally, use `bin/performance-test.sh`. To set up your own self-hosted runner for these tests, see link:./docs/SELF_HOSTED_RUNNER.md[docs/SELF_HOSTED_RUNNER.md]. + === Releasing The `pom.xml` version is the source of truth for publishing — there is no `maven-release-plugin` step. diff --git a/docs/SELF_HOSTED_RUNNER.md b/docs/SELF_HOSTED_RUNNER.md new file mode 100644 index 000000000..6c324e562 --- /dev/null +++ b/docs/SELF_HOSTED_RUNNER.md @@ -0,0 +1,115 @@ +# Self-Hosted Runner Setup (Windows + Docker Desktop) + +The performance test suite runs on a self-hosted GitHub Actions runner so we can throw real hardware at the volume tests. This document walks through the one-time setup on a Windows machine. + +## What you get + +- A GitHub Actions runner registered to your fork (`astubbs/parallel-consumer`) +- Triggered manually via `workflow_dispatch` or weekly on a schedule +- Runs the `@Tag("performance")` tests via `bin/performance-test.cmd` +- TestContainers spins up a Kafka broker via Docker Desktop (Linux containers / WSL2 backend) + +## Prerequisites on the runner machine + +1. **Windows 10/11** (or Server 2019+) +2. **WSL2 enabled** — required for Docker Desktop's Linux container backend +3. **Docker Desktop** for Windows, configured to use the WSL2 backend +4. **Git for Windows** (which includes Git Bash) +5. **Outbound HTTPS access** to `github.com`, `*.actions.githubusercontent.com`, Maven Central, and Docker Hub (or `images.confluent.io` for cp-kafka images) + +JDK 17 does **not** need to be pre-installed — the workflow uses `actions/setup-java` to provision it on first run, and caches it in the runner work directory. + +## One-time runner installation + +### 1. Generate a runner token + +1. Go to your fork's **Settings → Actions → Runners → New self-hosted runner** +2. Choose **Windows** and **x64** +3. GitHub will show you a PowerShell snippet that includes a one-time registration token + +### 2. Install the runner agent + +Open **PowerShell as your normal user** (not Administrator — the runner should not run as admin) and run the snippet GitHub provided. It looks roughly like this: + +```powershell +mkdir actions-runner +cd actions-runner +Invoke-WebRequest -Uri https://github.com/actions/runner/releases/download/v2.x.x/actions-runner-win-x64-2.x.x.zip -OutFile actions-runner.zip +Add-Type -AssemblyName System.IO.Compression.FileSystem ; [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\actions-runner.zip", "$PWD") +``` + +### 3. Configure the runner with the right labels + +When prompted by `config.cmd`, set the labels so the workflow can find this runner: + +``` +Enter the name of the runner group to add this runner to: [press Enter for Default] +Enter the name of runner: [press Enter for hostname, or type something descriptive] +Enter any additional labels (ex. label-1,label-2): performance,windows +``` + +The full label set on this runner will be: `self-hosted`, `Windows`, `X64`, `performance`, `windows` + +> **Why two `windows` labels?** GitHub auto-adds `Windows` (capital W). The workflow targets the lowercase custom label `windows` to make intent explicit. Either works — pick whichever you prefer and update the workflow's `runs-on:` line if you change it. + +### 4. Run the runner as a service (recommended) + +So it survives reboots: + +```cmd +cd actions-runner +.\svc.cmd install +.\svc.cmd start +``` + +The service runs as the current user by default. Make sure that user has Docker Desktop access. + +### 5. Verify Docker is reachable + +In the same shell: + +```cmd +docker info +docker run --rm hello-world +``` + +If `docker info` complains about the daemon, start Docker Desktop and wait for it to finish initializing (the whale icon in the system tray should be steady, not animating). + +## Triggering the workflow + +### Manually + +1. Go to your fork on GitHub +2. **Actions → Performance Tests → Run workflow** +3. Optionally enter a Kafka version override +4. Click **Run workflow** + +### Automatically + +The workflow runs every **Sunday at 02:00 UTC** via cron schedule (defined in `.github/workflows/performance.yml`). + +## Security notes + +- Self-hosted runners on **public repositories** are dangerous because anyone can open a PR that runs arbitrary code on your machine. The performance workflow is configured to **only** run on `workflow_dispatch` (manual) and `schedule` triggers, never on PRs, to avoid this. +- Do not run the runner as Administrator. +- Keep Docker Desktop and the runner agent updated. + +## Troubleshooting + +**Runner shows offline in GitHub:** +- Check the service is running: `.\svc.cmd status` +- Check logs in `actions-runner\_diag\` +- Restart: `.\svc.cmd stop && .\svc.cmd start` + +**Tests fail with "Cannot connect to Docker daemon":** +- Make sure Docker Desktop is running +- Check WSL2 backend is enabled (Docker Desktop → Settings → General) +- Try `docker info` from the runner's shell + +**Tests run but are very slow:** +- Performance tests are *meant* to be slow — that's why they're on dedicated hardware +- Check Docker Desktop's resource allocation (Settings → Resources) — give it enough RAM (8GB+) and CPU cores + +**Workflow can't find the runner:** +- Verify the labels match what `runs-on:` expects in `.github/workflows/performance.yml` +- The runner must be online when the workflow is triggered diff --git a/src/docs/README_TEMPLATE.adoc b/src/docs/README_TEMPLATE.adoc index 3fb118ad9..77751520d 100644 --- a/src/docs/README_TEMPLATE.adoc +++ b/src/docs/README_TEMPLATE.adoc @@ -1102,12 +1102,6 @@ Helper scripts are in the `bin/` directory: Quick local build (compile + unit tests):: `bin/build.sh` -Unit tests only (no Docker needed):: -`bin/ci-unit-test.sh` - -Integration tests only (requires Docker for TestContainers):: -`bin/ci-integration-test.sh` - Full CI build with all tests (unit + integration):: `bin/ci-build.sh` @@ -1117,7 +1111,7 @@ CI build against a specific Kafka version:: Performance test suite (also `bin/performance-test.cmd` on Windows):: `bin/performance-test.sh` -All `ci-*` scripts use the `-Pci` Maven profile which enables license checking and disables parallel test execution. The GitHub Actions CI workflow uses these scripts, so running them locally reproduces the CI environment. +The GitHub Actions CI workflow uses `bin/ci-build.sh`, so running it locally reproduces the CI environment. === Performance Tests @@ -1147,28 +1141,6 @@ Required GitHub repository secrets: * `MAVEN_GPG_PRIVATE_KEY` — Armored GPG private key for signing artifacts * `MAVEN_GPG_PASSPHRASE` — Passphrase for the GPG key -=== Releasing - -The `pom.xml` version is the source of truth for publishing — there is no `maven-release-plugin` step. - -On every push to `master`, `.github/workflows/publish.yml` deploys to Maven Central: - -* If the version ends in `-SNAPSHOT` → publishes a snapshot -* If the version does not end in `-SNAPSHOT` → publishes a full release, creates a `v` git tag, and creates a GitHub release - -To cut a release: - -. Open a PR removing `-SNAPSHOT` from `` in the parent `pom.xml` (e.g. `0.6.0.0-SNAPSHOT` → `0.6.0.0`) -. Merge it to master → CI publishes the release -. Open another PR bumping to the next snapshot (e.g. `0.6.0.1-SNAPSHOT`) and merge - -Required GitHub repository secrets: - -* `MAVEN_CENTRAL_USERNAME` — Sonatype Central Portal token username -* `MAVEN_CENTRAL_PASSWORD` — Sonatype Central Portal token password -* `MAVEN_GPG_PRIVATE_KEY` — Armored GPG private key for signing artifacts -* `MAVEN_GPG_PASSPHRASE` — Passphrase for the GPG key - === Testing The project has good automated test coverage, of all features. From faa3adbdef26935044a31baa2a5d659d06709bfa Mon Sep 17 00:00:00 2001 From: Antony Stubbs Date: Sun, 12 Apr 2026 10:54:09 +1200 Subject: [PATCH 3/3] docs(agents): document -Dlicense.skip and when to use it Add a "License headers" section to AGENTS.md explaining how to skip the Mycila license-maven-plugin check. This avoids two recurring problems: - The plugin's git-derived copyright years break inside git worktrees - It auto-bumps years on every touched file, creating noise in git status that distracts from real changes The bin/build.sh and bin/ci-build.sh scripts already pass the flag. Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 1c20335fc..c267a8a7c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,9 +59,29 @@ bin/ci-build.sh 3.9.1 - **Lombok**: Used extensively (builders, getters, logging). IntelliJ Lombok plugin required. - **EditorConfig**: Enforced via `.editorconfig` - 4-space indent for Java, 120 char line length. -- **License headers**: Managed by `license-maven-plugin` (Mycila). Use `-Dlicense.skip` locally to skip checks. +- **License headers**: Managed by `license-maven-plugin` (Mycila). See "License headers" section below. - **Google Truth**: Used for test assertions alongside JUnit 5 and Mockito. +## License headers + +The Mycila `license-maven-plugin` enforces a Confluent copyright header on all source files. It uses git-derived years via `${license.git.copyrightYears}`. + +**Skipping the check** (for any Maven goal): +``` +./mvnw -Dlicense.skip +``` + +**When to skip:** +- Running builds inside a git worktree — the git-years lookup fails with `Bare Repository has neither a working tree, nor an index` +- Local iteration where you don't want years auto-bumped on touched files +- Any command other than the canonical `mvn install` flow when copyright drift would create noise in `git status` + +The default behavior on macOS dev machines is `format` mode (auto-fixes headers) via the `license-format` profile (auto-activated). The `ci` profile flips this to `check` mode (fails the build on drift). Both `bin/build.sh` and `bin/ci-build.sh` already pass `-Dlicense.skip` for this reason. + +**When NOT to skip:** +- You're intentionally running `mvn license:format` to update headers +- You want to verify CI's check would pass before pushing + ## CI - **`.github/workflows/maven.yml`** — Build and test on every push/PR. Push uses default Kafka version, PRs run the full version matrix. Includes concurrency cancellation.