feat(orchestrator): ci-excluded task types

scripts/smoke-test.sh

@@ -24,14 +24,16 @@ node --input-type=module -e "import '$DIST_BIN'" 2>&1 | head -5 | grep -q 'PostH
 # builds and tsdown strips it; its env var name appearing in dist/*.js means
 # dead-code elimination regressed and a prod surface leaked. Sourcemaps keep
 # the original source, so only .js output counts.
-OVERRIDE_MARKER='WIZARD_CI_FLAG_OVERRIDES'
+OVERRIDE_MARKERS='WIZARD_CI_FLAG_OVERRIDES WIZARD_CI_EXCLUDE_TASKS'
 if [ "${WIZARD_BUILD_NODE_ENV:-production}" = "ci" ]; then
-  # CI builds must keep the path — its absence means the override silently
-  # stopped working and CI is back to testing live flags.
-  if ! grep -q "$OVERRIDE_MARKER" ./dist/*.js; then
-    echo 'Smoke test failed: CI build is missing the CI flag-override path' >&2
-    exit 1
-  fi
+  # CI builds must keep the paths — their absence means the overrides silently
+  # stopped working and CI is back to testing live behavior.
+  for marker in $OVERRIDE_MARKERS; do
+    if ! grep -q "$marker" ./dist/*.js; then
+      echo "Smoke test failed: CI build is missing the $marker path" >&2
+      exit 1
+    fi
+  done
   # And a real invocation must accept the env var. yargs claims every
   # POSTHOG_WIZARD_-prefixed env var as a CLI option and strict-rejects
   # unknown ones during command parse (--version/--help short-circuit and
@@ -44,10 +46,12 @@ if [ "${WIZARD_BUILD_NODE_ENV:-production}" = "ci" ]; then
     exit 1
   fi
 else
-  if grep -q "$OVERRIDE_MARKER" ./dist/*.js; then
-    echo 'Smoke test failed: CI flag-override code leaked into a production build' >&2
-    exit 1
-  fi
+  for marker in $OVERRIDE_MARKERS; do
+    if grep -q "$marker" ./dist/*.js; then
+      echo "Smoke test failed: $marker code leaked into a production build" >&2
+      exit 1
+    fi
+  done
 fi
 
 # ── 3. --ci rejected in production builds ────────────────────────────────────

src/env.ts

@@ -43,6 +43,7 @@ type RuntimeEnvKey =
   // Deliberately NOT POSTHOG_WIZARD_-prefixed: yargs .env('POSTHOG_WIZARD')
   // would claim it as an unknown CLI option and strict-reject the run.
   | 'WIZARD_CI_FLAG_OVERRIDES'
+  | 'WIZARD_CI_EXCLUDE_TASKS'
   // Wizard CLI configuration (yargs POSTHOG_WIZARD_ prefix)
   | 'POSTHOG_WIZARD_BENCHMARK_CONFIG'
   | 'POSTHOG_WIZARD_BENCHMARK_FILE'

src/lib/programs/orchestrator/__tests__/agent-prompt-loader.test.ts

@@ -131,6 +131,18 @@ describe('buildRegistry', () => {
     // A flowless prompt (e.g. the documentation example) joins no registry.
     expect(registry.get('example')).toBeUndefined();
   });
+
+  it('drops harness-excluded types; unrestricted runs keep them', () => {
+    const prompts = [
+      prompt({ type: 'plan', flow: 'f', seed: true }),
+      prompt({ type: 'build', flow: 'f' }),
+      prompt({ type: 'dashboard', flow: 'f' }),
+    ];
+    expect(
+      buildRegistry(prompts, 'f', { exclude: ['dashboard'] }).types,
+    ).toEqual(['build']);
+    expect(buildRegistry(prompts, 'f').types).toEqual(['build', 'dashboard']);
+  });
 });
 
 describe('resolveTask', () => {

src/lib/programs/orchestrator/agent-prompt-loader.ts

@@ -137,8 +137,15 @@ export interface AgentRegistry {
 export function buildRegistry(
   prompts: readonly AgentPrompt[],
   flow: string,
+  opts?: { exclude?: readonly string[] },
 ): AgentRegistry {
-  const inFlow = prompts.filter((p) => p.flow === flow);
+  // The harness can exclude task types (CI excludes dashboards). An excluded
+  // type does not exist for the run: the seed cannot enqueue it and no agent
+  // is ever spun up for it.
+  const excluded = new Set(opts?.exclude ?? []);
+  const inFlow = prompts.filter(
+    (p) => p.flow === flow && !excluded.has(p.type),
+  );
   const byType = new Map(inFlow.map((p) => [p.type, p]));
   return {
     types: inFlow.filter((p) => !p.seed).map((p) => p.type),
@@ -238,6 +245,7 @@ async function fetchText(url: string): Promise<string> {
 export async function loadAgentRegistry(
   skillsBaseUrl: string,
   flow: string,
+  opts?: { exclude?: readonly string[] },
 ): Promise<AgentRegistry> {
   const menuRaw = await fetchText(`${skillsBaseUrl}/agent-menu.json`);
   const menu = JSON.parse(menuRaw) as AgentMenu;
@@ -249,7 +257,7 @@ export async function loadAgentRegistry(
     }),
   );
 
-  return buildRegistry(prompts, flow);
+  return buildRegistry(prompts, flow, opts);
 }
 
 /**

src/lib/programs/orchestrator/orchestrator-runner.ts

@@ -23,6 +23,7 @@ import { detectNodePackageManagers } from '../../detection/package-manager';
 import { installSkillById } from '../../wizard-tools';
 import { getUI } from '../../../ui';
 import { analytics } from '../../../utils/analytics';
+import { ciExcludedTaskTypes } from '../../../utils/ci-flag-overrides';
 import { logToFile } from '../../../utils/debug';
 import type { ProgramConfig } from '../program-step';
 import type { BootstrapResult } from '../../agent/agent-runner';
@@ -88,6 +89,7 @@ export async function runOrchestrator(
   const registry = await loadAgentRegistry(
     boot.skillsBaseUrl,
     programConfig.id,
+    { exclude: ciExcludedTaskTypes() },
   );
   const seedPrompt = registry.seed;
   if (!seedPrompt) {

src/utils/__tests__/ci-flag-overrides.test.ts

@@ -1,4 +1,7 @@
-import { applyCiFlagOverrides } from '@utils/ci-flag-overrides';
+import {
+  applyCiFlagOverrides,
+  ciExcludedTaskTypes,
+} from '@utils/ci-flag-overrides';
 
 jest.mock('@utils/debug', () => ({
   logToFile: jest.fn(),
@@ -61,3 +64,34 @@ describe('applyCiFlagOverrides', () => {
     });
   });
 });
+
+describe('ciExcludedTaskTypes', () => {
+  afterEach(() => {
+    delete process.env.WIZARD_CI_EXCLUDE_TASKS;
+  });
+
+  it('is empty when nothing is excluded', () => {
+    expect(ciExcludedTaskTypes()).toEqual([]);
+  });
+
+  it('parses the comma-separated list, ignoring stray whitespace', () => {
+    process.env.WIZARD_CI_EXCLUDE_TASKS = 'dashboard, report ,';
+    expect(ciExcludedTaskTypes()).toEqual(['dashboard', 'report']);
+  });
+
+  it('is inert in production builds', () => {
+    const prevNodeEnv = process.env.NODE_ENV;
+    process.env.NODE_ENV = 'production';
+    process.env.WIZARD_CI_EXCLUDE_TASKS = 'dashboard';
+    let result: readonly string[] | undefined;
+    jest.isolateModules(() => {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const prod = require('@utils/ci-flag-overrides') as {
+        ciExcludedTaskTypes: typeof ciExcludedTaskTypes;
+      };
+      result = prod.ciExcludedTaskTypes();
+    });
+    process.env.NODE_ENV = prevNodeEnv;
+    expect(result).toEqual([]);
+  });
+});

src/utils/ci-flag-overrides.ts

@@ -3,14 +3,15 @@
  *
  * CI must route deterministically: a run that tests the orchestrator arm says
  * so explicitly instead of depending on a live feature flag someone can edit
- * mid-week. `WIZARD_CI_FLAG_OVERRIDES` is a JSON object of flag key →
- * value, merged over whatever PostHog returned.
+ * mid-week. The override env var (see the allowlist in `env.ts`) is a JSON
+ * object of flag key → value, merged over whatever PostHog returned.
  *
  * The override path exists only in CI builds (`pnpm build:ci`). Published
- * builds inline NODE_ENV as the literal "production", the guard below
- * collapses, and tsdown strips the rest from the bundle — and the smoke test
- * asserts the env var's name is physically absent from production output, so
- * this can never quietly become a production surface.
+ * builds inline NODE_ENV as the literal "production", the guards collapse,
+ * and tsdown strips the rest from the bundle — and the smoke test asserts the
+ * env var names are physically absent from production output (which is also
+ * why no comment in this file may spell them out), so this can never quietly
+ * become a production surface.
  */
 import { runtimeEnv } from '@env';
 import { logToFile } from './debug';
@@ -33,7 +34,7 @@ export function applyCiFlagOverrides(
     // A malformed override is a CI misconfiguration. Fail the run loudly
     // rather than silently testing whatever the live flags happen to say.
     throw new Error(
-      'WIZARD_CI_FLAG_OVERRIDES is not valid JSON (expected {"flag-key": value, ...}).',
+      'The CI flag-override env var is not valid JSON (expected {"flag-key": value, ...}).',
     );
   }
 
@@ -44,3 +45,23 @@ export function applyCiFlagOverrides(
   logToFile('[flags] CI overrides applied', overrides);
   return merged;
 }
+
+/**
+ * Task types excluded from this run. The exclusion env var (see the allowlist
+ * in `env.ts`) is a comma-separated list (e.g. `dashboard`), set by the CI
+ * harness that owns the policy — the wizard and the served content stay
+ * run-mode agnostic. CI-build only, same as the flag overrides: published
+ * builds strip this path.
+ */
+export function ciExcludedTaskTypes(): readonly string[] {
+  if (process.env.NODE_ENV === 'production') return [];
+
+  const raw = runtimeEnv('WIZARD_CI_EXCLUDE_TASKS');
+  if (!raw) return [];
+  const types = raw
+    .split(',')
+    .map((t) => t.trim())
+    .filter(Boolean);
+  if (types.length > 0) logToFile('[flags] CI task exclusions', types);
+  return types;
+}