Taskfile.yaml

version: '3'

# Only targets that compose, inject env, or encode a non-obvious default.
# Pure CLI wrappers (uv/sqlmesh/dagster) live in docs/commands.md.

vars: { VENV_DIR: .venv }
env:
  PYTHONPATH: .
  RUNTIME__DLTHUB_TELEMETRY: "false"
  SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: "true"

tasks:
  install:
    desc: "Bootstrap .venv, copy .env, uv sync, install pre-commit hook"
    cmds:
      - test -d {{.VENV_DIR}} || uv venv {{.VENV_DIR}}
      - test -f .env || cp .env.example .env
      - uv sync --locked
      - ./scripts/operations/setup_pre_commit.sh
      # Route SQLMesh's hard-coded `logs/` dir into `.logs/sqlmesh/` so the
      # repo keeps one log root. No equivalent env var / config knob exists
      # in SQLMesh — the symlink is the least-ugly option.
      - mkdir -p .logs/sqlmesh
      - test -e transforms/main/logs || ln -s ../../.logs/sqlmesh transforms/main/logs
    sources: [pyproject.toml, uv.lock, scripts/operations/setup_pre_commit.sh]

  ci:
    desc: "Every CI gate: ruff + mypy + pytest + secret scan + staging drift"
    deps: [install]
    cmds:
      - "{{.VENV_DIR}}/bin/ruff check ."
      - "{{.VENV_DIR}}/bin/ruff format --check ."
      - "{{.VENV_DIR}}/bin/mypy packages/"
      - "{{.VENV_DIR}}/bin/pytest"
      - "{{.VENV_DIR}}/bin/python scripts/platform/check_secrets.py ."
      - "{{.VENV_DIR}}/bin/python scripts/analytics/generate_staging.py --check"
      - "{{.VENV_DIR}}/bin/python scripts/analytics/generate_platform_health.py --check"

  docs:build:
    desc: "Generate dictionary, build MkDocs (strict)"
    deps: [install]
    cmds:
      - "{{.VENV_DIR}}/bin/python scripts/platform/generate_docs.py"
      - "{{.VENV_DIR}}/bin/mkdocs build --strict"

  plan:dev:
    desc: "SQLMesh plan against the dev virtual env (interactive)"
    dir: transforms/main
    cmds: ["../../{{.VENV_DIR}}/bin/sqlmesh plan dev {{.CLI_ARGS}}"]
  plan:prod:
    desc: "SQLMesh plan against prod — fails if prod is ahead of dev"
    dir: transforms/main
    cmds: ["../../{{.VENV_DIR}}/bin/sqlmesh plan prod {{.CLI_ARGS}}"]
  promote:
    desc: "Auto-apply dev → prod promotion (assumes dev verified)"
    dir: transforms/main
    cmds: ["../../{{.VENV_DIR}}/bin/sqlmesh plan prod --auto-apply {{.CLI_ARGS}}"]
  verify:dev:
    desc: "Run every Soda contract against the __dev schemas"
    cmds: ["{{.VENV_DIR}}/bin/python scripts/platform/verify_dev.py"]
  full-refresh:
    desc: "Concurrent Dagster ingestion through Polaris Iceberg, then project-wide SQLMesh and Soda — logs to .logs/"
    env: { DAGSTER_HOME: "{{.USER_WORKING_DIR}}/.dagster" }
    cmds:
      - "./scripts/operations/run-logged.sh full-refresh -- {{.VENV_DIR}}/bin/python scripts/sources/load_dlt_iceberg.py"
  verify:
    desc: "Smoke concurrent Dagster source jobs through Polaris Iceberg, then native SQLMesh — logs to .logs/"
    env: { DAGSTER_HOME: "{{.USER_WORKING_DIR}}/.dagster", DATABOX_SMOKE: "1" }
    cmds:
      - "./scripts/operations/run-logged.sh verify -- {{.VENV_DIR}}/bin/python scripts/sources/load_dlt_iceberg.py"
  catalog:backup-check:
    desc: "Validate pgBackRest stanza and WAL archive configuration"
    cmds: ["docker compose --env-file .env -f compose.iceberg.yml exec --user postgres postgres /usr/local/bin/run-pgbackrest --stanza=polaris check"]
  catalog:backup-full:
    desc: "Create the weekly full Polaris catalog backup"
    cmds: ["docker compose --env-file .env -f compose.iceberg.yml exec --user postgres postgres /usr/local/bin/run-pgbackrest --stanza=polaris --type=full backup"]
  catalog:backup-diff:
    desc: "Create the daily differential Polaris catalog backup"
    cmds: ["docker compose --env-file .env -f compose.iceberg.yml exec --user postgres postgres /usr/local/bin/run-pgbackrest --stanza=polaris --type=diff backup"]
  catalog:backup-info:
    desc: "Inspect pgBackRest backup and WAL state"
    cmds: ["docker compose --env-file .env -f compose.iceberg.yml exec --user postgres postgres /usr/local/bin/run-pgbackrest --stanza=polaris info"]
  catalog:recovery-drill:
    desc: "Run or reconcile-and-run the interactive isolated catalog PITR drill"
    cmds: ["{{.VENV_DIR}}/bin/python scripts/platform/catalog_recovery.py drill {{.CLI_ARGS}}"]
  catalog:recovery-stage2a:
    desc: "Run one isolated synthetic POSIX pgBackRest PITR proof"
    cmds: ["scripts/platform/catalog-recovery-stage2a"]
  catalog:recovery-stage2c:
    desc: "Prepare, execute, restore, or validate one isolated joint recovery proof"
    cmds: ["{{.VENV_DIR}}/bin/python scripts/platform/catalog_warehouse_recovery_stage2c.py {{.CLI_ARGS}}"]
  recovery:cleanup:
    desc: "Prepare or execute one exact retained recovery-drill cleanup"
    cmds: ["{{.VENV_DIR}}/bin/python scripts/platform/recovery_cleanup.py {{.CLI_ARGS}}"]
  iceberg:recovery-drill:prepare:
    desc: "Prepare one generated real Iceberg object-recovery drill"
    cmds: ["{{.VENV_DIR}}/bin/python scripts/platform/iceberg_recovery_drill.py prepare {{.CLI_ARGS}}"]
  iceberg:recovery-drill:execute:
    desc: "Execute or safely resume one approved Iceberg object-recovery drill"
    cmds: ["{{.VENV_DIR}}/bin/python scripts/platform/iceberg_recovery_drill.py execute {{.CLI_ARGS}}"]

  dagster:dev:
    desc: "Launch the local Dagster UI for Polaris Iceberg pipelines"
    env: { DAGSTER_HOME: "{{.USER_WORKING_DIR}}/.dagster" }
    cmds:
      - |
        bash -lc '
          set -euo pipefail
          repo=$PWD
          tmp=$(mktemp -d)
          cleanup() { rm -rf "$tmp"; }
          trap cleanup EXIT
          export RUNTIME__DLTHUB_TELEMETRY=false SQLMESH__DISABLE_ANONYMIZED_ANALYTICS=true PYTHONPATH="$repo${PYTHONPATH:+:$PYTHONPATH}"
          export VIRTUAL_ENV="$repo/{{.VENV_DIR}}"
          export PATH="$VIRTUAL_ENV/bin:$PATH"
          cd "$tmp"
          "$repo/{{.VENV_DIR}}/bin/dg" dev --target-path "$repo/packages/databox" --use-active-venv {{.CLI_ARGS}}
        '

  init:
    desc: "Rebrand a fresh fork from scaffold.yaml (see docs/template.md)"
    cmds: ["{{.VENV_DIR}}/bin/python scripts/platform/bootstrap.py {{.CLI_ARGS}}"]
  new-source:
    desc: "Scaffold a new dlt source (see docs/new-source.md)"
    cmds: ["{{.VENV_DIR}}/bin/python scripts/sources/new_source.py {{.CLI_ARGS}}"]

  db:reset:
    desc: "Delete local DuckDB files, including persisted trip plans"
    cmds: ["rm -rf data/*.duckdb data/*.duckdb.wal data/.quack-clients"]
  clean:
    desc: "Remove build + test + cache artifacts"
    cmds:
      - >-
        rm -rf build/ dist/ *.egg-info/ .pytest_cache/ .coverage htmlcov/
        .mypy_cache/ .ruff_cache/ site/ logs/
        packages/databox/data/ transforms/main/data/ transforms/main/.cache/
        .dagster_state/ .dlt_state/ .duckdb_state/ .sqlmesh_state/
      - >-
        find . \( -path ./.git -o -path ./{{.VENV_DIR}} -o -path ./data -o -path ./.dagster -o -path ./.logs \) -prune
        -o -type d \( -name __pycache__ -o -name '*.egg-info' \) -exec rm -rf {} +
  clean-all:
    desc: "Clean + drop .venv and data/"
    deps: [clean]
    cmds: ["rm -rf {{.VENV_DIR}} data"]