Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ requirements-sbom.txt

# Project-specific: local DB, generated reports and heatmaps (do not commit scan results or credentials)
audit_results.db
audit_results_rc_*.db
audit_results.db-journal
# Default CLI/dashboard SQLite at repo root (check-all / --validate-config / local runs)
/audit.db
Expand Down
175 changes: 175 additions & 0 deletions cli/bash_completion_inventory.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
"""Inventory for published CLI Bash completion (#1952, anti-drift with #1448).

The tuples below are the contract checked by ``tests/test_bash_completion_inventory.py``
against live ``--help`` output. Update this module when argparse or console scripts change.
"""

from __future__ import annotations

import re
from dataclasses import dataclass
from typing import Mapping

PUBLISHED_CONSOLE_SCRIPTS: tuple[str, ...] = (
"data-boar",
"data-boar-report",
"data-boar-xlsx",
"data-boar-pdf",
"data-boar-docx",
"data-boar-heatmap",
"data-boar-dsar",
"data-boar-audit",
)

_REPORT_FORMAT_CHOICES: frozenset[str] = frozenset(
{
"all",
"audit-trail",
"docx",
"dsar",
"heatmap",
"json",
"md",
"pdf",
"xlsx",
}
)

_REPORT_LONG_OPTIONS: frozenset[str] = frozenset(
{
"--config",
"--debug-traceback",
"--dsar-include-samples",
"--format",
"--help",
"--output",
"--output-dir",
"--session-id",
"--sqlite",
"--trial-rows-capped",
}
)

_DATA_BOAR_LONG_OPTIONS: frozenset[str] = frozenset(
{
"--allow-insecure-http",
"--allow-sample-export",
"--check-extras",
"--config",
"--confirm-upgrade-to",
"--content-type-check",
"--demo",
"--diff",
"--dsar-include-samples",
"--dsar-output",
"--export-audit-trail",
"--export-dsar",
"--export-findings-sink",
"--export-l1",
"--export-l3",
"--export-remediation-manifest",
"--fail-on-new-high",
"--governance-report",
"--help",
"--host",
"--https-cert-file",
"--https-key-file",
"--jurisdiction-hint",
"--l1-output",
"--l3-allow-unprotected",
"--l3-column",
"--l3-grant",
"--l3-max-rows",
"--l3-persist",
"--no-progress",
"--plan",
"--port",
"--prefilter-status",
"--progress",
"--reconcile-integrity-anchor",
"--regenerate-report",
"--reset-data",
"--resume",
"--scan-compressed",
"--scan-stego",
"--session",
"--technician",
"--tenant",
"--validate-config",
"--validate-crypto",
"--version",
"--web",
}
)


@dataclass(frozen=True)
class CommandCompletionInventory:
"""Known flags and enumerated values for one console script."""

long_options: frozenset[str]
short_options: frozenset[str]
choice_flags: Mapping[str, frozenset[str]]


def _report_inventory() -> CommandCompletionInventory:
return CommandCompletionInventory(
long_options=_REPORT_LONG_OPTIONS,
short_options=frozenset({"-h", "-o"}),
choice_flags={"--format": _REPORT_FORMAT_CHOICES},
)


COMMAND_COMPLETION_INVENTORY: dict[str, CommandCompletionInventory] = {
"data-boar": CommandCompletionInventory(
long_options=_DATA_BOAR_LONG_OPTIONS,
short_options=frozenset({"-h"}),
choice_flags={},
),
"data-boar-report": _report_inventory(),
"data-boar-xlsx": _report_inventory(),
"data-boar-pdf": _report_inventory(),
"data-boar-docx": _report_inventory(),
"data-boar-heatmap": _report_inventory(),
"data-boar-dsar": _report_inventory(),
"data-boar-audit": _report_inventory(),
}


def long_options_from_help_text(help_text: str) -> frozenset[str]:
"""Parse documented long options from argparse ``--help`` text."""
opts: set[str] = set()
for line in help_text.splitlines():
if line.startswith("usage:"):
for match in re.finditer(r"(--[a-z][a-z0-9-]+)", line):
opts.add(match.group(1))
elif line.startswith(" -"):
for token in line.split():
if token.startswith("--"):
opts.add(token.rstrip(",:"))
return frozenset(opts)


def short_options_from_help_text(help_text: str) -> frozenset[str]:
"""Parse documented short options from argparse ``--help`` text."""
opts: set[str] = set()
for line in help_text.splitlines():
if not line.startswith(" -"):
continue
for token in line.split():
if re.fullmatch(r"-[a-zA-Z]", token.rstrip(",")):
opts.add(token.rstrip(","))
usage = help_text.splitlines()[0] if help_text else ""
for match in re.finditer(r"(?<![\w-])(-[a-zA-Z])(?![\w-])", usage):
opts.add(match.group(1))
return frozenset(opts)


def choice_flags_from_help_text(help_text: str) -> dict[str, frozenset[str]]:
"""Parse ``--flag {a,b,c}`` choice lists from help."""
choices: dict[str, frozenset[str]] = {}
for match in re.finditer(r"(--[a-z][a-z0-9-]*)\s+\{([^}]+)\}", help_text):
flag = match.group(1)
values = frozenset(part.strip() for part in match.group(2).split(","))
choices[flag] = values
return choices
10 changes: 10 additions & 0 deletions docs/TECH_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,16 @@ uv sync
pip install -e .
```

### Bash tab completion (optional)

After `uv sync`, you can enable tab completion for the eight published console scripts (`data-boar`, `data-boar-report`, and the format shortcuts). Source the tracked script from your clone (adjust the path):

```bash
source /path/to/data-boar/scripts/bash-completion/data-boar.bash
```

Inventory and drift checks live in `cli/bash_completion_inventory.py` and `tests/test_bash_completion_inventory.py` (#1952; complements CLI↔docs anti-drift in #1448). This does not replace `docs/data_boar.1` / `docs/data_boar.5` or the `uv sync` install path above.

### `[noavx]` / x86-64-v1 (pip / pipx)

**`[noavx]` is not a PyPI extra.** On hosts without SSE4.2, POPCNT, or AVX, PyPI numpy **SIGILL**s. Restore **working ML** with the verified wheelhouse ([#929](https://github.com/DataBoar/data-boar/issues/929)): tag `wheelhouse-x86-64-v1-2026-07-29`, two-step `gh release download` then `pip install --no-index --find-links` (or `pipx runpip`). Full recipe: [TROUBLESHOOTING.md](TROUBLESHOOTING.md) *x86-64-v1 / wheelhouse install*. Pre-flight skips the crash; it does **not** replace that install.
Expand Down
10 changes: 10 additions & 0 deletions docs/TECH_GUIDE.pt_BR.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,16 @@ uv sync
pip install -e .
```

### Tab completion no Bash (opcional)

Depois do `uv sync`, você pode ativar o completion dos oito scripts publicados (`data-boar`, `data-boar-report` e os atalhos de formato). Carregue o script versionado do clone (ajuste o caminho):

```bash
source /caminho/para/data-boar/scripts/bash-completion/data-boar.bash
```

O inventário e os testes anti-drift estão em `cli/bash_completion_inventory.py` e `tests/test_bash_completion_inventory.py` (#1952; complementa o anti-drift CLI↔docs do #1448). Isso não substitui `docs/data_boar.1` / `docs/data_boar.5` nem o fluxo de instalação com `uv sync` acima.

### `[noavx]` / x86-64-v1 (pip / pipx)

**`[noavx]` não é extra do PyPI.** Em hosts sem SSE4.2, POPCNT ou AVX, o numpy do PyPI dá **SIGILL**. Restaure **ML de verdade** com o wheelhouse verificado ([#929](https://github.com/DataBoar/data-boar/issues/929)): tag `wheelhouse-x86-64-v1-2026-07-29`, dois passos `gh release download` e `pip install --no-index --find-links` (ou `pipx runpip`). Receita: [TROUBLESHOOTING.pt_BR.md](TROUBLESHOOTING.pt_BR.md) *x86-64-v1 / wheelhouse*. O pre-flight evita o crash; **não** substitui essa instalação.
Expand Down
2 changes: 2 additions & 0 deletions docs/plans/PLANS_HUB.md
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,8 @@ Do **not** edit the table manually; refresh with `python scripts/plans_hub_sync.
| **Open** | [PLAN_SECRETS_VAULT.md](PLAN_SECRETS_VAULT.md) | Plan: Secrets and password protection (vault and alternatives) | Phase A shipped (env refs, config redaction, docs); Phases B-C local vault + re-import remain backlog. | — |
| **Open** | [PLAN_SELENIUM_QA_TEST_SUITE.md](PLAN_SELENIUM_QA_TEST_SUITE.md) | Plan: Selenium-based QA test suite (on-demand, report and recommendations) | **Status:** Deferred **Date:** 2026-03-15 **Authors:** Fabio Leitao **Priority:** H3 | — |
| **Open** | [PLAN_SELF_UPGRADE_AND_VERSION_CHECK.md](PLAN_SELF_UPGRADE_AND_VERSION_CHECK.md) | Plan: Version check and self-upgrade (with container-aware behaviour) | **Status:** Deferred **Date:** 2026-03-15 **Authors:** Fabio Leitao **Priority:** H3 | — |
| **Open** | [PLAN_SHELL_SCRIPT_HARDENING.md](PLAN_SHELL_SCRIPT_HARDENING.md) | Plan: Shell script hardening (shellcheck + selective bats) | Tracked `.sh` inventory, error-level shellcheck in check-all, and hermetic bats for pure-parse helpers (#1926 part 2). | — |
| **Open** | [PLAN_SHELL_SCRIPT_HARDENING.pt_BR.md](PLAN_SHELL_SCRIPT_HARDENING.pt_BR.md) | Plano: Endurecimento de scripts shell (shellcheck + bats seletivo) | Inventário de `.sh` versionados, shellcheck severity=error no check-all e bats hermético para helpers de parse (#1926 parte 2). | — |
| **Open** | [PLAN_STRICTO_SENSU_RESEARCH_PATH.md](PLAN_STRICTO_SENSU_RESEARCH_PATH.md) | Plan: Stricto sensu research path (M.Sc. and PhD) on top of Data Boar | M.Sc./PhD research lines using Data Boar as experimental platform—detection, evidence, methodology; revisit when programs/advisors near. | — |
| **Open** | [PLAN_SUBJECT_CATEGORY_AXIS.md](PLAN_SUBJECT_CATEGORY_AXIS.md) | Plan: Optional subject category axis (data-subject / titular category) | Optional configurable subject-category axis (Bearer-style subject_mapping) complementing norm_tag; audit #1069 gap; candidate v1.9.0. | [PLAN_TAXONOMY_AXES.md](PLAN_TAXONOMY_AXES.md) [PLAN_COMPLIANCE_EXPANSION_GLOBAL_JURISDICTIONS.md](PLAN_COMPLIANCE_EXPANSION_GLOBAL_JURISDICTIONS.md) [PLAN_ADDITIONAL_DETECTION_TECHNIQUES_AND_FN_REDUCTION.md](PLAN_ADDITIONAL_DETECTION_TECHNIQUES_AND_FN_REDUCTION.md) |
| **Open** | [PLAN_SUBSCRIPTION_TIER_CANON.md](PLAN_SUBSCRIPTION_TIER_CANON.md) | Plan: Canonical subscription-band doc (no public prices) | Escada canônica de 6 faixas em SUBSCRIPTION_TIERS — sem página pública de preços | [PLAN_PRODUCT_TIERS_AND_OPEN_CORE.md](PLAN_PRODUCT_TIERS_AND_OPEN_CORE.md) |
Expand Down
124 changes: 124 additions & 0 deletions docs/plans/PLAN_SHELL_SCRIPT_HARDENING.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
# Plan: Shell script hardening (shellcheck + selective bats)

<!-- plans-hub-summary: Tracked `.sh` inventory, error-level shellcheck in check-all, and hermetic bats for pure-parse helpers (#1926 part 2). -->

**Status:** In progress
**Date:** 2026-10-11
**Authors:** Fabio Leitao
**Priority:** P3
**GitHub:** [#1926](https://github.com/DataBoar/data-boar/issues/1926)

**Synced with:** [PLANS_TODO.md](PLANS_TODO.md)

## Goal

Harden tracked Bash scripts without turning every lab wrapper into CI noise: enforce **shellcheck severity=error** in both `check-all` twins (#1926 part 1, already on `main`), document a **concrete per-file verdict**, and add a **small hermetic bats** slice wired as an **optional** local step (skip with a visible warning when `bats` is missing). **No bats job in CI.**

## Bats qualification criterion

A script **qualifies for bats** when it has **real branching, argument parsing, or subprocess orchestration** that can be exercised **hermetically** (tmp dirs only, no network, no `sudo`, no Docker). Scripts that only wrap long lab/ansible/docker flows are inventoried but **deferred**. Library `.sh` files meant to be **sourced** qualify when pure functions can be tested without side effects.

| Bats verdict | Meaning |
| ------------ | ------- |
| **yes** | Hermetic bats added under `tests/bats/` |
| **pester** | Covered by `tests/pester/` (bash invoked from PowerShell) |
| **defer** | Qualifies logically but needs lab/network/privilege |
| **no** | Thin wrapper or meta gate; shellcheck only |
| **n/a** | Example/template under `docs/private.example/` |

## Tracked `.sh` inventory (73 files)

Evaluated on branch `epic/boar-ga-hygiene-3` via `git ls-files '*.sh'`. **Logic** = branching/argument parsing/orchestration worth hardening review.

| Path | Logic | Bats | Notes |
| ---- | ----- | ---- | ----- |
| `audit.sh` | no | no | Tees into check-all wrapper |
| `deploy/compat-matrix-config-ref/run-across-corners.sh` | yes | defer | Docker/matrix orchestration |
| `deploy/compat-matrix-config-ref/run-config-matrix.sh` | yes | defer | Config matrix + containers |
| `deploy/lab-smoke-stack/init/mssql/apply_lab_smoke.sh` | yes | defer | DB init against lab stack |
| `deploy/lab-smoke-stack/init/oracle/apply_lab_smoke_pdb.sh` | yes | defer | Oracle lab init |
| `deploy/lab-smoke-stack/init/redis/01_lab_smoke_seed.sh` | yes | defer | Redis seed for lab smoke |
| `docs/private.example/homelab/lab-op-vc-container-rsync.example.sh` | n/a | n/a | Operator example only |
| `docs/private.example/scripts/session-b-git-filter-repo.example.sh` | n/a | n/a | Destructive example only |
| `prep_audit.sh` | no | no | Invokes audit/check-all path |
| `scripts/ansible-syntax-check.sh` | yes | defer | Ansible + subprocess |
| `scripts/check-all-security-scans.sh` | yes | defer | Security scan orchestration |
| `scripts/check-all.sh` | yes | no | Meta publish gate (self-test avoided) |
| `scripts/check-cross-platform-gaps.sh` | yes | **yes** | Arg mirror of `.ps1` → Python |
| `scripts/check-hubs.sh` | no | no | Thin doc-hub check wrapper |
| `scripts/check-sbom.sh` | yes | defer | External SBOM tooling |
| `scripts/ci-install-gitleaks-pinned.sh` | yes | defer | CI download/install |
| `scripts/ci_dependabot_requirements_sync.sh` | no | no | Single Python entry |
| `scripts/databoar-env-load.sh` | yes | defer | Env file sourcing for lab |
| `scripts/db-tool-bootstrap.sh` | yes | defer | DB client bootstrap |
| `scripts/demo.sh` | yes | **yes** | `--help` / case flags (headless path needs `uv`) |
| `scripts/docker/apply_wheelhouse_v1.sh` | yes | defer | Docker build context |
| `scripts/docker/build-nogil-local.sh` | yes | defer | Local Docker build |
| `scripts/docker/collect-runtime-rootfs.sh` | yes | defer | Docker inspection |
| `scripts/docker/compare-gil-nogil-demo-timing.sh` | yes | defer | Docker timing compare |
| `scripts/docker/docker-image-smoke.sh` | yes | defer | Image smoke |
| `scripts/dump-context.sh` | no | no | Context dump helper |
| `scripts/emit-provenance.sh` | yes | defer | Provenance emit tooling |
| `scripts/export_public_gemini_bundle.sh` | yes | defer | Bundle export orchestration |
| `scripts/gate-trailer-attest.sh` | yes | defer | Signing / git attest |
| `scripts/generate-sbom.sh` | yes | defer | SBOM generation |
| `scripts/grype-image-gate.sh` | yes | defer | Grype image scan |
| `scripts/homelab-host-report.sh` | yes | defer | SSH/collect lab report |
| `scripts/lab-allow-data-boar-inbound.sh` | yes | defer | Firewall/lab network |
| `scripts/lab-completao-container-smoke-original.sh` | yes | defer | Container completão smoke |
| `scripts/lab-completao-container-smoke.sh` | yes | defer | Container completão smoke |
| `scripts/lab-completao-container-smoke2.sh` | yes | defer | Container completão smoke |
| `scripts/lab-completao-host-smoke.sh` | yes | defer | Host completão smoke |
| `scripts/lab-node-01-ansible-labop-podman-apply.sh` | yes | defer | Ansible + podman |
| `scripts/lab-node-01-ansible-preflight.sh` | yes | defer | Ansible preflight |
| `scripts/labop-dep-doctor.sh` | yes | defer | Multi-branch lab doctor |
| `scripts/labop-firewall-lib.sh` | yes | defer | Firewall backends + sudo |
| `scripts/labop-fw-guard-ensure.sh` | yes | defer | Privileged ensure |
| `scripts/labop-gate-readiness.sh` | yes | defer | Gate readiness + SSH |
| `scripts/labop-nfs-server-ensure.sh` | yes | defer | NFS ensure + sudo |
| `scripts/labop-rfc1918-cidr-lib.sh` | yes | **yes** | Sourced pure CIDR matcher |
| `scripts/labop-share-client-install.sh` | yes | defer | Client install orchestration |
| `scripts/labop-smb-server-ensure.sh` | yes | defer | SMB ensure + sudo |
| `scripts/labop-sudoers-load-order-lib.sh` | yes | **pester** | `tests/pester/SudoersLoadOrder.Tests.ps1` |
| `scripts/lint-only.sh` | no | no | pre-commit wrapper |
| `scripts/native-nfpm-populate-staging.sh` | yes | defer | nfpm staging |
| `scripts/pre-commit-and-tests.sh` | no | no | check-all inner wrapper |
| `scripts/primary-linux-agent-sessions.sh` | yes | defer | Session orchestration |
| `scripts/private-git-sync.sh` | yes | defer | Nested git + remotes |
| `scripts/pypi-publish.sh` | yes | defer | Publish orchestration |
| `scripts/quick-test.sh` | yes | no | pytest wrapper |
| `scripts/run-gitleaks-strict.sh` | no | no | gitleaks wrapper |
| `scripts/run-homelab-host-report.sh` | yes | defer | SSH report wrapper |
| `scripts/run-osv-scanner.sh` | no | no | osv-scanner wrapper |
| `scripts/snmp-lab-ifwalk.sh` | yes | defer | SNMP lab walk |
| `scripts/*-bitwarden-cli-bootstrap.sh` | yes | defer | Host bootstrap (Bitwarden CLI) |
| `scripts/*-docker-remove-live-restore-and-restart.sh` | yes | defer | Docker daemon change |
| `scripts/*-fix-docker-list.sh` | yes | defer | Docker list fix |
| `scripts/*-install-veracrypt-console-debian13.sh` | yes | defer | VeraCrypt install |
| `scripts/*-session-warm.sh` | yes | defer | Session warm orchestration |
| `scripts/*-veracrypt-mount-private-repo.sh` | yes | defer | VeraCrypt mount |
| `scripts/tool-pins.sh` | no | no | Pin echo helper |
| `scripts/void-xbps-podman-validate.sh` | yes | defer | Void/podman validate |
| `scripts/wheelhouse/build_glibc_incontainer.sh` | yes | defer | In-container wheel build |
| `scripts/wheelhouse/build_musl_incontainer.sh` | yes | defer | In-container wheel build |
| `scripts/wheelhouse/run_cell.sh` | yes | defer | Wheelhouse cell runner |
| `scripts/wheelhouse/verify_connector_c_checksum.sh` | yes | defer | Checksum verify |
| `scripts/wheelhouse/verify_release_sha256sums.sh` | no | no | sha256sum verify |
| `scripts/workflow-security-lint.sh` | yes | defer | zizmor workflow lint |

**Bats slice (this phase):** 3 files, 11 tests — `check_cross_platform_gaps.bats`, `demo_help.bats`, `labop_rfc1918_cidr.bats`.

## Phases

| Step | Status |
| ---- | ------ |
| shellcheck `--severity=error` on all tracked `*.sh` in `check-all.sh` / `.ps1` | ✅ Done (on `main`) |
| Per-file inventory + plan doc (this file) | ✅ Done |
| Hermetic `tests/bats/` + optional `check-all` step | ✅ Done |
| Expand bats for `labop-sudoers-load-order-lib` on Linux (duplicate pester) | ⬜ Pending |
| Additional defer-row coverage via fixture labs | ⬜ Pending |

## Out of scope

- CI workflow job for bats (operator policy: local optional only).
- Replacing Maestro/lab completão bash with bats integration tests.
Loading
Loading