diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..17f2a78 --- /dev/null +++ b/.env.example @@ -0,0 +1,40 @@ +# ============================================================================= +# Portabase Agent — local dev environment +# Copy to `.env` and adjust. `.env` is git-ignored. +# cp .env.example .env +# +# This file is consumed by: +# - docker compose (variable interpolation, e.g. EDGE_KEY) +# - the justfile (`set dotenv-load := true` — used by the `seed-*` recipes) +# ============================================================================= + +# ── Agent → Portabase Server backend ──────────────────────────────────────── +# Base64 of {"serverUrl","agentId","masterKeyB64"}. Get this from YOUR deployed +# Portabase Server (Server UI → Agents → your agent → reveal EDGE_KEY). +# Only needed to run the AGENT (`just up`). Tests do NOT need this. +# Leave unset to fall back to the sample key baked into docker-compose.yml +# (targets http://localhost:8887). +EDGE_KEY= + +# ── Seed targets (host-side; match docker-compose.databases.yml) ───────────── +# Used only by `just seed-*`. These are host-reachable values. + +# Postgres (service db-postgres) +PG_CONTAINER=db-postgres +PG_USER=devuser +PG_DB=devdb +PG_PASSWORD=changeme + +# MySQL (service db-mysql, published on host port 3312) +MYSQL_PORT=3312 +MYSQL_USER=mysqldb +MYSQL_PASSWORD=changeme +MYSQL_DB=mysqldb + +# MSSQL (service db-mssql) +MSSQL_SA_PASSWORD=Portabase!Strong1 + +# ── Optional: Podman rootless socket ───────────────────────────────────────── +# If you use rootless Podman, the app-dev-entrypoint.sh sets DOCKER_SOCK for you. +# Uncomment to override the socket the containers bind-mount as /var/run/docker.sock. +# DOCKER_SOCK=/run/user/1000/podman/podman.sock diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 6910a4e..8468432 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -26,11 +26,16 @@ Please take a moment to review this guide. It will help you understand how to co 2. **Clone the repository** ```bash - git clone https://github.com/Portabase/agent-rust.git + git clone https://github.com/Portabase/agent.git ``` -3. **Set up the development environment** - Follow the steps in the `README.md` to install dependencies and configure the project. +3. **Set up the development environment**
+ You need to have Docker installed in order to run the tests. + All builds complete within Docker.
+ At runtime, the agent built within the container will decode the EDGE_KEY for the Portabase server to poll and send information to; you will want to create an .env file off the included example if you are not using the full E2E tests.

+ Run the agent against seeded DBs:
+ just up # creates portabase_network, starts DBs + the agent (cargo watch -x run) + just seed-all # load sample data into each DB

4. **Create a branch** Use the feature branch to work on changes. diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 4022ede..db5a448 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -10,7 +10,7 @@ services: - .:/app - cargo-registry:/usr/local/cargo/registry - cargo-git:/usr/local/cargo/git - - /var/run/docker.sock:/var/run/docker.sock + - ${DOCKER_SOCK:-/var/run/docker.sock}:/var/run/docker.sock environment: APP_ENV: test LOG: debug diff --git a/docker-compose.yml b/docker-compose.yml index ff72c6c..9cc7827 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - cargo-git:/usr/local/cargo/git - ./databases.json:/config/config.json #- ./databases.toml:/config/config.toml - - /var/run/docker.sock:/var/run/docker.sock + - ${DOCKER_SOCK:-/var/run/docker.sock}:/var/run/docker.sock # - cargo-target:/app/target - databases_sqlite-data:/sqlite-data/workspace/data - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data @@ -19,7 +19,20 @@ services: APP_ENV: development LOG: debug TZ: "Europe/Paris" - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZWZhYTM0YTQtZDY1NC00OGQ3LTgwNDYtNjRkMWExYTA1M2FlIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" + # Reach the container-local Redis by IP; "localhost" is remapped to the + # host by the extra_hosts entry below, which would break the connection. + CELERY_BROKER_URL: "redis://127.0.0.1:65515/" + # Image used for ephemeral docker-volume backup helpers. The agent can + # normally auto-detect its own image, but that self-detection relies on + # Docker-style /proc markers that rootless Podman doesn't expose, so we set + # it explicitly. Any tiny image works (it's only mounted, never run) and it + # must already be present locally: `docker pull busybox`. + PORTABASE_HELPER_IMAGE: "${PORTABASE_HELPER_IMAGE:-docker.io/library/busybox:latest}" + # Points the agent at your Portabase Server. Set EDGE_KEY in .env to the + # key from your deployed backend; the fallback below targets a local + # server at http://localhost:8887 (the agent just logs connection errors + # if nothing is listening there). + EDGE_KEY: "${EDGE_KEY:-eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZWZhYTM0YTQtZDY1NC00OGQ3LTgwNDYtNjRkMWExYTA1M2FlIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==}" #CHUNK_SIZE_MB: "1" #POOLING: 1 #DATABASES_CONFIG_FILE: "config.toml" diff --git a/docker/entrypoints/app-dev-entrypoint.sh b/docker/entrypoints/app-dev-entrypoint.sh index 57ea5dd..063b575 100755 --- a/docker/entrypoints/app-dev-entrypoint.sh +++ b/docker/entrypoints/app-dev-entrypoint.sh @@ -1,44 +1,151 @@ #!/bin/bash set -euo pipefail -check_docker() { - if ! docker info > /dev/null 2>&1; then - echo "Docker is not running. Attempting to start Docker..." - if [[ "$OSTYPE" == "darwin"* ]]; then - open -a Docker - echo "Waiting for Docker to start..." - until docker info > /dev/null 2>&1; do - sleep 2 - done - elif command -v systemctl >/dev/null 2>&1; then - sudo systemctl start docker - else - echo "Cannot start Docker automatically. Please start Docker manually." - exit 1 +# Container engine + compose command, resolved by detect_engine/detect_compose. +ENGINE="" +COMPOSE="" + +# True if the given engine binary exists and its daemon/service answers. +# Bounded with `timeout` (when available) so a wedged/slow daemon fails the +# check quickly instead of hanging the whole script on ` info`. +engine_ready() { + command -v "$1" >/dev/null 2>&1 || return 1 + if command -v timeout >/dev/null 2>&1; then + timeout 15 "$1" info >/dev/null 2>&1 + else + "$1" info >/dev/null 2>&1 + fi +} + +# Best-effort start of the Docker daemon (macOS app or systemd service). +try_start_docker() { + if [[ "$OSTYPE" == "darwin"* ]]; then + open -a Docker >/dev/null 2>&1 || return 1 + echo "Waiting for Docker to start..." + local count=0 + until docker info >/dev/null 2>&1; do + sleep 2 + count=$((count + 1)) + [ "$count" -ge 30 ] && return 1 + done + return 0 + elif command -v systemctl >/dev/null 2>&1; then + sudo systemctl start docker >/dev/null 2>&1 || return 1 + docker info >/dev/null 2>&1 + else + return 1 + fi +} + +# Pick a container engine, preferring one that is already running. +detect_engine() { + if engine_ready docker; then + ENGINE="docker" + elif engine_ready podman; then + ENGINE="podman" + elif command -v docker >/dev/null 2>&1; then + echo "Docker is installed but not running. Attempting to start it..." + if try_start_docker; then + ENGINE="docker" + elif command -v podman >/dev/null 2>&1; then + echo "Could not start Docker; falling back to Podman." + ENGINE="podman" fi + elif command -v podman >/dev/null 2>&1; then + # Podman CLI works without a running daemon for most commands. + ENGINE="podman" + fi + + if [ -z "$ENGINE" ]; then + echo "No working container engine found (need docker or podman)." >&2 + exit 1 + fi + echo "Using container engine: ${ENGINE}" +} + +# Resolve a compose implementation compatible with the chosen engine. +detect_compose() { + if $ENGINE compose version >/dev/null 2>&1; then + COMPOSE="$ENGINE compose" + elif command -v docker-compose >/dev/null 2>&1; then + COMPOSE="docker-compose" + elif command -v podman-compose >/dev/null 2>&1; then + COMPOSE="podman-compose" else - echo "Docker is running." + echo "No compose command found (${ENGINE} compose / docker-compose / podman-compose)." >&2 + exit 1 fi + echo "Using compose command: ${COMPOSE}" } +# Ensure the external network referenced by the compose files exists. +# `inspect` is a more reliable existence test than parsing `ls`, and the create +# is made idempotent so a concurrent/pre-existing network doesn't abort the run +# (podman's `network create` errors on an existing name; docker's does not). check_network() { local network_name="portabase_network" - if ! docker network ls --format '{{.Name}}' | grep -q "^${network_name}$"; then - echo "Docker network '${network_name}' not found. Creating..." - docker network create "${network_name}" - else - echo "Docker network '${network_name}' already exists." + if $ENGINE network inspect "${network_name}" >/dev/null 2>&1; then + echo "Network '${network_name}' already exists." + return 0 + fi + echo "Network '${network_name}' not found. Creating..." + $ENGINE network create "${network_name}" >/dev/null 2>&1 \ + || echo "Network '${network_name}' already exists (created concurrently)." +} + +# Ensure the external volume declared in docker-compose.yml exists. +check_volume() { + local volume_name="databases_sqlite-data" + if $ENGINE volume inspect "${volume_name}" >/dev/null 2>&1; then + echo "Volume '${volume_name}' already exists." + return 0 + fi + echo "Volume '${volume_name}' not found. Creating..." + $ENGINE volume create "${volume_name}" >/dev/null 2>&1 \ + || echo "Volume '${volume_name}' already exists (created concurrently)." +} + +# Resolve the Podman socket and expose it to compose via DOCKER_SOCK. +# The compose files bind-mount ${DOCKER_SOCK:-/var/run/docker.sock} into the +# agent so it can drive containers (testcontainers). Under rootless Podman the +# real socket lives under $XDG_RUNTIME_DIR, not /var/run/docker.sock, so we point +# the mount at it here. Without a socket, backup/restore can't connect. +check_podman_socket() { + [ "$ENGINE" = "podman" ] || return 0 + + local rootless_sock="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock" + local sock="" + if [ -S "$rootless_sock" ]; then + sock="$rootless_sock" + elif [ -S /var/run/docker.sock ]; then + sock="/var/run/docker.sock" fi + + if [ -n "$sock" ]; then + export DOCKER_SOCK="$sock" + echo "Using Podman socket: ${sock}" + return 0 + fi + + echo "[WARN] Running in Podman mode but no Podman socket was found." >&2 + echo "[WARN] Checked: ${rootless_sock} and /var/run/docker.sock" >&2 + echo "[WARN] The agent mounts this socket to manage containers; without it," >&2 + echo "[WARN] backup/restore operations will fail to connect." >&2 + echo "[WARN] Enable it with: systemctl --user enable --now podman.socket" >&2 + echo "[WARN] (then re-run 'just up'). Continuing anyway..." >&2 } -check_docker +detect_engine +detect_compose check_network +check_volume +check_podman_socket echo "Stopping old database containers..." -docker compose -f ./docker-compose.databases.yml down +$COMPOSE -f ./docker-compose.databases.yml down echo "Starting database containers..." -docker compose -f ./docker-compose.databases.yml up -d +$COMPOSE -f ./docker-compose.databases.yml up -d echo "Starting main services..." -docker compose -f ./docker-compose.yml up \ No newline at end of file +$COMPOSE -f ./docker-compose.yml up diff --git a/entrypoint.sh b/entrypoint.sh index 697ccc4..8353450 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -42,7 +42,10 @@ redis-server --port $REDIS_PORT --daemonize yes echo "[entrypoint] Waiting for Redis to be ready..." MAX_RETRIES=20 COUNT=0 -until redis-cli -h localhost -p "$REDIS_PORT" ping >/dev/null 2>&1 ; do +# Use 127.0.0.1 rather than "localhost": some compose setups remap "localhost" +# (e.g. extra_hosts localhost:host-gateway) so it no longer points at the +# container's own loopback where this Redis is bound. +until redis-cli -h 127.0.0.1 -p "$REDIS_PORT" ping >/dev/null 2>&1 ; do COUNT=$((COUNT+1)) if [ $COUNT -ge $MAX_RETRIES ]; then echo "[ERROR] Redis did not start after $MAX_RETRIES attempts" diff --git a/src/domain/docker_volume/docker.rs b/src/domain/docker_volume/docker.rs index 8784230..bfbbd8a 100644 --- a/src/domain/docker_volume/docker.rs +++ b/src/domain/docker_volume/docker.rs @@ -19,15 +19,28 @@ pub fn client() -> Result { } pub fn parse_container_id(mountinfo: &str, cgroup: &str) -> Option { + // Markers that precede the 64-hex container id, by runtime: + // overlay-containers/ Podman rootless (mountinfo: .../overlay-containers//userdata) + // libpod- Podman (cgroup: .../libpod-.scope) + // /containers/ Docker (mountinfo: /var/lib/docker/containers//...) + // /docker/ Docker (cgroup v1: /docker/) + // overlay-containers/ is tried before /containers/ so the Podman path (which + // also contains a "/containers/" substring) resolves to the real id. + const MARKERS: [&str; 4] = ["overlay-containers/", "libpod-", "/containers/", "/docker/"]; for src in [mountinfo, cgroup] { for line in src.lines() { - for marker in ["/containers/", "/docker/"] { - if let Some(idx) = line.find(marker) { + for marker in MARKERS { + let mut search_from = 0; + while let Some(rel) = line[search_from..].find(marker) { + let idx = search_from + rel; let rest = &line[idx + marker.len()..]; let id: String = rest.chars().take_while(|c| c.is_ascii_hexdigit()).collect(); if id.len() >= 64 { return Some(id[..64].to_string()); } + // This occurrence didn't yield an id (e.g. a "/containers/" + // substring that isn't the id); keep scanning the same line. + search_from = idx + marker.len(); } } } diff --git a/src/tests/domain/docker_volume.rs b/src/tests/domain/docker_volume.rs index 26c955d..4a827dd 100644 --- a/src/tests/domain/docker_volume.rs +++ b/src/tests/domain/docker_volume.rs @@ -23,6 +23,29 @@ fn parse_container_id_none_on_cgroup_v2() { assert_eq!(parse_container_id("", "0::/\n"), None); } +#[test] +fn parse_container_id_from_podman_rootless_mountinfo() { + // Rootless Podman: cgroup is "0::/" (no id); the id lives in mountinfo under + // overlay-containers/. The line also contains a "/containers/" substring + // (…/share/containers/storage/…) that must not be mistaken for the id. + let id = "c".repeat(64); + let mountinfo = format!( + "1234 1000 0:60 / /vol rw,relatime shared:1 - overlay overlay \ + rw,lowerdir=/home/u/.local/share/containers/storage/overlay/L1/diff,\ + upperdir=/home/u/.local/share/containers/storage/overlay-containers/{id}/userdata/upper" + ); + assert_eq!(parse_container_id(&mountinfo, "0::/\n"), Some(id)); +} + +#[test] +fn parse_container_id_from_podman_libpod_cgroup() { + let id = "d".repeat(64); + let cgroup = format!( + "0::/user.slice/user-1000.slice/user@1000.service/user.slice/libpod-{id}.scope/container\n" + ); + assert_eq!(parse_container_id("", &cgroup), Some(id)); +} + #[tokio::test] async fn docker_volume_ping_true_for_existing_volume() { use crate::domain::docker_volume::docker::client;