Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# =============================================================================
# Portabase Agent — local dev environment
# Copy to `.env` and adjust. `.env` is git-ignored.
# cp .env.example .env
#
# This file is consumed by:
# - docker compose (variable interpolation, e.g. EDGE_KEY)
# - the justfile (`set dotenv-load := true` — used by the `seed-*` recipes)
# =============================================================================

# ── Agent → Portabase Server backend ────────────────────────────────────────
# Base64 of {"serverUrl","agentId","masterKeyB64"}. Get this from YOUR deployed
# Portabase Server (Server UI → Agents → your agent → reveal EDGE_KEY).
# Only needed to run the AGENT (`just up`). Tests do NOT need this.
# Leave unset to fall back to the sample key baked into docker-compose.yml
# (targets http://localhost:8887).
EDGE_KEY=

# ── Seed targets (host-side; match docker-compose.databases.yml) ─────────────
# Used only by `just seed-*`. These are host-reachable values.

# Postgres (service db-postgres)
PG_CONTAINER=db-postgres
PG_USER=devuser
PG_DB=devdb
PG_PASSWORD=changeme

# MySQL (service db-mysql, published on host port 3312)
MYSQL_PORT=3312
MYSQL_USER=mysqldb
MYSQL_PASSWORD=changeme
MYSQL_DB=mysqldb

# MSSQL (service db-mssql)
MSSQL_SA_PASSWORD=Portabase!Strong1

# ── Optional: Podman rootless socket ─────────────────────────────────────────
# If you use rootless Podman, the app-dev-entrypoint.sh sets DOCKER_SOCK for you.
# Uncomment to override the socket the containers bind-mount as /var/run/docker.sock.
# DOCKER_SOCK=/run/user/1000/podman/podman.sock
11 changes: 8 additions & 3 deletions .github/CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,16 @@ Please take a moment to review this guide. It will help you understand how to co

2. **Clone the repository**
```bash
git clone https://github.com/Portabase/agent-rust.git
git clone https://github.com/Portabase/agent.git
Comment thread
Asuniia marked this conversation as resolved.
```

3. **Set up the development environment**
Follow the steps in the `README.md` to install dependencies and configure the project.
3. **Set up the development environment**<br/>
You need to have Docker installed in order to run the tests.
All builds complete within Docker.<br/>
At runtime, the agent built within the container will decode the EDGE_KEY for the Portabase server to poll and send information to; you will want to create an .env file off the included example if you are not using the full E2E tests.<br/><br/>
Run the agent against seeded DBs:<br/>
just up # creates portabase_network, starts DBs + the agent (cargo watch -x run)
just seed-all # load sample data into each DB<br/><br/>

4. **Create a branch**
Use the feature branch to work on changes.
Expand Down
2 changes: 1 addition & 1 deletion docker-compose.test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ services:
- .:/app
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
- /var/run/docker.sock:/var/run/docker.sock
- ${DOCKER_SOCK:-/var/run/docker.sock}:/var/run/docker.sock
environment:
APP_ENV: test
LOG: debug
Expand Down
17 changes: 15 additions & 2 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,28 @@ services:
- cargo-git:/usr/local/cargo/git
- ./databases.json:/config/config.json
#- ./databases.toml:/config/config.toml
- /var/run/docker.sock:/var/run/docker.sock
- ${DOCKER_SOCK:-/var/run/docker.sock}:/var/run/docker.sock
# - cargo-target:/app/target
- databases_sqlite-data:/sqlite-data/workspace/data
- ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
environment:
APP_ENV: development
LOG: debug
TZ: "Europe/Paris"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZWZhYTM0YTQtZDY1NC00OGQ3LTgwNDYtNjRkMWExYTA1M2FlIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
# Reach the container-local Redis by IP; "localhost" is remapped to the
# host by the extra_hosts entry below, which would break the connection.
CELERY_BROKER_URL: "redis://127.0.0.1:65515/"

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why? @RambokDev

# Image used for ephemeral docker-volume backup helpers. The agent can
# normally auto-detect its own image, but that self-detection relies on
# Docker-style /proc markers that rootless Podman doesn't expose, so we set
# it explicitly. Any tiny image works (it's only mounted, never run) and it
# must already be present locally: `docker pull busybox`.
PORTABASE_HELPER_IMAGE: "${PORTABASE_HELPER_IMAGE:-docker.io/library/busybox:latest}"
# Points the agent at your Portabase Server. Set EDGE_KEY in .env to the
# key from your deployed backend; the fallback below targets a local
# server at http://localhost:8887 (the agent just logs connection errors
# if nothing is listening there).
EDGE_KEY: "${EDGE_KEY:-eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZWZhYTM0YTQtZDY1NC00OGQ3LTgwNDYtNjRkMWExYTA1M2FlIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==}"
#CHUNK_SIZE_MB: "1"
#POOLING: 1
#DATABASES_CONFIG_FILE: "config.toml"
Expand Down
155 changes: 131 additions & 24 deletions docker/entrypoints/app-dev-entrypoint.sh
Original file line number Diff line number Diff line change
@@ -1,44 +1,151 @@
#!/bin/bash
set -euo pipefail

check_docker() {
if ! docker info > /dev/null 2>&1; then
echo "Docker is not running. Attempting to start Docker..."
if [[ "$OSTYPE" == "darwin"* ]]; then
open -a Docker
echo "Waiting for Docker to start..."
until docker info > /dev/null 2>&1; do
sleep 2
done
elif command -v systemctl >/dev/null 2>&1; then
sudo systemctl start docker
else
echo "Cannot start Docker automatically. Please start Docker manually."
exit 1
# Container engine + compose command, resolved by detect_engine/detect_compose.
ENGINE=""
COMPOSE=""

# True if the given engine binary exists and its daemon/service answers.
# Bounded with `timeout` (when available) so a wedged/slow daemon fails the
# check quickly instead of hanging the whole script on `<engine> info`.
engine_ready() {
command -v "$1" >/dev/null 2>&1 || return 1
if command -v timeout >/dev/null 2>&1; then
timeout 15 "$1" info >/dev/null 2>&1
else
"$1" info >/dev/null 2>&1
fi
}

# Best-effort start of the Docker daemon (macOS app or systemd service).
try_start_docker() {
if [[ "$OSTYPE" == "darwin"* ]]; then
open -a Docker >/dev/null 2>&1 || return 1
echo "Waiting for Docker to start..."
local count=0
until docker info >/dev/null 2>&1; do
sleep 2
count=$((count + 1))
[ "$count" -ge 30 ] && return 1
done
return 0
elif command -v systemctl >/dev/null 2>&1; then
sudo systemctl start docker >/dev/null 2>&1 || return 1
docker info >/dev/null 2>&1
else
return 1
fi
}

# Pick a container engine, preferring one that is already running.
detect_engine() {
if engine_ready docker; then
ENGINE="docker"
elif engine_ready podman; then
ENGINE="podman"
elif command -v docker >/dev/null 2>&1; then
echo "Docker is installed but not running. Attempting to start it..."
if try_start_docker; then
ENGINE="docker"
elif command -v podman >/dev/null 2>&1; then
echo "Could not start Docker; falling back to Podman."
ENGINE="podman"
fi
elif command -v podman >/dev/null 2>&1; then
# Podman CLI works without a running daemon for most commands.
ENGINE="podman"
fi

if [ -z "$ENGINE" ]; then
echo "No working container engine found (need docker or podman)." >&2
exit 1
fi
echo "Using container engine: ${ENGINE}"
}

# Resolve a compose implementation compatible with the chosen engine.
detect_compose() {
if $ENGINE compose version >/dev/null 2>&1; then
COMPOSE="$ENGINE compose"
elif command -v docker-compose >/dev/null 2>&1; then
COMPOSE="docker-compose"
elif command -v podman-compose >/dev/null 2>&1; then
COMPOSE="podman-compose"
else
echo "Docker is running."
echo "No compose command found (${ENGINE} compose / docker-compose / podman-compose)." >&2
exit 1
fi
echo "Using compose command: ${COMPOSE}"
}

# Ensure the external network referenced by the compose files exists.
# `inspect` is a more reliable existence test than parsing `ls`, and the create
# is made idempotent so a concurrent/pre-existing network doesn't abort the run
# (podman's `network create` errors on an existing name; docker's does not).
check_network() {
local network_name="portabase_network"
if ! docker network ls --format '{{.Name}}' | grep -q "^${network_name}$"; then
echo "Docker network '${network_name}' not found. Creating..."
docker network create "${network_name}"
else
echo "Docker network '${network_name}' already exists."
if $ENGINE network inspect "${network_name}" >/dev/null 2>&1; then
echo "Network '${network_name}' already exists."
return 0
fi
echo "Network '${network_name}' not found. Creating..."
$ENGINE network create "${network_name}" >/dev/null 2>&1 \
|| echo "Network '${network_name}' already exists (created concurrently)."
}

# Ensure the external volume declared in docker-compose.yml exists.
check_volume() {
local volume_name="databases_sqlite-data"
if $ENGINE volume inspect "${volume_name}" >/dev/null 2>&1; then
echo "Volume '${volume_name}' already exists."
return 0
fi
echo "Volume '${volume_name}' not found. Creating..."
$ENGINE volume create "${volume_name}" >/dev/null 2>&1 \
|| echo "Volume '${volume_name}' already exists (created concurrently)."
}

# Resolve the Podman socket and expose it to compose via DOCKER_SOCK.
# The compose files bind-mount ${DOCKER_SOCK:-/var/run/docker.sock} into the
# agent so it can drive containers (testcontainers). Under rootless Podman the
# real socket lives under $XDG_RUNTIME_DIR, not /var/run/docker.sock, so we point
# the mount at it here. Without a socket, backup/restore can't connect.
check_podman_socket() {
[ "$ENGINE" = "podman" ] || return 0

local rootless_sock="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock"
local sock=""
if [ -S "$rootless_sock" ]; then
sock="$rootless_sock"
elif [ -S /var/run/docker.sock ]; then
sock="/var/run/docker.sock"
fi

if [ -n "$sock" ]; then
export DOCKER_SOCK="$sock"
echo "Using Podman socket: ${sock}"
return 0
fi

echo "[WARN] Running in Podman mode but no Podman socket was found." >&2
echo "[WARN] Checked: ${rootless_sock} and /var/run/docker.sock" >&2
echo "[WARN] The agent mounts this socket to manage containers; without it," >&2
echo "[WARN] backup/restore operations will fail to connect." >&2
echo "[WARN] Enable it with: systemctl --user enable --now podman.socket" >&2
echo "[WARN] (then re-run 'just up'). Continuing anyway..." >&2
}

check_docker
detect_engine
detect_compose
check_network
check_volume
check_podman_socket

echo "Stopping old database containers..."
docker compose -f ./docker-compose.databases.yml down
$COMPOSE -f ./docker-compose.databases.yml down

echo "Starting database containers..."
docker compose -f ./docker-compose.databases.yml up -d
$COMPOSE -f ./docker-compose.databases.yml up -d

echo "Starting main services..."
docker compose -f ./docker-compose.yml up
$COMPOSE -f ./docker-compose.yml up
5 changes: 4 additions & 1 deletion entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,10 @@ redis-server --port $REDIS_PORT --daemonize yes
echo "[entrypoint] Waiting for Redis to be ready..."
MAX_RETRIES=20
COUNT=0
until redis-cli -h localhost -p "$REDIS_PORT" ping >/dev/null 2>&1 ; do
# Use 127.0.0.1 rather than "localhost": some compose setups remap "localhost"
# (e.g. extra_hosts localhost:host-gateway) so it no longer points at the
# container's own loopback where this Redis is bound.
until redis-cli -h 127.0.0.1 -p "$REDIS_PORT" ping >/dev/null 2>&1 ; do
COUNT=$((COUNT+1))
if [ $COUNT -ge $MAX_RETRIES ]; then
echo "[ERROR] Redis did not start after $MAX_RETRIES attempts"
Expand Down
17 changes: 15 additions & 2 deletions src/domain/docker_volume/docker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,28 @@ pub fn client() -> Result<Docker> {
}

pub fn parse_container_id(mountinfo: &str, cgroup: &str) -> Option<String> {
// Markers that precede the 64-hex container id, by runtime:
// overlay-containers/<id> Podman rootless (mountinfo: .../overlay-containers/<id>/userdata)
// libpod-<id> Podman (cgroup: .../libpod-<id>.scope)
// /containers/<id> Docker (mountinfo: /var/lib/docker/containers/<id>/...)
// /docker/<id> Docker (cgroup v1: /docker/<id>)
// overlay-containers/ is tried before /containers/ so the Podman path (which
// also contains a "/containers/" substring) resolves to the real id.
const MARKERS: [&str; 4] = ["overlay-containers/", "libpod-", "/containers/", "/docker/"];
for src in [mountinfo, cgroup] {
for line in src.lines() {
for marker in ["/containers/", "/docker/"] {
if let Some(idx) = line.find(marker) {
for marker in MARKERS {
let mut search_from = 0;
while let Some(rel) = line[search_from..].find(marker) {
let idx = search_from + rel;
let rest = &line[idx + marker.len()..];
let id: String = rest.chars().take_while(|c| c.is_ascii_hexdigit()).collect();
if id.len() >= 64 {
return Some(id[..64].to_string());
}
// This occurrence didn't yield an id (e.g. a "/containers/"
// substring that isn't the id); keep scanning the same line.
search_from = idx + marker.len();
}
}
}
Expand Down
23 changes: 23 additions & 0 deletions src/tests/domain/docker_volume.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,29 @@ fn parse_container_id_none_on_cgroup_v2() {
assert_eq!(parse_container_id("", "0::/\n"), None);
}

#[test]
fn parse_container_id_from_podman_rootless_mountinfo() {
// Rootless Podman: cgroup is "0::/" (no id); the id lives in mountinfo under
// overlay-containers/. The line also contains a "/containers/" substring
// (…/share/containers/storage/…) that must not be mistaken for the id.
let id = "c".repeat(64);
let mountinfo = format!(
"1234 1000 0:60 / /vol rw,relatime shared:1 - overlay overlay \
rw,lowerdir=/home/u/.local/share/containers/storage/overlay/L1/diff,\
upperdir=/home/u/.local/share/containers/storage/overlay-containers/{id}/userdata/upper"
);
assert_eq!(parse_container_id(&mountinfo, "0::/\n"), Some(id));
}

#[test]
fn parse_container_id_from_podman_libpod_cgroup() {
let id = "d".repeat(64);
let cgroup = format!(
"0::/user.slice/user-1000.slice/user@1000.service/user.slice/libpod-{id}.scope/container\n"
);
assert_eq!(parse_container_id("", &cgroup), Some(id));
}

#[tokio::test]
async fn docker_volume_ping_true_for_existing_volume() {
use crate::domain::docker_volume::docker::client;
Expand Down