diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index aa9011e..9a430b3 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -25,6 +25,9 @@ jobs: - name: Install grcov run: cargo install grcov + - name: Install test requirements + run: bash scripts/tests/requirements.sh + - name: Build run: cargo build --verbose diff --git a/Cargo.lock b/Cargo.lock index 04f8f24..4580da2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1450,7 +1450,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2089,7 +2089,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -2680,7 +2680,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3088,6 +3088,7 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "typenum", + "url", "uuid", "wiremock", ] @@ -3255,7 +3256,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.37", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -3293,7 +3294,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.60.2", ] @@ -3646,7 +3647,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3717,7 +3718,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4093,7 +4094,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4255,7 +4256,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5170,7 +5171,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index be852c9..20875b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,13 +49,15 @@ tokio-stream = "0.1.18" aes = "0.9.0-rc.4" typenum = "1.19.0" testcontainers = "0.27.1" -testcontainers-modules = { version = "0.15.0", features = ["postgres"] } +testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis"] } postgres = "0.19.12" +url = "2.5.8" [dev-dependencies] tokio = { version = "1", features = ["full"] } mockall = "0.13" testcontainers = "0.27.1" +testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis"] } wiremock = "0.6" diff --git a/databases.json b/databases.json index be93742..90b7d73 100644 --- a/databases.json +++ b/databases.json @@ -49,6 +49,22 @@ "type": "sqlite", "path": "/sqlite-data-2/workspace/data/app.db", "generated_id": "16678179-ff7e-4c97-8c83-0adeff214681" + }, + { + "name": "Test database 8 - Redis", + "type": "redis", + "port": 6379, + "host": "db-redis", + "generated_id": "16678166-ff7e-4c97-8c83-0adeff214681" + }, + { + "name": "Test database 9 - Redis Auth", + "type": "redis", + "password": "m6o2L0Ukw3Og5RJUKh", + "port": 6379, + "username": "default", + "host": "db-redis-auth", + "generated_id": "16678160-ff7e-4c97-8c83-0adeff214681" } ] } diff --git a/docker-compose.yml b/docker-compose.yml index e746f02..d6fbc5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,8 +12,8 @@ services: - cargo-registry:/usr/local/cargo/registry - cargo-git:/usr/local/cargo/git - cargo-target:/app/target -# - sqlite-data:/sqlite-data/workspace/data -# - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data + # - sqlite-data:/sqlite-data/workspace/data + # - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data environment: APP_ENV: development LOG: debug @@ -39,70 +39,93 @@ services: - POSTGRES_PASSWORD=changeme networks: - portabase -# -# db-mariadb: -# container_name: db-mariadb -# image: mariadb:latest -# ports: -# - "3311:3306" -# environment: -# - MYSQL_DATABASE=mariadb -# - MYSQL_USER=mariadb -# - MYSQL_PASSWORD=changeme -# - MYSQL_RANDOM_ROOT_PASSWORD=yes -# volumes: -# - mariadb-data:/var/lib/mysql -# networks: -# - portabase -# -# -# db-mongodb-auth: -# container_name: db-mongodb-auth -# image: mongo:latest -# ports: -# - "27082:27017" -# environment: -# MONGO_INITDB_ROOT_USERNAME: root -# MONGO_INITDB_ROOT_PASSWORD: rootpassword -# MONGO_INITDB_DATABASE: testdbauth -# command: mongod --auth -# networks: -# - portabase -# volumes: -# - mongodb-data-auth:/data/db -# healthcheck: -# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] -# interval: 5s -# timeout: 5s -# retries: 10 -# -# db-mongodb: -# container_name: db-mongodb -# image: mongo:latest -# ports: -# - "27083:27017" -# volumes: -# - mongodb-data:/data/db -# healthcheck: -# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] -# interval: 5s -# timeout: 5s -# retries: 10 -# environment: -# MONGO_INITDB_DATABASE: testdb -# networks: -# - portabase + # + # db-mariadb: + # container_name: db-mariadb + # image: mariadb:latest + # ports: + # - "3311:3306" + # environment: + # - MYSQL_DATABASE=mariadb + # - MYSQL_USER=mariadb + # - MYSQL_PASSWORD=changeme + # - MYSQL_RANDOM_ROOT_PASSWORD=yes + # volumes: + # - mariadb-data:/var/lib/mysql + # networks: + # - portabase + # + # + # db-mongodb-auth: + # container_name: db-mongodb-auth + # image: mongo:latest + # ports: + # - "27082:27017" + # environment: + # MONGO_INITDB_ROOT_USERNAME: root + # MONGO_INITDB_ROOT_PASSWORD: rootpassword + # MONGO_INITDB_DATABASE: testdbauth + # command: mongod --auth + # networks: + # - portabase + # volumes: + # - mongodb-data-auth:/data/db + # healthcheck: + # test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + # interval: 5s + # timeout: 5s + # retries: 10 + # + # db-mongodb: + # container_name: db-mongodb + # image: mongo:latest + # ports: + # - "27083:27017" + # volumes: + # - mongodb-data:/data/db + # healthcheck: + # test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] + # interval: 5s + # timeout: 5s + # retries: 10 + # environment: + # MONGO_INITDB_DATABASE: testdb + # networks: + # - portabase -# sqlite: -# container_name: db-sqlite -# image: keinos/sqlite3 -# volumes: -# - sqlite-data:/workspace/data -# working_dir: /workspace -# command: tail -f /dev/null -# stdin_open: true -# tty: true + # sqlite: + # container_name: db-sqlite + # image: keinos/sqlite3 + # volumes: + # - sqlite-data:/workspace/data + # working_dir: /workspace + # command: tail -f /dev/null + # stdin_open: true + # tty: true + db-redis: + image: redis:latest + container_name: db-redis + ports: + - "6379:6379" + volumes: + - redis-data:/data + command: [ "redis-server", "--appendonly", "yes" ] + networks: + - portabase + + db-redis-auth: + image: redis:latest + container_name: db-redis-auth + ports: + - "6380:6379" + volumes: + - redis-data-auth:/data + environment: + - REDIS_PASSWORD=m6o2L0Ukw3Og5RJUKh + command: [ "redis-server", "--requirepass", "m6o2L0Ukw3Og5RJUKh", "--appendonly", "yes" ] + networks: + - portabase volumes: cargo-registry: @@ -110,10 +133,12 @@ volumes: cargo-target: postgres-data: -# mariadb-data: -# mongodb-data: -# mongodb-data-auth: -# sqlite-data: + # mariadb-data: + # mongodb-data: + # mongodb-data-auth: + # sqlite-data: + redis-data: + redis-data-auth: networks: portabase: diff --git a/docker/Dockerfile b/docker/Dockerfile index 5ae11de..66e38d9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,6 +16,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ curl \ mariadb-client \ sqlite3 \ + redis-tools \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -94,6 +95,7 @@ RUN apt-get update && apt-get install -y \ zlib1g \ mariadb-client \ sqlite3 \ + redis-tools \ && rm -rf /var/lib/apt/lists/* diff --git a/scripts/tests/requirements.sh b/scripts/tests/requirements.sh new file mode 100755 index 0000000..2239327 --- /dev/null +++ b/scripts/tests/requirements.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -e + +POSTGRES_BASE="/usr/local/postgresql" +echo "Detecting OS and architecture..." +OS_TYPE="$(uname -s)" +ARCH="$(uname -m)" + +install_pg_binaries() { + echo "Installing PostgreSQL binaries for versions 12-18..." + + for v in 12 13 14 15 16 17 18; do + TARGET_DIR="$POSTGRES_BASE/$v/bin" + sudo mkdir -p "$TARGET_DIR" + + if [[ "$OS_TYPE" == "Linux" ]]; then + if [[ "$ARCH" == "x86_64" ]]; then + SRC_DIR="./assets/tools/amd64/postgresql/postgresql-$v/bin" + elif [[ "$ARCH" == "aarch64" ]]; then + SRC_DIR="./assets/tools/arm64/postgresql/postgresql-$v/bin" + else + echo "Unsupported architecture: $ARCH" + continue + fi + + if [[ -d "$SRC_DIR" ]]; then + echo "Copying PostgreSQL $v binaries from $SRC_DIR to $TARGET_DIR" + sudo cp -r "$SRC_DIR"/* "$TARGET_DIR/" + else + echo "Binaries for PostgreSQL $v not found for Linux, skipping..." + continue + fi + + elif [[ "$OS_TYPE" == "Darwin" ]]; then + PG_SRC="$(brew --prefix postgresql@$v)/bin" 2>/dev/null || true + + if [[ ! -d "$PG_SRC" ]]; then + echo "PostgreSQL $v not installed via Homebrew. Trying to install..." + if ! brew install postgresql@$v; then + echo "PostgreSQL $v not available, skipping..." + continue + fi + PG_SRC="$(brew --prefix postgresql@$v)/bin" + fi + + echo "Copying PostgreSQL $v binaries from $PG_SRC to $TARGET_DIR" + sudo cp -r "$PG_SRC"/* "$TARGET_DIR/" + fi + + sudo chown -R "$(whoami)" "$TARGET_DIR" + chmod +x "$TARGET_DIR"/* + done + + echo "PostgreSQL binaries installed under $POSTGRES_BASE" +} + +if [[ "$OS_TYPE" == "Linux" ]]; then + if command -v apt >/dev/null 2>&1; then + echo "Linux detected with apt. Installing prerequisites..." + sudo apt update + sudo apt install -y wget gnupg lsb-release redis-tools + install_pg_binaries + else + echo "Unsupported Linux distribution. Only apt-based distros are supported." + exit 1 + fi + +elif [[ "$OS_TYPE" == "Darwin" ]]; then + if command -v brew >/dev/null 2>&1; then + echo "macOS detected. Installing prerequisites..." + brew install redis + + sudo mkdir -p "$POSTGRES_BASE" + sudo chown -R "$(whoami)" "$POSTGRES_BASE" + + install_pg_binaries + else + echo "Homebrew not found. Please install Homebrew first: https://brew.sh/" + exit 1 + fi + +else + echo "Unsupported OS: $OS_TYPE" + exit 1 +fi + +echo "Tools installation completed successfully." \ No newline at end of file diff --git a/src/core/agent.rs b/src/core/agent.rs index 471cfd7..23e56f1 100644 --- a/src/core/agent.rs +++ b/src/core/agent.rs @@ -43,9 +43,10 @@ impl Agent { let ping_result = self.status_service.ping(&config.databases).await?; for db in ping_result.databases.iter() { + let database = config.databases.iter().find(|cfg_db|cfg_db.generated_id == db.generated_id).unwrap(); info!( - "Generated Id: {} | backup action: {} | restore action: {}", - db.generated_id, db.data.backup.action, db.data.restore.action + "Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}", + db.generated_id, db.data.backup.action, db.data.restore.action, database.name, ); let _ = self.cron_service.sync(db).await; diff --git a/src/domain/factory.rs b/src/domain/factory.rs index 46ff74e..dc72d16 100644 --- a/src/domain/factory.rs +++ b/src/domain/factory.rs @@ -2,18 +2,19 @@ use crate::domain::mongodb::database::MongoDatabase; use crate::domain::mysql::database::MySQLDatabase; use crate::domain::postgres::database::PostgresDatabase; use crate::domain::postgres::{detect_format_from_file, detect_format_from_size}; +use crate::domain::redis::database::RedisDatabase; +use crate::domain::sqlite::database::SqliteDatabase; use crate::services::config::{DatabaseConfig, DbType}; use anyhow::Result; use std::path::{Path, PathBuf}; use std::sync::Arc; -use crate::domain::sqlite::database::SqliteDatabase; #[async_trait::async_trait] pub trait Database: Send + Sync { fn file_extension(&self) -> &'static str; async fn ping(&self) -> Result; - async fn backup(&self, backup_dir: &Path) -> Result; - async fn restore(&self, restore_file: &Path) -> Result<()>; + async fn backup(&self, backup_dir: &Path, is_test: Option) -> Result; + async fn restore(&self, restore_file: &Path, is_test: Option) -> Result<()>; } pub struct DatabaseFactory; @@ -29,6 +30,7 @@ impl DatabaseFactory { DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)), DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)), DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)), + DbType::Redis => Arc::new(RedisDatabase::new(cfg)), } } @@ -42,6 +44,7 @@ impl DatabaseFactory { DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)), DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)), DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)), + DbType::Redis => Arc::new(RedisDatabase::new(cfg)), } } } diff --git a/src/domain/mod.rs b/src/domain/mod.rs index 12cc3a0..1663d08 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -3,4 +3,5 @@ pub mod postgres; pub mod mysql; mod mongodb; mod sqlite; +mod redis; diff --git a/src/domain/mongodb/database.rs b/src/domain/mongodb/database.rs index 4e31ac1..f0ec057 100644 --- a/src/domain/mongodb/database.rs +++ b/src/domain/mongodb/database.rs @@ -27,22 +27,27 @@ impl Database for MongoDatabase { ping::run(self.cfg.clone()).await } - async fn backup(&self, dir: &Path) -> Result { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; - let res = backup::run( - self.cfg.clone(), - dir.to_path_buf(), - self.file_extension(), - ) - .await; - FileLock::release(&self.cfg.generated_id).await?; + async fn backup(&self, dir: &Path, is_test: Option) -> Result { + let test_mode = is_test.unwrap_or(false); + if !test_mode { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + } + let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await; + if !test_mode { + FileLock::release(&self.cfg.generated_id).await?; + } res } - async fn restore(&self, file: &Path) -> Result<()> { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + async fn restore(&self, file: &Path, is_test: Option) -> Result<()> { + let test_mode = is_test.unwrap_or(false); + if !test_mode { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + } let res = restore::run(self.cfg.clone(), file.to_path_buf()).await; - FileLock::release(&self.cfg.generated_id).await?; + if !test_mode { + FileLock::release(&self.cfg.generated_id).await?; + } res } } diff --git a/src/domain/mysql/database.rs b/src/domain/mysql/database.rs index 13fa06f..d616f83 100644 --- a/src/domain/mysql/database.rs +++ b/src/domain/mysql/database.rs @@ -36,17 +36,28 @@ impl Database for MySQLDatabase { ping::run(self.cfg.clone(), self.build_env().clone()).await } - async fn backup(&self, dir: &Path) -> Result { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + + async fn backup(&self, dir: &Path, is_test: Option) -> Result { + let test_mode = is_test.unwrap_or(false); + if !test_mode { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + } let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env().clone(), self.file_extension()).await; - FileLock::release(&self.cfg.generated_id).await?; + if !test_mode { + FileLock::release(&self.cfg.generated_id).await?; + } res } - async fn restore(&self, file: &Path) -> Result<()> { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + async fn restore(&self, file: &Path, is_test: Option) -> Result<()> { + let test_mode = is_test.unwrap_or(false); + if !test_mode { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + } let res = restore::run(self.cfg.clone(), file.to_path_buf()).await; - FileLock::release(&self.cfg.generated_id).await?; + if !test_mode { + FileLock::release(&self.cfg.generated_id).await?; + } res } } diff --git a/src/domain/postgres/backup.rs b/src/domain/postgres/backup.rs index c7dbfcb..f6f71be 100644 --- a/src/domain/postgres/backup.rs +++ b/src/domain/postgres/backup.rs @@ -11,6 +11,7 @@ pub async fn run( cfg: DatabaseConfig, format: PostgresDumpFormat, backup_dir: PathBuf, + is_test: Option ) -> Result { tokio::task::spawn_blocking(move || -> Result { debug!("Starting backup for database {}", cfg.name); @@ -26,7 +27,8 @@ pub async fn run( } }; - let pg_dump = select_pg_path(&version).join("pg_dump"); + let pg_dump = select_pg_path(&version, is_test).join("pg_dump"); + debug!("Using pg_dump at {:?}", pg_dump); match format { diff --git a/src/domain/postgres/connection.rs b/src/domain/postgres/connection.rs index 92648f3..3145f9a 100644 --- a/src/domain/postgres/connection.rs +++ b/src/domain/postgres/connection.rs @@ -28,9 +28,14 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result { Ok(version) } -pub fn select_pg_path(version: &str) -> std::path::PathBuf { +pub fn select_pg_path(version: &str, is_test: Option) -> std::path::PathBuf { let major = version.split('.').next().unwrap_or("17"); - format!("/usr/lib/postgresql/{}/bin", major).into() + + if is_test.unwrap_or(false) { + format!("/usr/local/postgresql/{}/bin", major).into() + } else { + format!("/usr/lib/postgresql/{}/bin", major).into() + } } pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> { diff --git a/src/domain/postgres/database.rs b/src/domain/postgres/database.rs index 618c1c3..7af5376 100644 --- a/src/domain/postgres/database.rs +++ b/src/domain/postgres/database.rs @@ -2,11 +2,7 @@ use anyhow::Result; use async_trait::async_trait; use std::path::{Path, PathBuf}; -use super::{ - backup, - format::PostgresDumpFormat, - ping, restore, -}; +use super::{backup, format::PostgresDumpFormat, ping, restore}; use crate::domain::factory::Database; use crate::services::config::DatabaseConfig; use crate::utils::locks::{DbOpLock, FileLock}; @@ -35,17 +31,27 @@ impl Database for PostgresDatabase { ping::run(self.cfg.clone()).await } - async fn backup(&self, dir: &Path) -> Result { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; - let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await; - FileLock::release(&self.cfg.generated_id).await?; + async fn backup(&self, dir: &Path, is_test: Option) -> Result { + let test_mode = is_test.unwrap_or(false); + if !test_mode { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + } + let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf(), is_test).await; + if !test_mode { + FileLock::release(&self.cfg.generated_id).await?; + } res } - async fn restore(&self, file: &Path) -> Result<()> { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; - let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf()).await; - FileLock::release(&self.cfg.generated_id).await?; + async fn restore(&self, file: &Path, is_test: Option) -> Result<()> { + let test_mode = is_test.unwrap_or(false); + if !test_mode { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + } + let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf(), is_test).await; + if !test_mode { + FileLock::release(&self.cfg.generated_id).await?; + } res } } diff --git a/src/domain/postgres/restore.rs b/src/domain/postgres/restore.rs index 0458d82..a5181f7 100644 --- a/src/domain/postgres/restore.rs +++ b/src/domain/postgres/restore.rs @@ -11,6 +11,7 @@ pub async fn run( cfg: DatabaseConfig, format: PostgresDumpFormat, restore_file: PathBuf, + is_test: Option, ) -> Result<()> { tokio::task::spawn_blocking(move || -> Result<()> { debug!("Starting restore for database {}", cfg.name); @@ -26,7 +27,8 @@ pub async fn run( } }; - let pg_restore = select_pg_path(&version).join("pg_restore"); + let pg_restore = select_pg_path(&version, is_test).join("pg_restore"); + debug!("Using pg_restore at {:?}", pg_restore); if let Err(e) = futures::executor::block_on(terminate_connections(&cfg)) { diff --git a/src/domain/redis/backup.rs b/src/domain/redis/backup.rs new file mode 100644 index 0000000..9fbe995 --- /dev/null +++ b/src/domain/redis/backup.rs @@ -0,0 +1,65 @@ +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result}; +use std::path::PathBuf; +use std::process::Command; +use tracing::{debug, error, info}; + +pub async fn run( + cfg: DatabaseConfig, + backup_dir: PathBuf, + file_extension: &'static str, +) -> Result { + tokio::task::spawn_blocking(move || -> Result { + debug!("Starting Redis backup for database {}", cfg.name); + + let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension)); + + let mut cmd = Command::new("redis-cli"); + + cmd.arg("-h") + .arg(&cfg.host) + .arg("-p") + .arg(cfg.port.to_string()); + + if !cfg.username.is_empty() { + cmd.arg("--user").arg(&cfg.username); + } + + if !cfg.password.is_empty() { + cmd.arg("-a").arg(&cfg.password); + } + + cmd.arg("--rdb").arg(&file_path); + + debug!("Command Backup: {:?}", cmd); + + let output = cmd.output().context("Redis backup command failed")?; + + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + + if !output.status.success() { + if stderr.contains("NOAUTH") { + error!( + "Redis backup failed for {}: Authentication required (NOAUTH)", + cfg.name + ); + anyhow::bail!( + "Redis backup failed for {}: Authentication required", + cfg.name + ); + } else { + error!("Redis backup failed for {}: {}", cfg.name, stderr); + anyhow::bail!("Redis backup failed for {}: {}", cfg.name, stderr); + } + } + + info!( + "Redis backup completed for {}. Output: {}", + cfg.name, stdout + ); + + Ok(file_path) + }) + .await? +} diff --git a/src/domain/redis/database.rs b/src/domain/redis/database.rs new file mode 100644 index 0000000..e536b18 --- /dev/null +++ b/src/domain/redis/database.rs @@ -0,0 +1,45 @@ +use anyhow::{Result, bail}; +use async_trait::async_trait; +use std::path::{Path, PathBuf}; + +use crate::domain::factory::Database; +use crate::domain::redis::{backup, ping}; +use crate::services::config::DatabaseConfig; +use crate::utils::locks::{DbOpLock, FileLock}; + +pub struct RedisDatabase { + cfg: DatabaseConfig, +} + +impl RedisDatabase { + pub fn new(cfg: DatabaseConfig) -> Self { + Self { cfg } + } +} + +#[async_trait] +impl Database for RedisDatabase { + fn file_extension(&self) -> &'static str { + ".rdb" + } + + async fn ping(&self) -> Result { + ping::run(self.cfg.clone()).await + } + + async fn backup(&self, dir: &Path, is_test: Option) -> Result { + let test_mode = is_test.unwrap_or(false); + if !test_mode { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + } + let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await; + if !test_mode { + FileLock::release(&self.cfg.generated_id).await?; + } + res + } + + async fn restore(&self, _file: &Path, _is_test: Option) -> Result<()> { + bail!("Restore not supported for Redis databases") + } +} diff --git a/src/domain/redis/mod.rs b/src/domain/redis/mod.rs new file mode 100644 index 0000000..3997ede --- /dev/null +++ b/src/domain/redis/mod.rs @@ -0,0 +1,3 @@ +mod backup; +pub mod database; +mod ping; diff --git a/src/domain/redis/ping.rs b/src/domain/redis/ping.rs new file mode 100644 index 0000000..d06df71 --- /dev/null +++ b/src/domain/redis/ping.rs @@ -0,0 +1,55 @@ +use tracing::{debug, info}; +use crate::services::config::DatabaseConfig; +use tokio::process::Command; +use tokio::time::{timeout, Duration}; +use anyhow::{Result, Context}; + +pub async fn run(cfg: DatabaseConfig) -> Result { + let mut cmd = Command::new("redis-cli"); + cmd.arg("-h") + .arg(&cfg.host) + .arg("-p") + .arg(cfg.port.to_string()); + + if !cfg.username.is_empty() { + cmd.arg("--user").arg(&cfg.username); + } + + if !cfg.password.is_empty() { + cmd.arg("-a").arg(&cfg.password); + } + + cmd.arg("PING"); + + debug!("Command Ping: {:?}", cmd); + + + let result = timeout(Duration::from_secs(10), cmd.output()).await; + + match result { + Ok(output) => { + let output = output.context("Failed to execute redis-cli")?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + info!("Redis stdout: {}", stdout); + info!("Redis stderr: {}", stderr); + + if stderr.contains("NOAUTH") { + info!("Redis authentication failed (NOAUTH required)"); + return Ok(false); + } + + if !output.status.success() { + info!("Redis command failed with status: {:?}", output.status); + return Ok(false); + } + + Ok(stdout.contains("PONG")) + } + Err(_) => { + info!("Timeout connecting to Redis at {}:{}", cfg.host, cfg.port); + Ok(false) + } + } +} \ No newline at end of file diff --git a/src/domain/sqlite/database.rs b/src/domain/sqlite/database.rs index cbc575a..5bcebab 100644 --- a/src/domain/sqlite/database.rs +++ b/src/domain/sqlite/database.rs @@ -27,17 +27,26 @@ impl Database for SqliteDatabase { ping::run(self.cfg.clone()).await } - async fn backup(&self, dir: &Path) -> Result { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + async fn backup(&self, dir: &Path, is_test: Option) -> Result { + let test_mode = is_test.unwrap_or(false); + if !test_mode { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + } let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await; - FileLock::release(&self.cfg.generated_id).await?; + if !test_mode { + FileLock::release(&self.cfg.generated_id).await?; + } res } - - async fn restore(&self, file: &Path) -> Result<()> { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + async fn restore(&self, file: &Path, is_test: Option) -> Result<()> { + let test_mode = is_test.unwrap_or(false); + if !test_mode { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + } let res = restore::run(self.cfg.clone(), file.to_path_buf()).await; - FileLock::release(&self.cfg.generated_id).await?; + if !test_mode { + FileLock::release(&self.cfg.generated_id).await?; + } res } } diff --git a/src/services/backup/runner.rs b/src/services/backup/runner.rs index 5a46b24..3e7708c 100644 --- a/src/services/backup/runner.rs +++ b/src/services/backup/runner.rs @@ -40,7 +40,7 @@ impl BackupService { }); } - match db.backup(tmp_path).await { + match db.backup(tmp_path, Some(false)).await { Ok(file) => Ok(BackupResult { generated_id, diff --git a/src/services/config.rs b/src/services/config.rs index af98f07..bdd990a 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -19,6 +19,7 @@ pub enum DbType { Postgresql, MongoDB, Sqlite, + Redis // Add other DB types if needed } @@ -30,6 +31,7 @@ impl DbType { DbType::Postgresql => "postgresql", DbType::MongoDB => "mongodb", DbType::Sqlite => "sqlite", + DbType::Redis => "redis", } } } @@ -162,17 +164,17 @@ impl ConfigService { }; let host = match db.db_type { - DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB => required(&db.host, &db.name, "host")?, + DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis => required(&db.host, &db.name, "host")?, DbType::Sqlite => optional(&db.host), }; let port = match db.db_type { - DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB => required(&db.port, &db.name, "port")?, + DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis => required(&db.port, &db.name, "port")?, DbType::Sqlite => db.port.unwrap_or(0), }; let database_name = match db.db_type { - DbType::Sqlite => optional(&db.database), + DbType::Sqlite | DbType::Redis => optional(&db.database), _ => required(&db.database, &db.name, "database")? }; diff --git a/src/services/restore/runner.rs b/src/services/restore/runner.rs index eaefb02..36751dd 100644 --- a/src/services/restore/runner.rs +++ b/src/services/restore/runner.rs @@ -31,7 +31,7 @@ impl RestoreService { }); } - match db.restore(&backup_file).await { + match db.restore(&backup_file, Some(false)).await { Ok(_) => Ok(RestoreResult { generated_id, diff --git a/src/tests/domain/mod.rs b/src/tests/domain/mod.rs index bc79954..d68fe4f 100644 --- a/src/tests/domain/mod.rs +++ b/src/tests/domain/mod.rs @@ -1 +1,2 @@ -mod postgres; \ No newline at end of file +mod postgres; +mod redis; \ No newline at end of file diff --git a/src/tests/domain/postgres.rs b/src/tests/domain/postgres.rs index f413838..ca0543f 100644 --- a/src/tests/domain/postgres.rs +++ b/src/tests/domain/postgres.rs @@ -1,35 +1,109 @@ +use crate::domain::factory::DatabaseFactory; +use crate::services::config::{DatabaseConfig, DbType}; +use crate::tests::init_tracing_for_test; +use crate::utils::compress::{compress_to_tar_gz_large, decompress_large_tar_gz}; use oauth2::url; +use std::path::PathBuf; +use tempfile::TempDir; use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, ImageExt}; use testcontainers_modules::postgres::Postgres; -use crate::services::config::{DatabaseConfig, DbType}; +use tracing::{error, info}; use url::Host; -use crate::domain::factory::DatabaseFactory; - -#[tokio::test] -async fn postgres_ping_test() { +async fn create_config() -> (ContainerAsync, DatabaseConfig) { let container = Postgres::default() + .with_env_var("POSTGRES_DB", "testdb") + .with_env_var("POSTGRES_USER", "testuser") + .with_env_var("POSTGRES_PASSWORD", "changeme") + .with_tag("17") .start() .await .unwrap(); - let host = container.get_host().await.unwrap_or(Host::parse("127.0.0.1").unwrap()); - let port = container.get_host_port_ipv4(5432).await.unwrap_or(5432) ; + let host = container + .get_host() + .await + .unwrap_or(Host::parse("127.0.0.1").unwrap()); + + let port = container.get_host_port_ipv4(5432).await.unwrap_or(5432); let config = DatabaseConfig { name: "My test Postgres Database".to_string(), - database: "postgres".to_string(), + database: "testdb".to_string(), db_type: DbType::Postgresql, - username: "postgres".to_string(), - password: "postgres".to_string(), + username: "testuser".to_string(), + password: "changeme".to_string(), port, host: host.to_string(), generated_id: "40875631-e3d2-4dfe-a26b-2a347ecc64fd".to_string(), path: "".to_string(), }; + (container, config) +} + +#[tokio::test] +async fn postgres_ping_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + let db = DatabaseFactory::create_for_backup(config.clone()).await; let reachable = db.ping().await.unwrap_or_else(|_| false); assert_eq!(reachable, true); -} \ No newline at end of file +} + +#[tokio::test] +async fn postgres_backup_restore_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + + let temp_dir = TempDir::new().unwrap(); + let backup_path = temp_dir.path(); + + let db = DatabaseFactory::create_for_backup(config.clone()).await; + + let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + + assert!(file_path.is_file()); + + let compression = compress_to_tar_gz_large(&file_path).await.unwrap(); + + assert!(compression.compressed_path.is_file()); + + let files = decompress_large_tar_gz(compression.compressed_path.as_path(), temp_dir.path()) + .await + .unwrap(); + + let backup_file: PathBuf; + + if files.len() == 1 { + backup_file = files[0].clone() + } else { + backup_file = "".into() + } + + let db = DatabaseFactory::create_for_restore(config.clone(), &backup_file).await; + + let reachable = db.ping().await.unwrap_or(false); + + info!("Reachable: {}", reachable); + + assert_eq!(reachable, true); + + info!("Running pg_restore: {:?}", backup_file); + + match db.restore(&backup_file, Some(true)).await { + Ok(_) => { + info!("Restore succeeded for {}", config.generated_id); + assert!(true) + } + Err(e) => { + error!("Restore failed for {}: {:?}", config.generated_id, e); + assert!(false) + } + } +} diff --git a/src/tests/domain/redis.rs b/src/tests/domain/redis.rs new file mode 100644 index 0000000..f704aab --- /dev/null +++ b/src/tests/domain/redis.rs @@ -0,0 +1,65 @@ +use tempfile::TempDir; +use testcontainers::runners::AsyncRunner; +use testcontainers::ContainerAsync; +use testcontainers_modules::redis::Redis; +use url::Host; + +use crate::domain::factory::DatabaseFactory; +use crate::services::config::{DatabaseConfig, DbType}; +use crate::tests::init_tracing_for_test; + +async fn create_config() -> (ContainerAsync, DatabaseConfig) { + let container = Redis::default().start().await.unwrap(); + + let host = container + .get_host() + .await + .unwrap_or(Host::parse("127.0.0.1").unwrap()); + + let port = container + .get_host_port_ipv4(6379) + .await + .unwrap_or(6379); + + let config = DatabaseConfig { + name: "Test Redis".to_string(), + database: "redis".to_string(), + username: "".to_string(), + password: "".to_string(), + db_type: DbType::Redis, + port, + host: host.to_string(), + generated_id: "40875631-e3d2-4dfe-a26b-2a347ecc64fd".to_string(), + path: "".to_string(), + }; + + (container, config) +} + +#[tokio::test] +async fn redis_ping_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + + let db = DatabaseFactory::create_for_backup(config.clone()).await; + let reachable = db.ping().await.unwrap_or(false); + + assert!(reachable); +} + +#[tokio::test] +async fn redis_backup_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + + let temp_dir = TempDir::new().unwrap(); + let backup_path = temp_dir.path(); + + let db = DatabaseFactory::create_for_backup(config.clone()).await; + + let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + + assert!(file_path.is_file()); +} \ No newline at end of file diff --git a/src/tests/mod.rs b/src/tests/mod.rs index b7ad210..6ece72c 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -1,2 +1,18 @@ mod utils; -mod domain; \ No newline at end of file +mod domain; + + + +use once_cell::sync::Lazy; +use tracing_subscriber; + +static TRACING: Lazy<()> = Lazy::new(|| { + let _ = tracing_subscriber::fmt() + .with_test_writer() + .with_env_filter("debug") + .try_init(); +}); + +fn init_tracing_for_test() -> () { + Lazy::force(&TRACING); +} \ No newline at end of file