From bab9220495d10314d825c1803f0db87f70d2add3 Mon Sep 17 00:00:00 2001 From: TheGuitarleader <26614720+TheGuitarleader@users.noreply.github.com> Date: Fri, 12 Jun 2026 03:14:23 -0500 Subject: [PATCH 1/2] Now caches vault files locally reducing engress on s3 providers --- Parallel.Cli/Commands/FetchCommand.cs | 46 ++++++++++++ Parallel.Cli/Commands/IgnoreCommand.cs | 4 +- Parallel.Cli/Commands/SnapshotsCommand.cs | 4 +- Parallel.Cli/Commands/VaultsCommand.cs | 7 +- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 74 +++++++++++--------- Parallel.Core/IO/Syncing/ISyncManager.cs | 2 +- Parallel.Core/Settings/ParallelConfig.cs | 17 +---- Parallel.Core/Settings/RemoteVaultConfig.cs | 2 +- Parallel.Core/Settings/StorageCredentials.cs | 9 +-- Parallel.Core/Storage/S3StorageProvider.cs | 21 +++--- README.md | 44 ++++++------ 11 files changed, 134 insertions(+), 96 deletions(-) create mode 100644 Parallel.Cli/Commands/FetchCommand.cs diff --git a/Parallel.Cli/Commands/FetchCommand.cs b/Parallel.Cli/Commands/FetchCommand.cs new file mode 100644 index 0000000..2e7d906 --- /dev/null +++ b/Parallel.Cli/Commands/FetchCommand.cs @@ -0,0 +1,46 @@ +// Copyright 2026 Kyle Ebbinga + +using System.CommandLine; +using Parallel.Cli.Utils; +using Parallel.Core.IO.Syncing; +using Parallel.Core.Settings; + +namespace Parallel.Cli.Commands +{ + public class FetchCommand : Command + { + private readonly Option _configOpt = new(["--config", "-c"], "The vault configuration to use."); + + public FetchCommand() : base("fetch", "Fetches information from a vault.") + { + this.AddOption(_configOpt); + this.SetHandler(HandleFetchAsync, _configOpt); + } + + private async Task HandleFetchAsync(string config) + { + LocalVaultConfig? localVault = ParallelConfig.GetVault(config); + if (localVault != null) + { + await FetchVaultAsync(localVault); + return; + } + + await Program.Settings.ForEachVaultAsync(FetchVaultAsync); + } + + private async Task FetchVaultAsync(LocalVaultConfig localVault) + { + CommandLine.WriteLine($"Retrieving vault information...", ConsoleColor.DarkGray); + ISyncManager? syncManager = SyncManager.CreateNew(localVault); + if (syncManager == null || !await syncManager.ConnectAsync(true)) + { + CommandLine.WriteLine(localVault, "Failed to connect to vault!", ConsoleColor.Red); + return; + } + + CommandLine.WriteLine(syncManager.LocalVault, $"Successfully fetched vault data for: '{localVault.Name}'", ConsoleColor.Green); + await syncManager.DisconnectAsync(); + } + } +} \ No newline at end of file diff --git a/Parallel.Cli/Commands/IgnoreCommand.cs b/Parallel.Cli/Commands/IgnoreCommand.cs index 2b65c1f..d450792 100644 --- a/Parallel.Cli/Commands/IgnoreCommand.cs +++ b/Parallel.Cli/Commands/IgnoreCommand.cs @@ -57,7 +57,7 @@ private async Task AddPathAsync(LocalVaultConfig vault, string path) return; } - CommandLine.WriteLine(vault, $"Successfully added '{path}'", ConsoleColor.Green); + CommandLine.WriteLine(vault, $"Successfully ignored '{path}'", ConsoleColor.Green); await syncManager.DisconnectAsync(); } @@ -89,7 +89,7 @@ private async Task RemovePathAsync(LocalVaultConfig vault, string path) return; } - CommandLine.WriteLine(vault, $"Successfully removed '{path}'", ConsoleColor.Green); + CommandLine.WriteLine(vault, $"Successfully included '{path}'", ConsoleColor.Green); await syncManager.DisconnectAsync(); } } diff --git a/Parallel.Cli/Commands/SnapshotsCommand.cs b/Parallel.Cli/Commands/SnapshotsCommand.cs index 9c38a6e..720a2bb 100644 --- a/Parallel.Cli/Commands/SnapshotsCommand.cs +++ b/Parallel.Cli/Commands/SnapshotsCommand.cs @@ -150,8 +150,8 @@ private async Task RestoreSnapshotAsync(LocalVaultConfig vault, string? name, st CommandLine.WriteLine($"Loading snapshot {snapshotFile}"); - string remoteSnapshotFile = PathBuilder.GetSnapshotFile(vault, snapshotFilename); - string localSnapshotFile = Path.Combine(PathBuilder.TempDirectory, snapshotFilename + ".json"); + //string remoteSnapshotFile = PathBuilder.GetSnapshotFile(vault, snapshotFilename); + //string localSnapshotFile = Path.Combine(PathBuilder.TempDirectory, snapshotFilename + ".json"); } } } \ No newline at end of file diff --git a/Parallel.Cli/Commands/VaultsCommand.cs b/Parallel.Cli/Commands/VaultsCommand.cs index aead5bf..4129322 100644 --- a/Parallel.Cli/Commands/VaultsCommand.cs +++ b/Parallel.Cli/Commands/VaultsCommand.cs @@ -53,6 +53,10 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") string? bucketInput = CommandLine.ReadString("Bucket Name (Leave empty for default)"); string bucketName = string.IsNullOrEmpty(bucketInput) ? "parallel" : bucketInput; spc.RootDirectory = bucketName; + + string? regionInput = CommandLine.ReadString("Region Name (Leave empty for default)"); + string regionName = string.IsNullOrEmpty(regionInput) ? "us-east-1" : regionInput; + spc.Region = regionName; spc.Address = CommandLine.ReadString("Endpoint"); spc.Username = CommandLine.ReadString("Access Key"); @@ -73,9 +77,6 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") string? inputName = CommandLine.ReadString("Name (Leave empty for machine name)"); string profileName = string.IsNullOrEmpty(inputName) ? Environment.MachineName : inputName; - spc.Encrypt = CommandLine.ReadBool("Encrypt files? (y/n)", false); - spc.EncryptionKey = spc.Encrypt ? HashGenerator.GenerateHash(32, true) : null; - LocalVaultConfig localVault = new(profileId, profileName, spc); localVault.Enabled = CommandLine.ReadBool("Enabled? (y/n)", true); Program.Settings.Vaults.Add(localVault); diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index c95ae2a..a6a23ec 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -46,9 +46,8 @@ protected BaseSyncManager(LocalVaultConfig localVault) } /// - public async Task ConnectAsync() + public async Task ConnectAsync(bool force = false) { - Log.Debug("[{LocalVaultId}] Connecting...", LocalVault.Id); string root = PathBuilder.GetRootDirectory(LocalVault); if (!await StorageProvider.ExistsAsync(root)) { @@ -56,43 +55,50 @@ public async Task ConnectAsync() Log.Debug("Created root directory: {Root}", root); } - - - if (!await StorageProvider.ExistsAsync(PathBuilder.GetConfigurationFile(LocalVault))) - { - RemoteVault = new RemoteVaultConfig(LocalVault); - RemoteVault.IgnoreDirectories.Add(PathBuilder.GetRootDirectory(LocalVault)); - RemoteVault.Save(TempConfigFile); - - Log.Debug("Created config file: {TempConfigFile}", TempConfigFile); - } - else + // Checks temp files for a local download of the config file + if (!File.Exists(TempConfigFile) || File.GetLastWriteTimeUtc(TempConfigFile) <= DateTime.UtcNow.AddHours(-6) || force) { - await StorageProvider.DownloadFileAsync(new LocalFile(TempConfigFile), PathBuilder.GetConfigurationFile(LocalVault)); - RemoteVaultConfig? config = RemoteVaultConfig.Load(TempConfigFile); - if (config == null) return false; - RemoteVault = config; - LocalVault.Name = config.Name; - Log.Debug("Downloaded file: {TempConfigFile}", TempConfigFile); + Log.Debug($"No local config file found. Fetching..."); + if (!await StorageProvider.ExistsAsync(PathBuilder.GetConfigurationFile(LocalVault))) + { + RemoteVault = new RemoteVaultConfig(LocalVault); + RemoteVault.IgnoreDirectories.Add(PathBuilder.GetRootDirectory(LocalVault)); + RemoteVault.Save(TempConfigFile); + + Log.Debug("Created config file: {TempConfigFile}", TempConfigFile); + } + else + { + await StorageProvider.DownloadFileAsync(new LocalFile(TempConfigFile), PathBuilder.GetConfigurationFile(LocalVault)); + Log.Debug("Downloaded file: {TempConfigFile}", TempConfigFile); + } } - if (!await StorageProvider.ExistsAsync(PathBuilder.GetDatabaseFile(LocalVault))) + // Checks temp files for a local download of the database file + if (!File.Exists(TempDbFile) || File.GetLastWriteTimeUtc(TempDbFile) <= DateTime.UtcNow.AddHours(-3) || force) { - if (File.Exists(TempDbFile)) File.Delete(TempDbFile); - Database = new SqliteContext(TempDbFile); - await Database.InitializeAsync(); - - Log.Debug("Create db file: {TempDbFile}", TempDbFile); - } - else - { - string remoteDbFile = PathBuilder.GetDatabaseFile(LocalVault); - //await StorageProvider.CloneFileAsync(remoteDbFile, remoteDbFile + ".old"); - await StorageProvider.DownloadFileAsync(new LocalFile(TempDbFile), remoteDbFile); - Database = new SqliteContext(TempDbFile); - - Log.Debug("Downloaded file: {TempDbFile}", TempDbFile); + Log.Debug($"No local database file found. Fetching..."); + if (!await StorageProvider.ExistsAsync(PathBuilder.GetDatabaseFile(LocalVault))) + { + Log.Debug("Create db file: {TempDbFile}", TempDbFile); + if (File.Exists(TempDbFile)) File.Delete(TempDbFile); + Database = new SqliteContext(TempDbFile); + await Database.InitializeAsync(); + } + else + { + string remoteDbFile = PathBuilder.GetDatabaseFile(LocalVault); + await StorageProvider.DownloadFileAsync(new LocalFile(TempDbFile), remoteDbFile); + Log.Debug("Downloaded file: {TempDbFile}", TempDbFile); + } } + + // Load the temp files + Database = new SqliteContext(TempDbFile); + RemoteVaultConfig? config = RemoteVaultConfig.Load(TempConfigFile); + if (config == null) return false; + RemoteVault = config; + LocalVault.Name = config.Name; Log.Information("[{LocalVaultId}] Connected", LocalVault.Id); return true; diff --git a/Parallel.Core/IO/Syncing/ISyncManager.cs b/Parallel.Core/IO/Syncing/ISyncManager.cs index 6d44ac8..3929ac6 100644 --- a/Parallel.Core/IO/Syncing/ISyncManager.cs +++ b/Parallel.Core/IO/Syncing/ISyncManager.cs @@ -41,7 +41,7 @@ public interface ISyncManager /// /// Establishes a connection to the associated and downloads the needed files. /// - Task ConnectAsync(); + Task ConnectAsync(bool force = false); /// /// Closes the current connection and releases its resources. diff --git a/Parallel.Core/Settings/ParallelConfig.cs b/Parallel.Core/Settings/ParallelConfig.cs index 94e65c4..f51073a 100644 --- a/Parallel.Core/Settings/ParallelConfig.cs +++ b/Parallel.Core/Settings/ParallelConfig.cs @@ -17,11 +17,6 @@ public class ParallelConfig /// private static string ConfigFile { get; } = Path.Combine(PathBuilder.ProgramData, "Configuration.json"); - /// - /// The location of files for different file system credentials./>. - /// - public static string VaultsDir { get; } = Path.Combine(PathBuilder.ProgramData, "Vaults"); - /// /// Gets a set of static options for . /// @@ -31,16 +26,10 @@ public class ParallelConfig }; /// - /// The address that will accept incoming commands. - /// Default: 127.0.0.1 - /// - public string Address { get; set; } = "127.0.0.1"; - - /// - /// The port number to listen for commands on. - /// Default: 8192 + /// Gets or sets the amount of time, in hours, to cache vault files. + /// Default: 1 /// - public int ListenerPort { get; set; } = 8192; + public int CacheDuration { get; set; } = 1; /// /// Gets or sets the maximum number of concurrent vaults that can run. diff --git a/Parallel.Core/Settings/RemoteVaultConfig.cs b/Parallel.Core/Settings/RemoteVaultConfig.cs index 2566100..c74eb4e 100644 --- a/Parallel.Core/Settings/RemoteVaultConfig.cs +++ b/Parallel.Core/Settings/RemoteVaultConfig.cs @@ -115,7 +115,7 @@ private static HashSet CreateIgnoreDirectories() public void Save(string path) { - File.WriteAllText(path, JsonConvert.SerializeObject(this, Formatting.Indented)); + File.WriteAllText(path, JsonConvert.SerializeObject(this)); } } } \ No newline at end of file diff --git a/Parallel.Core/Settings/StorageCredentials.cs b/Parallel.Core/Settings/StorageCredentials.cs index 44ae871..c387e4e 100644 --- a/Parallel.Core/Settings/StorageCredentials.cs +++ b/Parallel.Core/Settings/StorageCredentials.cs @@ -37,14 +37,9 @@ public class StorageCredentials public string? Password { get; set; } /// - /// If the file system is encrypting files. + /// The region to use for S3 authentication. /// - public bool Encrypt { get; set; } = false; - - /// - /// The master key used for encryption. - /// - public string? EncryptionKey { get; set; } = null; + public string? Region { get; set; } public bool ForceStyle { get; set; } diff --git a/Parallel.Core/Storage/S3StorageProvider.cs b/Parallel.Core/Storage/S3StorageProvider.cs index ba08a9f..ca7b02a 100644 --- a/Parallel.Core/Storage/S3StorageProvider.cs +++ b/Parallel.Core/Storage/S3StorageProvider.cs @@ -2,6 +2,7 @@ using System.IO.Compression; using System.IO.Pipelines; +using Amazon; using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; @@ -32,7 +33,10 @@ public S3StorageProvider(LocalVaultConfig localVault) AmazonS3Config config = new AmazonS3Config() { ServiceURL = localVault.Credentials.Address, - ForcePathStyle = localVault.Credentials.ForceStyle + ForcePathStyle = localVault.Credentials.ForceStyle, + AuthenticationRegion = localVault.Credentials.Region, + DisableS3ExpressSessionAuth = true, + UseHttp = localVault.Credentials.Address?.StartsWith("http", StringComparison.OrdinalIgnoreCase) ?? false }; _client = new AmazonS3Client(localVault.Credentials.Username, Encryption.Decode(localVault.Credentials.Password), config); @@ -80,15 +84,14 @@ public async Task DeleteFileAsync(string path) public async Task ExistsAsync(string path) { - try - { - await _client.GetObjectMetadataAsync(_bucket, path); - return true; - } - catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) + ListObjectsV2Response? response = await _client.ListObjectsV2Async(new ListObjectsV2Request { - return false; - } + BucketName = _bucket, + Prefix = path, + MaxKeys = 1 + }); + + return response.KeyCount > 0; } public Task GetDirectoryName(string path) diff --git a/README.md b/README.md index e85a534..4696eb9 100644 --- a/README.md +++ b/README.md @@ -6,31 +6,27 @@ Your files under your control. ## What is Parallel? -Parallel is a modular, cross-platform file backup and synchronization tool built for people who want full control of their files. It ditches the cloud-first assumptions and gives you full transparency and control over how, when, and where your files move. No contracts, no vendor lock-ins, no silent overwrites. Just clean, dependable syncing on your terms. +Parallel is a **snapshot‑based backup and sync engine** built for people who want real control over their data. It doesn’t assume you want everything in the cloud, and it doesn’t hide what it’s doing. It builds a local‑first, verifiable history of your files across Windows, macOS, and Linux. -Parallel was originally built to handle **terabytes of data** because Dropbox simply couldn’t. When commercial cloud services reach their limits, Parallel stepped in to offer **unbounded scale**, **local-first logic**, and **configurable workflows** that respect your storage, bandwidth, and rules. +Under the hood, Parallel works much more like **Git across the whole filesystem** than a typical sync tool. Every file is hashed, deduped, and stored as an immutable object. Snapshots are fast and incremental, so you can keep a long history without wasting space. -## Why Parallel? +Parallel started as a way to handle **multi‑terabyte datasets** that commercial services kept choking on. The goal was simple: no limits, no lock‑in, and no guessing what the software is doing with your files. -Parallel is completely free and open source. You provide the storage, and Parallel handles the sync. Whether it’s an external drive, a NAS, a remote SSH server, or an S3-compatible cloud like [Storj](https://www.storj.io/) or [Wasabi](https://wasabi.com/), Parallel adapts to what you own. +## Why Parallel? -Your computer already gives you enough to fight with, your files don't have to be one of them. Parallel keeps backups simple, transparent, and under your control with no cloud drama. +Parallel is free, open source, and built around the idea that your storage is your business. You point it at whatever you own, whether it's an external drive, a NAS, an SSH server, or an S3‑compatible bucket like [Storj](https://www.storj.io/) or [Wasabi](https://wasabi.com/), and Parallel handles the rest. -| Feature | **Parallel** | **Dropbox** | **OneDrive** | **iCloud** | **File History (Windows)** | -|-------------------------------|--------------|-------------|--------------|------------|-----------------------------| -| **Open Source** | ✅ Yes | ❌ No | ❌ No | ❌ No | ❌ No -| **Local-first** | ✅ Always | ❌ Cloud-first | ❌ Cloud-first | ⚠️ Hybrid (Apple ecosystem) | ✅ Yes -| **Modular storage options** | ✅ Any (NAS, SSH, S3) | ❌ Vendor-locked | ❌ Vendor-locked | ❌ Vendor-locked | ❌ Local only -| **Compression** | ✅ Always | ❌ No | ❌ No | ❌ No | ❌ No -| **Cross-platform** | ✅ Yes | ✅ Yes | ✅ Yes | ⚠️ Apple-centric | ❌ Windows only -| **Version History** | ✅ Yes | ✅ Yes | ✅ Yes | ✅ Yes | ⚠️ Limited -| **System Snapshots** | ✅ Yes | ❌ No | ❌ No | ❌ No | ⚠️ Limited -| **Offline access** | ✅ Full | ⚠️ Partial | ⚠️ Partial | ⚠️ Partial | ✅ Full -| **Free to use** | ✅ Always | ⚠️ 2GB free | ⚠️ 5GB free | ⚠️ 5GB free | ✅ Yes -| **Max storage** | ✅ Unlimited (your hardware) | ⚠️ 2GB (free), 3TB (personal), 15TB (enterprise) | ⚠️ 5TB (personal), 25TB (enterprise) | ⚠️ 5GB–12TB (paid tiers) | ⚠️ Limited by drive size +It’s not just a sync tool. It gives you: +- Modern file compression to save on storage. +- Content‑addressed storage for automatic dedupe. +- Integrity checks on every file. +- System‑wide versioning, not just per‑folder history. +- Cross‑platform snapshot sync. +- Tools for cleaning up old data and finding duplicates. +Your computer already has enough ways to frustrate you. Your backup system shouldn’t be one of them. Parallel keeps things straightforward, predictable, and under your control, with no silent overwrites, no surprise limits, no nonsense. -## 📦 Quick Start Guide +## Quick Start Guide #### 1. Install Parallel Download the latest [release](https://github.com/EntexInteractive/Parallel/releases/latest) or build from source: ``` @@ -45,7 +41,7 @@ curl -sSL https://raw.githubusercontent.com/EntexInteractive/Parallel/main/insta ``` #### 2. Set up your vaults -Vaults are storage targets where Parallel sends and receives files. This can be an external drive, NAS share, SSH server, or S3-compatible cloud. +Vaults are storage targets where Parallel sends and receives files. This can be an external drive, a NAS share, an SSH server, or an S3-compatible cloud. ``` parallel vaults add ``` @@ -67,16 +63,18 @@ Parallel can restore files from a vault with: parallel restore --path "C:\Windows\System32" parallel restore -p "C:\Windows\System32\cmd.exe" ``` -Parallel keeps revisions of files. To restore files to a previous version and not the latest, you can use the `--before` option and provide a valid timestamp string. See more about DateTime [parsing](https://learn.microsoft.com/en-us/dotnet/api/system.datetime.parse?view=net-10.0#StringToParse). +Parallel keeps revisions of files. To restore files to a previous version and not the latest, you can use the `--before` option and provide a valid timestamp string. ``` parallel restore --path "C:\Windows\System32" --before "2025-12-16 5:11 PM" parallel restore -p "C:\Windows\System32\cmd.exe" --before "12/16/25" ``` -## 🧪 Status +## Status + +Parallel is in active development and still considered early-beta. The core engine file syncing and restoring is functional on Windows and Linux, but the project is evolving quickly. Expect breaking changes, new features landing often, and some rough edges. -Parallel is currently in early development. Expect rapid iteration, breaking changes, and lots of modular experimentation. Contributions, feedback, and testing are welcome! +If you’re comfortable testing early software and giving feedback, you’re exactly the kind of person Parallel is built for right now. -## 💬 Contact +## Contact For questions and ideas, reach out via our [GitHub Issues](https://github.com/EntexInteractive/Parallel/issues). From 2e79cceb4bab429b1b357b2f257b63c787aee9d3 Mon Sep 17 00:00:00 2001 From: TheGuitarleader <26614720+TheGuitarleader@users.noreply.github.com> Date: Sat, 13 Jun 2026 02:41:52 -0500 Subject: [PATCH 2/2] Fixed various issues --- Parallel.Cli/Commands/PruneCommand.cs | 13 +++++++++++-- Parallel.Cli/Commands/RestoreCommand.cs | 2 +- Parallel.Cli/Commands/ScrubCommand.cs | 2 +- Parallel.Cli/Commands/SnapshotsCommand.cs | 1 + Parallel.Cli/Commands/StatsCommand.cs | 12 +++++------- Parallel.Cli/Commands/SyncCommand.cs | 4 ++-- Parallel.Cli/Parallel.Cli.csproj | 1 + Parallel.Cli/Program.cs | 11 ++--------- ...{ProgressReport.cs => ProgressReporter.cs} | 8 ++++---- .../Database/Contexts/SqliteContext.cs | 15 +++++++++++++-- Parallel.Core/Database/IDatabase.cs | 3 ++- .../Diagnostics/IProgressReporter.cs | 5 ----- ...Reporter.cs => LoggingProgressReporter.cs} | 19 ++++++++++++++----- Parallel.Core/IO/PathBuilder.cs | 12 ++++++++++++ Parallel.Core/IO/Syncing/BaseSyncManager.cs | 8 ++++++++ Parallel.Core/Storage/IStorageProvider.cs | 6 ++++++ Parallel.Core/Storage/LocalStorageProvider.cs | 6 ++++++ Parallel.Core/Storage/S3StorageProvider.cs | 16 ++++++++++++++++ Parallel.Core/Storage/SshStorageProvider.cs | 6 ++++++ 19 files changed, 111 insertions(+), 39 deletions(-) rename Parallel.Cli/Utils/{ProgressReport.cs => ProgressReporter.cs} (82%) rename Parallel.Core/Diagnostics/{NullProgressReporter.cs => LoggingProgressReporter.cs} (50%) diff --git a/Parallel.Cli/Commands/PruneCommand.cs b/Parallel.Cli/Commands/PruneCommand.cs index 9f9b348..784e9e0 100644 --- a/Parallel.Cli/Commands/PruneCommand.cs +++ b/Parallel.Cli/Commands/PruneCommand.cs @@ -126,7 +126,16 @@ private async Task PrunePathAsync(LocalVaultConfig vault, string path, DateTime private async Task PruneInternalAsync(ISyncManager syncManager, string path, DateTime timestamp, bool force, bool dryRun) { CommandLine.WriteLine(syncManager.RemoteVault, $"Scanning for files in {path}...", ConsoleColor.DarkGray); - IReadOnlyList files = await (syncManager.Database?.GetFilesAsync(path, timestamp, true) ?? Task.FromResult>([])); + IReadOnlyList files; + if (force) + { + files = await (syncManager.Database?.GetFilesAsync(path, timestamp) ?? Task.FromResult>([])); + } + else + { + files = await (syncManager.Database?.GetFilesAsync(path, timestamp, true) ?? Task.FromResult>([])); + } + if (files.Count == 0) { CommandLine.WriteLine($"No prunable files were found!", ConsoleColor.Yellow); @@ -153,7 +162,7 @@ private async Task PruneInternalAsync(ISyncManager syncManager, string path, Dat else { CommandLine.WriteLine(syncManager.RemoteVault, $"Pruning {files.Count:N0} files...", ConsoleColor.DarkGray); - int prunedFiles = await syncManager.PruneFilesAsync(files, new ProgressReport(syncManager.RemoteVault, files.Count)); + int prunedFiles = await syncManager.PruneFilesAsync(files, new ProgressReporter(syncManager.RemoteVault, files.Count)); CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pruned {prunedFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); await syncManager.DisconnectAsync(); } diff --git a/Parallel.Cli/Commands/RestoreCommand.cs b/Parallel.Cli/Commands/RestoreCommand.cs index 4eb9296..e5d31dd 100644 --- a/Parallel.Cli/Commands/RestoreCommand.cs +++ b/Parallel.Cli/Commands/RestoreCommand.cs @@ -136,7 +136,7 @@ private async Task RestoreInternalAsync(ISyncManager syncManager, string path, D else { CommandLine.WriteLine(syncManager.RemoteVault, $"Restoring {restoreFiles.Count:N0} files...", ConsoleColor.DarkGray); - IProgressReporter progressReporter = verbose ? new ProgressReport(syncManager.RemoteVault, restoreFiles.Count) : new NullProgressReporter(); + IProgressReporter progressReporter = verbose ? new ProgressReporter(syncManager.RemoteVault, restoreFiles.Count) : new LoggingProgressReporter(syncManager.RemoteVault); int restoredFiles = await syncManager.RestoreFilesAsync(restoreFiles.ToArray(), progressReporter); CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully restored {restoredFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); diff --git a/Parallel.Cli/Commands/ScrubCommand.cs b/Parallel.Cli/Commands/ScrubCommand.cs index 43d635f..1b846c6 100644 --- a/Parallel.Cli/Commands/ScrubCommand.cs +++ b/Parallel.Cli/Commands/ScrubCommand.cs @@ -107,7 +107,7 @@ private async Task ScrubInternalAsync(ISyncManager syncManager, string path, boo } CommandLine.WriteLine(syncManager.RemoteVault, $"Scrubbing {files.Count:N0} files...", ConsoleColor.DarkGray); - IProgressReporter progressReporter = verbose ? new ProgressReport(syncManager.RemoteVault, files.Count) : new NullProgressReporter(); + IProgressReporter progressReporter = verbose ? new ProgressReporter(syncManager.RemoteVault, files.Count) : new LoggingProgressReporter(syncManager.RemoteVault); int scrubbedFiles = await syncManager.ScrubFilesAsync(files, progressReporter); CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully scrubbed {scrubbedFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); diff --git a/Parallel.Cli/Commands/SnapshotsCommand.cs b/Parallel.Cli/Commands/SnapshotsCommand.cs index 720a2bb..23d2a00 100644 --- a/Parallel.Cli/Commands/SnapshotsCommand.cs +++ b/Parallel.Cli/Commands/SnapshotsCommand.cs @@ -4,6 +4,7 @@ using System.CommandLine; using System.Diagnostics; using Parallel.Cli.Utils; +using Parallel.Core.Diagnostics; using Parallel.Core.IO; using Parallel.Core.IO.Syncing; using Parallel.Core.Models; diff --git a/Parallel.Cli/Commands/StatsCommand.cs b/Parallel.Cli/Commands/StatsCommand.cs index 42d5c6b..6ce8b6a 100644 --- a/Parallel.Cli/Commands/StatsCommand.cs +++ b/Parallel.Cli/Commands/StatsCommand.cs @@ -42,13 +42,12 @@ private async Task DisplayDiskInformationAsync(LocalVaultConfig vault) } IDatabase? db = syncManager.Database; - long localSize = await (db?.GetLocalSizeAsync() ?? Task.FromResult(0L)); - long remoteSize = await (db?.GetRemoteSizeAsync() ?? Task.FromResult(0L)); + long localSize = await (db?.GetCurrentSizeAsync() ?? Task.FromResult(0L)); long totalSize = await (db?.GetTotalSizeAsync() ?? Task.FromResult(0L)); long totalFiles = await (db?.GetTotalFilesAsync() ?? Task.FromResult(0L)); long totalLocalFiles = await (db?.GetTotalFilesAsync(false) ?? Task.FromResult(0L)); long totalDeletedFiles = await (db?.GetTotalFilesAsync(true) ?? Task.FromResult(0L)); - double spaceSaved = Math.Round((localSize - remoteSize) / (double)localSize * 100, 2); + long totalRevisedFiles = await (db?.GetTotalRevisedFilesAsync() ?? Task.FromResult(0L)); CommandLine.WriteLine($"Using vault '{syncManager.RemoteVault.Name}' ({vault.Id}):"); CommandLine.WriteLine($"Service Type: {vault.Credentials.Service}"); @@ -56,10 +55,9 @@ private async Task DisplayDiskInformationAsync(LocalVaultConfig vault) CommandLine.WriteLine($"Managed Files: {totalFiles:N0}"); CommandLine.WriteLine($"Local Files: {totalLocalFiles:N0}"); CommandLine.WriteLine($"Deleted Files: {totalDeletedFiles:N0}"); - CommandLine.WriteLine($"Revisions: {totalFiles - (totalLocalFiles + totalDeletedFiles):N0}"); - CommandLine.WriteLine($"Total Size: {Formatter.FromBytes(localSize)}"); - CommandLine.WriteLine($"Local Size: {Formatter.FromBytes(totalSize)}"); - CommandLine.WriteLine($"Remote Size: {Formatter.FromBytes(remoteSize)} ({(double.IsNaN(spaceSaved) ? 0 : spaceSaved)}%)"); + CommandLine.WriteLine($"Revisions: {totalRevisedFiles:N0}"); + CommandLine.WriteLine($"Local Size: {Formatter.FromBytes(localSize)}"); + CommandLine.WriteLine($"Remote Size: {Formatter.FromBytes(totalSize)}"); if (vault.Credentials.Service.Equals(FileService.Local)) { diff --git a/Parallel.Cli/Commands/SyncCommand.cs b/Parallel.Cli/Commands/SyncCommand.cs index f5212ae..060cdad 100644 --- a/Parallel.Cli/Commands/SyncCommand.cs +++ b/Parallel.Cli/Commands/SyncCommand.cs @@ -132,7 +132,7 @@ private async Task SyncInternalAsync(ISyncManager syncManager, string path, bool if (!backupFolders.Any(dir => path.StartsWith(dir, StringComparison.OrdinalIgnoreCase)) || FileScanner.IsIgnored(path, ignoredFolders)) { - CommandLine.WriteLine(syncManager.RemoteVault, $"The provided {(isFile ? "file" : "folder")} is set to be ignored!", ConsoleColor.Yellow); + CommandLine.WriteLine(syncManager.RemoteVault, $"The provided {(isFile ? "file" : "folder")} is not set to be synced!", ConsoleColor.Yellow); return; } @@ -147,7 +147,7 @@ private async Task SyncInternalAsync(ISyncManager syncManager, string path, bool } CommandLine.WriteLine(syncManager.RemoteVault, $"Syncing {files.Length:N0} files...", ConsoleColor.DarkGray); - IProgressReporter progressReporter = verbose ? new ProgressReport(syncManager.RemoteVault, successFiles) : new NullProgressReporter(); + IProgressReporter progressReporter = verbose ? new ProgressReporter(syncManager.RemoteVault, successFiles) : new LoggingProgressReporter(syncManager.RemoteVault); int backedUpFiles = await syncManager.BackupFilesAsync(files, progressReporter, force); CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully synced {backedUpFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); diff --git a/Parallel.Cli/Parallel.Cli.csproj b/Parallel.Cli/Parallel.Cli.csproj index 01c1dfe..3f1c2a9 100644 --- a/Parallel.Cli/Parallel.Cli.csproj +++ b/Parallel.Cli/Parallel.Cli.csproj @@ -43,6 +43,7 @@ + diff --git a/Parallel.Cli/Program.cs b/Parallel.Cli/Program.cs index 4000ca6..7c37c0a 100644 --- a/Parallel.Cli/Program.cs +++ b/Parallel.Cli/Program.cs @@ -27,9 +27,9 @@ public static async Task Main(string[] args) Settings = ParallelConfig.Load(); AssemblyName assembly = Assembly.GetExecutingAssembly().GetName(); #if DEBUG - Log.Logger = new LoggerConfiguration().MinimumLevel.Debug().WriteTo.Sink(EventTracker).WriteTo.Console().CreateLogger(); + Log.Logger = new LoggerConfiguration().MinimumLevel.Debug().WriteTo.Console().CreateLogger(); #else - Log.Logger = new LoggerConfiguration().WriteTo.Sink(EventTracker).CreateLogger(); + Log.Logger = new LoggerConfiguration().WriteTo.File(PathBuilder.LogFile).CreateLogger(); #endif Log.Information($"{assembly.Name} [Version {assembly.Version}]"); @@ -51,13 +51,6 @@ public static async Task Main(string[] args) { // Clean successful logs await Log.CloseAndFlushAsync(); - if (EventTracker.ErrorCount > 0) - { - string logDir = Path.Combine(PathBuilder.TempDirectory, "Logs"); - if (!Directory.Exists(logDir)) Directory.CreateDirectory(logDir); - await File.WriteAllLinesAsync(Path.Combine(logDir, $"{DateTime.Now:MM-dd-yyyy hh-mm-ss}.log"), EventTracker.Logs.Reverse()); - } - Settings.Save(); } } diff --git a/Parallel.Cli/Utils/ProgressReport.cs b/Parallel.Cli/Utils/ProgressReporter.cs similarity index 82% rename from Parallel.Cli/Utils/ProgressReport.cs rename to Parallel.Cli/Utils/ProgressReporter.cs index ef3f20e..10e6e91 100644 --- a/Parallel.Cli/Utils/ProgressReport.cs +++ b/Parallel.Cli/Utils/ProgressReporter.cs @@ -7,14 +7,14 @@ namespace Parallel.Cli.Utils { - public class ProgressReport : IProgressReporter + public class ProgressReporter : IProgressReporter { private Stopwatch _sw; private readonly LocalVaultConfig _localVault; private int _current; private readonly int _total; - public ProgressReport(LocalVaultConfig localVault, int totalFiles) + public ProgressReporter(LocalVaultConfig localVault, int totalFiles) { _sw = Stopwatch.StartNew(); _localVault = localVault; @@ -25,8 +25,8 @@ public ProgressReport(LocalVaultConfig localVault, int totalFiles) public void Report(ProgressOperation operation, LocalFile file) { Interlocked.Increment(ref _current); - int percent = _current * 100 / _total; - CommandLine.WriteLine($"[{_localVault.Id}] ({percent}%) {operation}: {file.Fullname}"); + double percent = _current * 100.0 / _total; + CommandLine.WriteLine($"[{_localVault.Id}] ({percent:N1}%) {operation}: {file.Fullname}"); } /// diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 6d761ea..e2cf226 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -53,7 +53,7 @@ public async Task RemoveFileAsync(LocalFile file) } /// - public async Task GetLocalSizeAsync() + public async Task GetCurrentSizeAsync() { string sql = "SELECT COALESCE(SUM(f.localsize), 0) FROM objects f JOIN (SELECT fullname, MAX(lastupdate) AS max_lastupdate FROM objects WHERE deleted = 0 GROUP BY fullname) latest ON f.fullname = latest.fullname AND f.lastupdate = latest.max_lastupdate;"; return await _semaphore.QuerySingleAsync(sql); @@ -87,6 +87,12 @@ public async Task GetTotalFilesAsync(bool deleted) return await _semaphore.QuerySingleAsync(sql, new { deleted }); } + public async Task GetTotalRevisedFilesAsync() + { + string sql = $"SELECT COUNT(*) FROM objects WHERE lastupdate NOT IN (SELECT MAX(lastupdate) FROM objects GROUP BY fullname);"; + return await _semaphore.QuerySingleAsync(sql); + } + /// public async Task> GetLatestFilesAsync(string path, DateTime timestamp) { @@ -97,11 +103,16 @@ public async Task> GetLatestFilesAsync(string path, Dat /// public async Task> GetLatestFilesAsync(string path, DateTime timestamp, bool deleted) { - Log.Debug($"SELECT * FROM (SELECT * FROM objects WHERE fullname LIKE '{path}%' AND lastupdate <= {new UnixTime(timestamp).TotalMilliseconds} AND deleted = {deleted} ORDER BY lastwrite DESC) GROUP BY fullname;"); string sql = "SELECT * FROM (SELECT * FROM objects WHERE fullname LIKE @Path AND lastupdate <= @Time AND deleted = @deleted ORDER BY lastwrite DESC) GROUP BY fullname;"; return await _semaphore.QueryAsync(sql, new { Path = $"{path}%", Time = new UnixTime(timestamp).TotalMilliseconds, deleted }); } + public async Task> GetRevisedFilesAsync(string path) + { + string sql = $"SELECT * FROM objects WHERE fullname LIKE '{path}%' AND lastupdate NOT IN (SELECT MAX(lastupdate) FROM objects GROUP BY fullname) ORDER BY lastupdate DESC;"; + return await _semaphore.QueryAsync(sql, new { Path = $"{path}%" }); + } + /// public async Task> GetFilesAsync(string path, DateTime timestamp) { diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index 067aca5..ba716f0 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -111,11 +111,12 @@ public interface IDatabase /// Task GetFileAsync(string path); - Task GetLocalSizeAsync(); + Task GetCurrentSizeAsync(); Task GetRemoteSizeAsync(); Task GetTotalSizeAsync(); Task GetTotalFilesAsync(); Task GetTotalFilesAsync(bool deleted); + Task GetTotalRevisedFilesAsync(); /// /// Gets a list of directories in ascending order. diff --git a/Parallel.Core/Diagnostics/IProgressReporter.cs b/Parallel.Core/Diagnostics/IProgressReporter.cs index cd400f1..2baab53 100644 --- a/Parallel.Core/Diagnostics/IProgressReporter.cs +++ b/Parallel.Core/Diagnostics/IProgressReporter.cs @@ -23,11 +23,6 @@ public interface IProgressReporter /// void Report(ProgressOperation operation, LocalFile file); - /// - /// Resets the ticking. - /// - void Reset(); - /// /// Reports a failed update. /// diff --git a/Parallel.Core/Diagnostics/NullProgressReporter.cs b/Parallel.Core/Diagnostics/LoggingProgressReporter.cs similarity index 50% rename from Parallel.Core/Diagnostics/NullProgressReporter.cs rename to Parallel.Core/Diagnostics/LoggingProgressReporter.cs index 772656e..b261883 100644 --- a/Parallel.Core/Diagnostics/NullProgressReporter.cs +++ b/Parallel.Core/Diagnostics/LoggingProgressReporter.cs @@ -1,19 +1,28 @@ // Copyright 2026 Kyle Ebbinga +using System.Diagnostics; using Parallel.Core.Models; +using Parallel.Core.Settings; namespace Parallel.Core.Diagnostics { /// /// Represents a null . /// - public class NullProgressReporter : IProgressReporter + public class LoggingProgressReporter : IProgressReporter { + private readonly LocalVaultConfig _localVault; + + public LoggingProgressReporter(LocalVaultConfig localVault) + { + _localVault = localVault; + } + /// - public void Report(ProgressOperation operation, LocalFile file) { } - - /// - public void Reset() { } + public void Report(ProgressOperation operation, LocalFile file) + { + Log.Information($"[{_localVault.Id}] {operation}: {file.Fullname}"); + } /// public void Failed(LocalFile file, string message) diff --git a/Parallel.Core/IO/PathBuilder.cs b/Parallel.Core/IO/PathBuilder.cs index 4177106..159ae10 100644 --- a/Parallel.Core/IO/PathBuilder.cs +++ b/Parallel.Core/IO/PathBuilder.cs @@ -29,6 +29,18 @@ public static string TempDirectory public static string TempFile => Path.Combine(TempDirectory, UnixTime.Now.TotalMilliseconds + ".tmp"); + public static string LogDirectory + { + get + { + string logDir = Path.Combine(PathBuilder.TempDirectory, "Logs"); + if (!Directory.Exists(logDir)) Directory.CreateDirectory(logDir); + return logDir; + } + } + + public static string LogFile => Path.Combine(LogDirectory, UnixTime.Now.TotalMilliseconds + ".log"); + /// /// Gets the corresponding directory for program data based on the . /// diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index a6a23ec..b962154 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -1,5 +1,6 @@ // Copyright 2026 Kyle Ebbinga +using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; using Parallel.Core.Database; using Parallel.Core.Database.Contexts; @@ -48,6 +49,13 @@ protected BaseSyncManager(LocalVaultConfig localVault) /// public async Task ConnectAsync(bool force = false) { + if (!await StorageProvider.CheckConnectionAsync()) + { + Log.Error("[{LocalVaultId}] Failed to connect to vault!", LocalVault.Id); + return false; + } + + string root = PathBuilder.GetRootDirectory(LocalVault); if (!await StorageProvider.ExistsAsync(root)) { diff --git a/Parallel.Core/Storage/IStorageProvider.cs b/Parallel.Core/Storage/IStorageProvider.cs index 54f9124..69cfa75 100644 --- a/Parallel.Core/Storage/IStorageProvider.cs +++ b/Parallel.Core/Storage/IStorageProvider.cs @@ -10,6 +10,12 @@ namespace Parallel.Core.Storage /// public interface IStorageProvider : IDisposable { + /// + /// Checks the connection to the storage provider. + /// + /// + Task CheckConnectionAsync(); + /// /// Creates all directories and subdirectories in the specified path unless they already exist. /// diff --git a/Parallel.Core/Storage/LocalStorageProvider.cs b/Parallel.Core/Storage/LocalStorageProvider.cs index ffef43f..71280ea 100644 --- a/Parallel.Core/Storage/LocalStorageProvider.cs +++ b/Parallel.Core/Storage/LocalStorageProvider.cs @@ -33,6 +33,12 @@ public void Dispose() GC.SuppressFinalize(this); } + /// + public Task CheckConnectionAsync() + { + return Task.FromResult(Directory.Exists(_vaultConfig.Credentials.RootDirectory)); + } + /// public Task CreateDirectoryAsync(string path) { diff --git a/Parallel.Core/Storage/S3StorageProvider.cs b/Parallel.Core/Storage/S3StorageProvider.cs index ca7b02a..031458c 100644 --- a/Parallel.Core/Storage/S3StorageProvider.cs +++ b/Parallel.Core/Storage/S3StorageProvider.cs @@ -1,5 +1,6 @@ // Copyright 2026 Kyle Ebbinga +using System.Diagnostics; using System.IO.Compression; using System.IO.Pipelines; using Amazon; @@ -7,6 +8,7 @@ using Amazon.S3; using Amazon.S3.Model; using Amazon.S3.Transfer; +using Parallel.Core.Diagnostics; using Parallel.Core.Models; using Parallel.Core.Security; using Parallel.Core.Settings; @@ -49,6 +51,19 @@ public void Dispose() GC.SuppressFinalize(this); } + public async Task CheckConnectionAsync() + { + try + { + ListBucketsResponse? response = await _client.ListBucketsAsync(); + return response.HttpStatusCode == System.Net.HttpStatusCode.OK; + } + catch + { + return false; + } + } + public async Task CreateDirectoryAsync(string path) { if (!path.EndsWith("/")) path += "/"; @@ -122,6 +137,7 @@ public Task GetDirectoryName(string path) public async Task UploadFileAsync(LocalFile file, string remotePath, bool overwrite = false, CancellationToken ct = default) { + Stopwatch sw = Stopwatch.StartNew(); if (!overwrite && await ExistsAsync(remotePath)) { Log.Debug("Skipping file: {RemotePath}", remotePath); diff --git a/Parallel.Core/Storage/SshStorageProvider.cs b/Parallel.Core/Storage/SshStorageProvider.cs index 6ac4273..1e5ff6c 100644 --- a/Parallel.Core/Storage/SshStorageProvider.cs +++ b/Parallel.Core/Storage/SshStorageProvider.cs @@ -48,6 +48,12 @@ public void Dispose() GC.SuppressFinalize(this); } + public Task CheckConnectionAsync() + { + InsureConnection(); + return Task.FromResult(_client.IsConnected); + } + /// public async Task CreateDirectoryAsync(string path) {