diff --git a/Parallel.Cli/Commands/PruneCommand.cs b/Parallel.Cli/Commands/PruneCommand.cs index 6c67e0d..a2ac88a 100644 --- a/Parallel.Cli/Commands/PruneCommand.cs +++ b/Parallel.Cli/Commands/PruneCommand.cs @@ -3,6 +3,7 @@ using System.CommandLine; using System.Diagnostics; using Parallel.Cli.Utils; +using Parallel.Core.IO; using Parallel.Core.IO.Syncing; using Parallel.Core.Models; using Parallel.Core.Settings; @@ -14,11 +15,17 @@ public class PruneCommand : Command { private Stopwatch _sw = new Stopwatch(); - private readonly Option _sourceOpt = new(["--path", "-p"], "The source path to clean."); + private readonly Argument _sourceArg = new("path", "The path to add or remove."); + private readonly Option _sourceOpt = new(["--path", "-p"], "The source path to prune."); private readonly Option _configOpt = new(["--config", "-c"], "The vault configuration to use."); - private readonly Option _beforeOpt = new(["--before"], "Prune files before a certain timestamp."); - private readonly Option _daysOpt = new(["--days", "-d"], "The amount of days to hang onto files."); + private readonly Option _beforeOpt = new(["--before"], "Specified timestamp as the pruning reference point."); + private readonly Option _daysOpt = new(["--days", "-d"], "Specified number of days before the reference point."); private readonly Option _forceOpt = new(["--force", "-f"], "Forces pruning, bypassing safe guards."); + private readonly Option _dryRunOpt = new(["--dry-run"], "Previews the command without executing it."); + + private readonly Command addCmd = new("add", "Adds a new path to the prune list."); + private readonly Command listCmd = new("list", "Shows all directories in the prune list."); + private readonly Command removeCmd = new("remove", "Removes a directory from the prune list."); public PruneCommand() : base("prune", "Prunes the oldest files from vaults.") { @@ -27,10 +34,23 @@ public PruneCommand() : base("prune", "Prunes the oldest files from vaults.") this.AddOption(_beforeOpt); this.AddOption(_daysOpt); this.AddOption(_forceOpt); - this.SetHandler(HandlePruneAsync, _sourceOpt, _configOpt, _beforeOpt, _daysOpt, _forceOpt); + this.AddOption(_dryRunOpt); + this.SetHandler(HandlePruneAsync, _sourceOpt, _configOpt, _beforeOpt, _daysOpt, _forceOpt, _dryRunOpt); + + this.AddCommand(addCmd); + addCmd.AddArgument(_sourceArg); + addCmd.AddOption(_configOpt); + addCmd.SetHandler(HandleAddAsync, _sourceArg, _configOpt); + + this.AddCommand(removeCmd); + removeCmd.AddArgument(_sourceArg); + removeCmd.AddOption(_configOpt); + removeCmd.SetHandler(HandleRemoveAsync, _sourceArg, _configOpt); } - private async Task HandlePruneAsync(string? path, string? config, DateTime before, int days, bool force) + #region Pruning + + private async Task HandlePruneAsync(string? path, string? config, DateTime before, int days, bool force, bool dryRun) { _sw = Stopwatch.StartNew(); LocalVaultConfig? localVault = ParallelConfig.GetVault(config); @@ -38,37 +58,33 @@ private async Task HandlePruneAsync(string? path, string? config, DateTime befor { if (!string.IsNullOrEmpty(path)) { - await PrunePathAsync(localVault, path, before, days, force); + await PrunePathAsync(localVault, path, before, days, force, dryRun); } else { - await PruneSystemAsync(localVault, before, days, force); + await PruneSystemAsync(localVault, before, days, force, dryRun); } } else { if (!string.IsNullOrEmpty(path)) { - await Program.Settings.ForEachVaultAsync(vault => PrunePathAsync(vault, path, before, days, force)); + await Program.Settings.ForEachVaultAsync(vault => PrunePathAsync(vault, path, before, days, force, dryRun)); } else { - await Program.Settings.ForEachVaultAsync(vault => PruneSystemAsync(vault, before, days, force)); + await Program.Settings.ForEachVaultAsync(vault => PruneSystemAsync(vault, before, days, force, dryRun)); } } } private DateTime GetPruneDateTime(int prunePeriod, DateTime before, int days) { - if (before > DateTime.MinValue) - { - return before.AddTicks(-1); - } - - return days > 0 ? DateTime.Now.AddDays(-days) : DateTime.Now.AddDays(-prunePeriod); + DateTime timestamp = before > DateTime.MinValue ? before : DateTime.Now; + return days > 0 ? timestamp.AddDays(-days) : timestamp.AddDays(-prunePeriod); } - private async Task PruneSystemAsync(LocalVaultConfig vault, DateTime before, int days, bool force) + private async Task PruneSystemAsync(LocalVaultConfig vault, DateTime before, int days, bool force, bool dryRun) { ISyncManager? syncManager = SyncManager.CreateNew(vault); if (syncManager == null || !await syncManager.ConnectAsync()) @@ -77,21 +93,21 @@ private async Task PruneSystemAsync(LocalVaultConfig vault, DateTime before, int return; } - try + DateTime timestamp = GetPruneDateTime(syncManager.RemoteVault.PrunePeriod, before, days); + HashSet directories = syncManager.RemoteVault.PruneDirectories; + if (directories.Count == 0) { - DateTime timestamp = GetPruneDateTime(syncManager.RemoteVault.PrunePeriod, before, days); - foreach (string path in syncManager.RemoteVault.PruneDirectories) - { - await PruneInternalAsync(syncManager, path, timestamp, force); - } + CommandLine.WriteLine($"No prunable directories have been set!", ConsoleColor.Yellow); + return; } - finally + + foreach (string path in directories) { - await syncManager.DisconnectAsync(); + await PruneInternalAsync(syncManager, path, timestamp, force, dryRun); } } - private async Task PrunePathAsync(LocalVaultConfig vault, string path, DateTime before, int days, bool force) + private async Task PrunePathAsync(LocalVaultConfig vault, string path, DateTime before, int days, bool force, bool dryRun) { ISyncManager? syncManager = SyncManager.CreateNew(vault); if (syncManager == null || !await syncManager.ConnectAsync()) @@ -100,18 +116,13 @@ private async Task PrunePathAsync(LocalVaultConfig vault, string path, DateTime return; } - try - { - DateTime timestamp = GetPruneDateTime(syncManager.RemoteVault.PrunePeriod, before, days); - await PruneInternalAsync(syncManager, path, timestamp, force); - } - finally - { - await syncManager.DisconnectAsync(); - } + DateTime timestamp = GetPruneDateTime(syncManager.RemoteVault.PrunePeriod, before, days); + Log.Debug($"Pruning files before {timestamp}"); + + await PruneInternalAsync(syncManager, path, timestamp, force, dryRun); } - private async Task PruneInternalAsync(ISyncManager syncManager, string path, DateTime timestamp, bool force) + private async Task PruneInternalAsync(ISyncManager syncManager, string path, DateTime timestamp, bool force, bool dryRun) { CommandLine.WriteLine(syncManager.RemoteVault, $"Scanning for files in {path}...", ConsoleColor.DarkGray); IReadOnlyList files = await (syncManager.Database?.GetFilesAsync(path, timestamp, true) ?? Task.FromResult>([])); @@ -122,7 +133,7 @@ private async Task PruneInternalAsync(ISyncManager syncManager, string path, Dat return; } - if (!force) + if (!force && !dryRun) { CommandLine.WriteLine($"This will permanently delete {files.Count:N0} files!", ConsoleColor.Yellow); if (!CommandLine.ReadBool("Do you wish to continue? [yes/no]", false)) @@ -132,9 +143,94 @@ private async Task PruneInternalAsync(ISyncManager syncManager, string path, Dat } } - CommandLine.WriteLine(syncManager.RemoteVault, $"Pruning {files.Count:N0} files...", ConsoleColor.DarkGray); - int prunedFiles = await syncManager.PruneFilesAsync(files, new ProgressReport(syncManager.RemoteVault, files.Count)); - CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pruned {prunedFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); + if (dryRun) + { + string fileName = PathBuilder.TempFile; + await File.WriteAllLinesAsync(fileName, files.Select(f => f.Fullname).OrderBy(f => f)); + CommandLine.WriteLine($"This operation will prune {files.Count:N0} files ({Formatter.FromBytes(files.Sum(f => f.LocalSize))})", ConsoleColor.Green); + CommandLine.WriteLine($"A detailed list can be found here: {fileName}", ConsoleColor.DarkGray); + } + else + { + CommandLine.WriteLine(syncManager.RemoteVault, $"Pruning {files.Count:N0} files...", ConsoleColor.DarkGray); + int prunedFiles = await syncManager.PruneFilesAsync(files, new ProgressReport(syncManager.RemoteVault, files.Count)); + CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pruned {prunedFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); + await syncManager.DisconnectAsync(); + } + } + + #endregion + + #region Add + + private async Task HandleAddAsync(string path, string? config) + { + LocalVaultConfig? localVault = string.IsNullOrEmpty(config) ? ParallelConfig.Load().Vaults.FirstOrDefault(v => v.Enabled) : ParallelConfig.GetVault(config); + if (!string.IsNullOrEmpty(config) && localVault != null) + { + await AddPathAsync(localVault, path); + } + else + { + await Program.Settings.ForEachVaultAsync(vault => AddPathAsync(vault, path)); + } } + + private async Task AddPathAsync(LocalVaultConfig vault, string path) + { + ISyncManager? syncManager = SyncManager.CreateNew(vault); + if (syncManager == null || !await syncManager.ConnectAsync()) + { + CommandLine.WriteLine(vault, "Failed to connect to vault!", ConsoleColor.Red); + return; + } + + if (!syncManager.RemoteVault.PruneDirectories.Add(path)) + { + CommandLine.WriteLine(vault, $"Unable to add path: '{path}'", ConsoleColor.Yellow); + return; + } + + CommandLine.WriteLine(vault, $"Successfully added '{path}'", ConsoleColor.Green); + await syncManager.DisconnectAsync(); + } + + #endregion + + #region Remove + + private async Task HandleRemoveAsync(string path, string? config) + { + LocalVaultConfig? localVault = string.IsNullOrEmpty(config) ? ParallelConfig.Load().Vaults.FirstOrDefault(v => v.Enabled) : ParallelConfig.GetVault(config); + if (!string.IsNullOrEmpty(config) && localVault != null) + { + await RemovePathAsync(localVault, path); + } + else + { + await Program.Settings.ForEachVaultAsync(vault => RemovePathAsync(vault, path)); + } + } + + private async Task RemovePathAsync(LocalVaultConfig vault, string path) + { + ISyncManager? syncManager = SyncManager.CreateNew(vault); + if (syncManager == null || !await syncManager.ConnectAsync()) + { + CommandLine.WriteLine(vault, "Failed to connect to vault!", ConsoleColor.Red); + return; + } + + if (!syncManager.RemoteVault.PruneDirectories.Remove(path)) + { + CommandLine.WriteLine(vault, $"Unable to remove path: '{path}'", ConsoleColor.Yellow); + return; + } + + CommandLine.WriteLine(vault, $"Successfully removed '{path}'", ConsoleColor.Green); + await syncManager.DisconnectAsync(); + } + + #endregion } } \ No newline at end of file diff --git a/Parallel.Cli/Commands/RestoreCommand.cs b/Parallel.Cli/Commands/RestoreCommand.cs index ff3b899..3b55ce0 100644 --- a/Parallel.Cli/Commands/RestoreCommand.cs +++ b/Parallel.Cli/Commands/RestoreCommand.cs @@ -22,9 +22,9 @@ public class RestoreCommand : Command private readonly Option _sourceOpt = new(["--path", "-p"], "The source path to restore."); private readonly Option _configOpt = new(["--config", "-c"], "The vault configuration to use."); private readonly Option _beforeOpt = new(["--before"], "Restores files before a certain timestamp."); - private readonly Option _remapOpt = new(["--remap"], "The output directory remapping."); - private readonly Option _forceOpt = new(["--force", "-f"], "Forces overwriting any files."); - private readonly Option _dryRunOpt = new(["--dry-run"], "Previews a command without executing it."); + private readonly Option _remapOpt = new(["--remap"], "The new directory to map restored files to."); + private readonly Option _forceOpt = new(["--force", "-f"], "Forces restoring, bypassing safe guards."); + private readonly Option _dryRunOpt = new(["--dry-run"], "Previews the command without executing it."); public RestoreCommand() : base("restore", "Restores files from the backup.") { diff --git a/Parallel.Cli/Commands/CheckCommand.cs b/Parallel.Cli/Commands/ValidateCommand.cs similarity index 96% rename from Parallel.Cli/Commands/CheckCommand.cs rename to Parallel.Cli/Commands/ValidateCommand.cs index dfacd93..54915b1 100644 --- a/Parallel.Cli/Commands/CheckCommand.cs +++ b/Parallel.Cli/Commands/ValidateCommand.cs @@ -11,7 +11,7 @@ namespace Parallel.Cli.Commands { - public class CheckCommand : Command + public class ValidateCommand : Command { private Stopwatch _sw = new Stopwatch(); @@ -19,7 +19,7 @@ public class CheckCommand : Command private readonly Option _sourceOpt = new(["--path", "-p"], "The source path to back up."); private readonly Option _configOpt = new(["--config", "-c"], "The vault configuration to use."); - public CheckCommand() : base("check", "Syncs the system with the vaults.") + public ValidateCommand() : base("check", "Syncs the system with the vaults.") { this.AddOption(_sourceOpt); this.AddOption(_configOpt); diff --git a/Parallel.Cli/Program.cs b/Parallel.Cli/Program.cs index f2314c9..0da2301 100644 --- a/Parallel.Cli/Program.cs +++ b/Parallel.Cli/Program.cs @@ -36,7 +36,7 @@ public static async Task Main(string[] args) RootCommand rootCommand = new("Parallel file manager - Easily back up and synchronize massive amounts of files, save system states, and free up drive space."); IEnumerable types = Assembly.GetExecutingAssembly().GetTypes().Where(t => typeof(Command).IsAssignableFrom(t) && !t.IsAbstract); foreach (Type type in types) rootCommand.AddCommand((Command)Activator.CreateInstance(type)!); - //await File.WriteAllTextAsync(Path.Combine(PathBuilder.TempDirectory, "Command.md"), MarkdownGenerator.Generate(rootCommand)); + await File.WriteAllTextAsync(Path.Combine(PathBuilder.TempDirectory, "Command.md"), MarkdownGenerator.Generate(rootCommand)); try { diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 2b0504f..25f339f 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -46,10 +46,10 @@ public async Task AddFileAsync(LocalFile file) } /// - public async Task RemoveFileAsync(LocalFile file) + public async Task RemoveFileAsync(LocalFile file) { - string sql = $"DELETE FROM objects WHERE fullname = @Fullname AND checksum = @LocalCheckSum;"; - await _semaphore.ExecuteAsync(sql, new { file.Fullname, file.LocalCheckSum }); + string sql = $"DELETE FROM objects WHERE fullname = @Fullname AND localCheckSum = @LocalCheckSum;"; + return await _semaphore.ExecuteAsync(sql, new { file.Fullname, file.LocalCheckSum }) > 0; } /// diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index a36620b..c291eb1 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -67,7 +67,7 @@ public interface IDatabase /// True if successful, false otherwise Task AddFileAsync(LocalFile file); - Task RemoveFileAsync(LocalFile file); + Task RemoveFileAsync(LocalFile file); /// /// Gets a list of files by newest revision. diff --git a/Parallel.Core/IO/PathBuilder.cs b/Parallel.Core/IO/PathBuilder.cs index 84cebd9..3406b7d 100644 --- a/Parallel.Core/IO/PathBuilder.cs +++ b/Parallel.Core/IO/PathBuilder.cs @@ -27,7 +27,7 @@ public static string TempDirectory } } - public static string TempFile => Path.Combine(TempDirectory, DateTime.Now.ToString("MM-dd-yyyy hh-mm-ss") + ".tmp"); + public static string TempFile => Path.Combine(TempDirectory, UnixTime.Now.TotalMilliseconds + ".tmp"); /// /// Gets the corresponding directory for program data based on the . diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index 0ff6f7d..3d38fab 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -149,9 +149,9 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options { try { - await (Database != null ? Database.RemoveFileAsync(file) : Task.CompletedTask); + if (!await (Database?.RemoveFileAsync(file) ?? Task.FromResult(false))) return; if (!await (Database?.AddHistoryAsync(HistoryType.Pruned, file) ?? Task.FromResult(false))) Log.Error("Failed to add history: {Fullname}", file.Fullname); - await StorageProvider.DeleteFileAsync(PathBuilder.GetObjectPath(RemoteVault, file.LocalCheckSum!)); + await StorageProvider.DeleteFileAsync(PathBuilder.GetObjectPath(RemoteVault, file.RemoteCheckSum!)); progress.Report(ProgressOperation.Pruned, file); } catch (Exception ex) diff --git a/Parallel.Core/Settings/RemoteVaultConfig.cs b/Parallel.Core/Settings/RemoteVaultConfig.cs index ec032b6..2566100 100644 --- a/Parallel.Core/Settings/RemoteVaultConfig.cs +++ b/Parallel.Core/Settings/RemoteVaultConfig.cs @@ -42,7 +42,7 @@ public class RemoteVaultConfig : LocalVaultConfig public HashSet IgnoreDirectories { get; } = CreateIgnoreDirectories(); /// - /// A collection of deleted directories allowed to be pruned. + /// A collection of directories allowed to be pruned. /// Recommended when using a cloud-based to save on storage costs. /// Default: Empty /// @@ -82,6 +82,7 @@ private static HashSet CreateIgnoreDirectories() { list.Add("$RECYCLE.BIN/"); // For NTFS file systems list.Add("*.lnk"); // Shortcuts to other paths + list.Add("desktop.ini"); } // Ignore folders on Linux machines diff --git a/Parallel.Service/Tasks/TaskQueuer.cs b/Parallel.Service/Tasks/TaskQueuer.cs index 2fbfff5..913b7fd 100644 --- a/Parallel.Service/Tasks/TaskQueuer.cs +++ b/Parallel.Service/Tasks/TaskQueuer.cs @@ -70,10 +70,12 @@ public async Task> WaitAsync(CancellationToken ct) while (true) { if (!_queue.TryDequeue(out QueuedTask? candidate)) continue; + if (_tasks.IsEmpty) continue; + QueuedTask next = _tasks.Values.OrderByDescending(t => t.Priority + (DateTime.UtcNow - t.EnqueuedAt).TotalSeconds * 0.1).First(); if (ReferenceEquals(candidate, next)) { - _logger.LogDebug($"Starting task with key: '{candidate.Key}', remaining: {_queue.Count - 1}"); + _logger.LogDebug($"Starting task with key: '{candidate.Key}', remaining: {_queue.Count}"); _tasks.TryRemove(candidate.Key, out _); return candidate.TaskFunc; }