Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 135 additions & 39 deletions Parallel.Cli/Commands/PruneCommand.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.CommandLine;
using System.Diagnostics;
using Parallel.Cli.Utils;
using Parallel.Core.IO;
using Parallel.Core.IO.Syncing;
using Parallel.Core.Models;
using Parallel.Core.Settings;
Expand All @@ -14,11 +15,17 @@ public class PruneCommand : Command
{
private Stopwatch _sw = new Stopwatch();

private readonly Option<string> _sourceOpt = new(["--path", "-p"], "The source path to clean.");
private readonly Argument<string> _sourceArg = new("path", "The path to add or remove.");
private readonly Option<string> _sourceOpt = new(["--path", "-p"], "The source path to prune.");
private readonly Option<string> _configOpt = new(["--config", "-c"], "The vault configuration to use.");
private readonly Option<DateTime> _beforeOpt = new(["--before"], "Prune files before a certain timestamp.");
private readonly Option<int> _daysOpt = new(["--days", "-d"], "The amount of days to hang onto files.");
private readonly Option<DateTime> _beforeOpt = new(["--before"], "Specified timestamp as the pruning reference point.");
private readonly Option<int> _daysOpt = new(["--days", "-d"], "Specified number of days before the reference point.");
private readonly Option<bool> _forceOpt = new(["--force", "-f"], "Forces pruning, bypassing safe guards.");
private readonly Option<bool> _dryRunOpt = new(["--dry-run"], "Previews the command without executing it.");

private readonly Command addCmd = new("add", "Adds a new path to the prune list.");
private readonly Command listCmd = new("list", "Shows all directories in the prune list.");
private readonly Command removeCmd = new("remove", "Removes a directory from the prune list.");

public PruneCommand() : base("prune", "Prunes the oldest files from vaults.")
{
Expand All @@ -27,48 +34,57 @@ public PruneCommand() : base("prune", "Prunes the oldest files from vaults.")
this.AddOption(_beforeOpt);
this.AddOption(_daysOpt);
this.AddOption(_forceOpt);
this.SetHandler(HandlePruneAsync, _sourceOpt, _configOpt, _beforeOpt, _daysOpt, _forceOpt);
this.AddOption(_dryRunOpt);
this.SetHandler(HandlePruneAsync, _sourceOpt, _configOpt, _beforeOpt, _daysOpt, _forceOpt, _dryRunOpt);

this.AddCommand(addCmd);
addCmd.AddArgument(_sourceArg);
addCmd.AddOption(_configOpt);
addCmd.SetHandler(HandleAddAsync, _sourceArg, _configOpt);

this.AddCommand(removeCmd);
removeCmd.AddArgument(_sourceArg);
removeCmd.AddOption(_configOpt);
removeCmd.SetHandler(HandleRemoveAsync, _sourceArg, _configOpt);
}

private async Task HandlePruneAsync(string? path, string? config, DateTime before, int days, bool force)
#region Pruning

private async Task HandlePruneAsync(string? path, string? config, DateTime before, int days, bool force, bool dryRun)
{
_sw = Stopwatch.StartNew();
LocalVaultConfig? localVault = ParallelConfig.GetVault(config);
if (localVault != null)
{
if (!string.IsNullOrEmpty(path))
{
await PrunePathAsync(localVault, path, before, days, force);
await PrunePathAsync(localVault, path, before, days, force, dryRun);
}
else
{
await PruneSystemAsync(localVault, before, days, force);
await PruneSystemAsync(localVault, before, days, force, dryRun);
}
}
else
{
if (!string.IsNullOrEmpty(path))
{
await Program.Settings.ForEachVaultAsync(vault => PrunePathAsync(vault, path, before, days, force));
await Program.Settings.ForEachVaultAsync(vault => PrunePathAsync(vault, path, before, days, force, dryRun));
}
else
{
await Program.Settings.ForEachVaultAsync(vault => PruneSystemAsync(vault, before, days, force));
await Program.Settings.ForEachVaultAsync(vault => PruneSystemAsync(vault, before, days, force, dryRun));
}
}
}

private DateTime GetPruneDateTime(int prunePeriod, DateTime before, int days)
{
if (before > DateTime.MinValue)
{
return before.AddTicks(-1);
}

return days > 0 ? DateTime.Now.AddDays(-days) : DateTime.Now.AddDays(-prunePeriod);
DateTime timestamp = before > DateTime.MinValue ? before : DateTime.Now;
return days > 0 ? timestamp.AddDays(-days) : timestamp.AddDays(-prunePeriod);
}

private async Task PruneSystemAsync(LocalVaultConfig vault, DateTime before, int days, bool force)
private async Task PruneSystemAsync(LocalVaultConfig vault, DateTime before, int days, bool force, bool dryRun)
{
ISyncManager? syncManager = SyncManager.CreateNew(vault);
if (syncManager == null || !await syncManager.ConnectAsync())
Expand All @@ -77,21 +93,21 @@ private async Task PruneSystemAsync(LocalVaultConfig vault, DateTime before, int
return;
}

try
DateTime timestamp = GetPruneDateTime(syncManager.RemoteVault.PrunePeriod, before, days);
HashSet<string> directories = syncManager.RemoteVault.PruneDirectories;
if (directories.Count == 0)
{
DateTime timestamp = GetPruneDateTime(syncManager.RemoteVault.PrunePeriod, before, days);
foreach (string path in syncManager.RemoteVault.PruneDirectories)
{
await PruneInternalAsync(syncManager, path, timestamp, force);
}
CommandLine.WriteLine($"No prunable directories have been set!", ConsoleColor.Yellow);
return;
}
finally

foreach (string path in directories)
{
await syncManager.DisconnectAsync();
await PruneInternalAsync(syncManager, path, timestamp, force, dryRun);
}
}

private async Task PrunePathAsync(LocalVaultConfig vault, string path, DateTime before, int days, bool force)
private async Task PrunePathAsync(LocalVaultConfig vault, string path, DateTime before, int days, bool force, bool dryRun)
{
ISyncManager? syncManager = SyncManager.CreateNew(vault);
if (syncManager == null || !await syncManager.ConnectAsync())
Expand All @@ -100,18 +116,13 @@ private async Task PrunePathAsync(LocalVaultConfig vault, string path, DateTime
return;
}

try
{
DateTime timestamp = GetPruneDateTime(syncManager.RemoteVault.PrunePeriod, before, days);
await PruneInternalAsync(syncManager, path, timestamp, force);
}
finally
{
await syncManager.DisconnectAsync();
}
DateTime timestamp = GetPruneDateTime(syncManager.RemoteVault.PrunePeriod, before, days);
Log.Debug($"Pruning files before {timestamp}");

await PruneInternalAsync(syncManager, path, timestamp, force, dryRun);
}

private async Task PruneInternalAsync(ISyncManager syncManager, string path, DateTime timestamp, bool force)
private async Task PruneInternalAsync(ISyncManager syncManager, string path, DateTime timestamp, bool force, bool dryRun)
{
CommandLine.WriteLine(syncManager.RemoteVault, $"Scanning for files in {path}...", ConsoleColor.DarkGray);
IReadOnlyList<LocalFile> files = await (syncManager.Database?.GetFilesAsync(path, timestamp, true) ?? Task.FromResult<IReadOnlyList<LocalFile>>([]));
Expand All @@ -122,7 +133,7 @@ private async Task PruneInternalAsync(ISyncManager syncManager, string path, Dat
return;
}

if (!force)
if (!force && !dryRun)
{
CommandLine.WriteLine($"This will permanently delete {files.Count:N0} files!", ConsoleColor.Yellow);
if (!CommandLine.ReadBool("Do you wish to continue? [yes/no]", false))
Expand All @@ -132,9 +143,94 @@ private async Task PruneInternalAsync(ISyncManager syncManager, string path, Dat
}
}

CommandLine.WriteLine(syncManager.RemoteVault, $"Pruning {files.Count:N0} files...", ConsoleColor.DarkGray);
int prunedFiles = await syncManager.PruneFilesAsync(files, new ProgressReport(syncManager.RemoteVault, files.Count));
CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pruned {prunedFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green);
if (dryRun)
{
string fileName = PathBuilder.TempFile;
await File.WriteAllLinesAsync(fileName, files.Select(f => f.Fullname).OrderBy(f => f));
CommandLine.WriteLine($"This operation will prune {files.Count:N0} files ({Formatter.FromBytes(files.Sum(f => f.LocalSize))})", ConsoleColor.Green);
CommandLine.WriteLine($"A detailed list can be found here: {fileName}", ConsoleColor.DarkGray);
}
else
{
CommandLine.WriteLine(syncManager.RemoteVault, $"Pruning {files.Count:N0} files...", ConsoleColor.DarkGray);
int prunedFiles = await syncManager.PruneFilesAsync(files, new ProgressReport(syncManager.RemoteVault, files.Count));
CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pruned {prunedFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green);
await syncManager.DisconnectAsync();
}
}

#endregion

#region Add

private async Task HandleAddAsync(string path, string? config)
{
LocalVaultConfig? localVault = string.IsNullOrEmpty(config) ? ParallelConfig.Load().Vaults.FirstOrDefault(v => v.Enabled) : ParallelConfig.GetVault(config);
if (!string.IsNullOrEmpty(config) && localVault != null)
{
await AddPathAsync(localVault, path);
}
else
{
await Program.Settings.ForEachVaultAsync(vault => AddPathAsync(vault, path));
}
}

private async Task AddPathAsync(LocalVaultConfig vault, string path)
{
ISyncManager? syncManager = SyncManager.CreateNew(vault);
if (syncManager == null || !await syncManager.ConnectAsync())
{
CommandLine.WriteLine(vault, "Failed to connect to vault!", ConsoleColor.Red);
return;
}

if (!syncManager.RemoteVault.PruneDirectories.Add(path))
{
CommandLine.WriteLine(vault, $"Unable to add path: '{path}'", ConsoleColor.Yellow);
return;
}

CommandLine.WriteLine(vault, $"Successfully added '{path}'", ConsoleColor.Green);
await syncManager.DisconnectAsync();
}

#endregion

#region Remove

private async Task HandleRemoveAsync(string path, string? config)
{
LocalVaultConfig? localVault = string.IsNullOrEmpty(config) ? ParallelConfig.Load().Vaults.FirstOrDefault(v => v.Enabled) : ParallelConfig.GetVault(config);
if (!string.IsNullOrEmpty(config) && localVault != null)
{
await RemovePathAsync(localVault, path);
}
else
{
await Program.Settings.ForEachVaultAsync(vault => RemovePathAsync(vault, path));
}
}

private async Task RemovePathAsync(LocalVaultConfig vault, string path)
{
ISyncManager? syncManager = SyncManager.CreateNew(vault);
if (syncManager == null || !await syncManager.ConnectAsync())
{
CommandLine.WriteLine(vault, "Failed to connect to vault!", ConsoleColor.Red);
return;
}

if (!syncManager.RemoteVault.PruneDirectories.Remove(path))
{
CommandLine.WriteLine(vault, $"Unable to remove path: '{path}'", ConsoleColor.Yellow);
return;
}

CommandLine.WriteLine(vault, $"Successfully removed '{path}'", ConsoleColor.Green);
await syncManager.DisconnectAsync();
}

#endregion
}
}
6 changes: 3 additions & 3 deletions Parallel.Cli/Commands/RestoreCommand.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ public class RestoreCommand : Command
private readonly Option<string> _sourceOpt = new(["--path", "-p"], "The source path to restore.");
private readonly Option<string> _configOpt = new(["--config", "-c"], "The vault configuration to use.");
private readonly Option<DateTime> _beforeOpt = new(["--before"], "Restores files before a certain timestamp.");
private readonly Option<string> _remapOpt = new(["--remap"], "The output directory remapping.");
private readonly Option<bool> _forceOpt = new(["--force", "-f"], "Forces overwriting any files.");
private readonly Option<bool> _dryRunOpt = new(["--dry-run"], "Previews a command without executing it.");
private readonly Option<string> _remapOpt = new(["--remap"], "The new directory to map restored files to.");
private readonly Option<bool> _forceOpt = new(["--force", "-f"], "Forces restoring, bypassing safe guards.");
private readonly Option<bool> _dryRunOpt = new(["--dry-run"], "Previews the command without executing it.");

public RestoreCommand() : base("restore", "Restores files from the backup.")
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@

namespace Parallel.Cli.Commands
{
public class CheckCommand : Command
public class ValidateCommand : Command
{
private Stopwatch _sw = new Stopwatch();

private readonly Argument<string> _sourceArg = new("path", "The path to add or remove.");
private readonly Option<string> _sourceOpt = new(["--path", "-p"], "The source path to back up.");
private readonly Option<string> _configOpt = new(["--config", "-c"], "The vault configuration to use.");

public CheckCommand() : base("check", "Syncs the system with the vaults.")
public ValidateCommand() : base("check", "Syncs the system with the vaults.")
{
this.AddOption(_sourceOpt);
this.AddOption(_configOpt);
Expand Down
2 changes: 1 addition & 1 deletion Parallel.Cli/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public static async Task Main(string[] args)
RootCommand rootCommand = new("Parallel file manager - Easily back up and synchronize massive amounts of files, save system states, and free up drive space.");
IEnumerable<Type> types = Assembly.GetExecutingAssembly().GetTypes().Where(t => typeof(Command).IsAssignableFrom(t) && !t.IsAbstract);
foreach (Type type in types) rootCommand.AddCommand((Command)Activator.CreateInstance(type)!);
//await File.WriteAllTextAsync(Path.Combine(PathBuilder.TempDirectory, "Command.md"), MarkdownGenerator.Generate(rootCommand));
await File.WriteAllTextAsync(Path.Combine(PathBuilder.TempDirectory, "Command.md"), MarkdownGenerator.Generate(rootCommand));

try
{
Expand Down
6 changes: 3 additions & 3 deletions Parallel.Core/Database/Contexts/SqliteContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,10 @@ public async Task<bool> AddFileAsync(LocalFile file)
}

/// <inheritdoc />
public async Task RemoveFileAsync(LocalFile file)
public async Task<bool> RemoveFileAsync(LocalFile file)
{
string sql = $"DELETE FROM objects WHERE fullname = @Fullname AND checksum = @LocalCheckSum;";
await _semaphore.ExecuteAsync(sql, new { file.Fullname, file.LocalCheckSum });
string sql = $"DELETE FROM objects WHERE fullname = @Fullname AND localCheckSum = @LocalCheckSum;";
return await _semaphore.ExecuteAsync(sql, new { file.Fullname, file.LocalCheckSum }) > 0;
}

/// <inheritdoc />
Expand Down
2 changes: 1 addition & 1 deletion Parallel.Core/Database/IDatabase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public interface IDatabase
/// <returns>True if successful, false otherwise</returns>
Task<bool> AddFileAsync(LocalFile file);

Task RemoveFileAsync(LocalFile file);
Task<bool> RemoveFileAsync(LocalFile file);

/// <summary>
/// Gets a list of files by newest revision.
Expand Down
2 changes: 1 addition & 1 deletion Parallel.Core/IO/PathBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public static string TempDirectory
}
}

public static string TempFile => Path.Combine(TempDirectory, DateTime.Now.ToString("MM-dd-yyyy hh-mm-ss") + ".tmp");
public static string TempFile => Path.Combine(TempDirectory, UnixTime.Now.TotalMilliseconds + ".tmp");

/// <summary>
/// Gets the corresponding directory for program data based on the <see cref="OSPlatform"/>.
Expand Down
4 changes: 2 additions & 2 deletions Parallel.Core/IO/Syncing/FileSyncManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,9 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options
{
try
{
await (Database != null ? Database.RemoveFileAsync(file) : Task.CompletedTask);
if (!await (Database?.RemoveFileAsync(file) ?? Task.FromResult(false))) return;
if (!await (Database?.AddHistoryAsync(HistoryType.Pruned, file) ?? Task.FromResult(false))) Log.Error("Failed to add history: {Fullname}", file.Fullname);
await StorageProvider.DeleteFileAsync(PathBuilder.GetObjectPath(RemoteVault, file.LocalCheckSum!));
await StorageProvider.DeleteFileAsync(PathBuilder.GetObjectPath(RemoteVault, file.RemoteCheckSum!));
progress.Report(ProgressOperation.Pruned, file);
}
catch (Exception ex)
Expand Down
3 changes: 2 additions & 1 deletion Parallel.Core/Settings/RemoteVaultConfig.cs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public class RemoteVaultConfig : LocalVaultConfig
public HashSet<string> IgnoreDirectories { get; } = CreateIgnoreDirectories();

/// <summary>
/// A collection of deleted directories allowed to be pruned.
/// A collection of directories allowed to be pruned.
/// <para>Recommended when using a cloud-based <see cref="FileService"/> to save on storage costs.</para>
/// <para>Default: Empty</para>
/// </summary>
Expand Down Expand Up @@ -82,6 +82,7 @@ private static HashSet<string> CreateIgnoreDirectories()
{
list.Add("$RECYCLE.BIN/"); // For NTFS file systems
list.Add("*.lnk"); // Shortcuts to other paths
list.Add("desktop.ini");
}

// Ignore folders on Linux machines
Expand Down
4 changes: 3 additions & 1 deletion Parallel.Service/Tasks/TaskQueuer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,12 @@ public async Task<Func<Task>> WaitAsync(CancellationToken ct)
while (true)
{
if (!_queue.TryDequeue(out QueuedTask? candidate)) continue;
if (_tasks.IsEmpty) continue;

QueuedTask next = _tasks.Values.OrderByDescending(t => t.Priority + (DateTime.UtcNow - t.EnqueuedAt).TotalSeconds * 0.1).First();
if (ReferenceEquals(candidate, next))
{
_logger.LogDebug($"Starting task with key: '{candidate.Key}', remaining: {_queue.Count - 1}");
_logger.LogDebug($"Starting task with key: '{candidate.Key}', remaining: {_queue.Count}");
_tasks.TryRemove(candidate.Key, out _);
return candidate.TaskFunc;
}
Expand Down
Loading