diff --git a/go.mod b/go.mod index 49875ffa18..7cfffc591a 100644 --- a/go.mod +++ b/go.mod @@ -403,3 +403,5 @@ replace go-micro.dev/v4 => github.com/butonic/go-micro/v4 v4.11.4-0.202609292135 exclude github.com/mattn/go-sqlite3 v2.0.3+incompatible replace github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-20250512152754-23325793059a + +replace github.com/opencloud-eu/reva/v2 => github.com/pbleser-oc/reva/v2 v2.0.0-20261028165100-414fb6332286 diff --git a/go.sum b/go.sum index d114ff237e..cf6fcf94f6 100644 --- a/go.sum +++ b/go.sum @@ -761,8 +761,6 @@ github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+l github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI= github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346 h1:3ptbiFhSK4fzY/rC3idcYUveiwepMoOov1sBpc7Rg3k= github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346/go.mod h1:lTM8JeGblNpoMySTW7Lui2+c5TTLI95mwxtdUIHHrhU= -github.com/opencloud-eu/reva/v2 v2.52.1 h1:c4tjmqtbJeqDjztTNq4Jvl3dWWSMHhXqV1JixKLKbT4= -github.com/opencloud-eu/reva/v2 v2.52.1/go.mod h1:rAwuI3PX8n3RxBXHfpMtAKqfwxTIKhZol1Bi3Qj39p8= github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI= github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -782,6 +780,8 @@ github.com/pablodz/inotifywaitgo v0.0.12 h1:DVJJTFMDlSAdO46yisPF2vzozs/r+4k6ih8M github.com/pablodz/inotifywaitgo v0.0.12/go.mod h1:JWyDC6H+5oZRdUJUgKuaye+8Ph5hEs6HVzVoPKzWSGI= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pbleser-oc/reva/v2 v2.0.0-20261028165100-414fb6332286 h1:j8nT6toT4B70CcCda6zCbVE2JQxYyL2vNTNTaRtVV9Y= +github.com/pbleser-oc/reva/v2 v2.0.0-20261028165100-414fb6332286/go.mod h1:rAwuI3PX8n3RxBXHfpMtAKqfwxTIKhZol1Bi3Qj39p8= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= diff --git a/services/storage-users/README.md b/services/storage-users/README.md index 4e90d10033..1369c89bf8 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -240,7 +240,7 @@ opencloud storage-users blobstore get --space-id=b19ec764-5398-458a-8ff1-1925bd9 ### Manage Trash-Bin Items -This command set provides commands to get an overview of trash-bin items, restore items and purge old items of `personal` spaces and `project` spaces (spaces that have been created manually). `trash-bin` commands require a `spaceID` as parameter. +This command set provides commands to get an overview of trash-bin items, restore items and purge old items of `personal` spaces and `project` spaces (spaces that have been created manually). `trash-bin` commands require a `spaceID` as parameter. ```bash opencloud storage-users trash-bin @@ -307,6 +307,127 @@ When the CLI tool restores the item with the `replace` option, the existing item opencloud storage-users trash-bin restore [command options] ['spaceID' required] ['itemID' required] ``` +### Clean Up Old Revisions + +Every change to a file keeps its previous content as a revision (version). With `OC_REVISIONS_CLEANUP_STRATEGY`, the `storage-users` service deletes old revisions according to a cleanup strategy, whenever a file gets a new revision. The `decomposed`, `decomposeds3` and `posix` storage drivers are supported, with `posix` also for the revisions created from changes on disk. The default strategy `disabled` keeps all revisions and, as one would expect, does not perform any cleanup process for revisions at all. + +The service only cleans up a file when it gets a new revision, so files that are never changed again keep their revisions. The `revisions cleanup` command cleans up the revisions of whole spaces, folders or files on demand, to cover those files as well, or to apply a different strategy once, e.g. after changing the strategy or to free storage. + +```bash +opencloud storage-users revisions cleanup [flags] +``` + +The command uses the storage directly, with the configuration of the `storage-users` service, and can run while the service is running. It needs access to the storage and, for the `posix` driver, to NATS. With the single binary, NATS runs within the server: without a running server, the command gives up after 10 seconds. It needs no service account. + +#### Strategies + +A strategy is a comma-separated list of up to three parts. + +Each `X` is either +* a number, which is a number of revisions, always counting from the newest one, +* or a duration with one of the units `w` (weeks), `d` (days), `h` (hours), `m` (minutes) or `s` (seconds). + +Durations in days or weeks can use an uppercase unit, `D` or `W`, to count from midnight (server time) instead of from now: `1d` means the last 24 hours, `1D` since yesterday 00:00. + +| Strategy | Meaning | Examples | +|---|---|---| +| `auto` | Thin out revisions the older they get: one per 10 seconds in the last minute, one per minute in the last hour, one per hour in the last 24 hours, one per day in the last 30 days, one per week beyond | `auto` | +| `X` | Delete everything beyond `X` | `10`, `30d`, `0` deletes all revisions | +| `X, auto` | Keep everything within `X`, thin out the older revisions | `14d, auto` | +| `auto, X` | Delete everything beyond `X`, thin out the rest | `auto, 90d`, `auto, 50` | +| `X1, X2` | Keep everything within `X1`, delete everything beyond `X2`; one must be a number, the other a duration | `5, 30d`, `7d, 10` | +| `X1, auto, X2` | Keep everything within `X1`, delete everything beyond `X2`, thin out in between | `7d, auto, 90d`, `1D, auto, 30d` | +| `disabled` | Never delete revisions | `disabled` | + +Note that the age of a revision is the time its content was written, i.e. the modification time the file had before it was changed, which is also the date the version list shows for it. It is not the time the file was changed, which turned that content into a revision. A document that was unchanged for two months gets a revision that is two months old when it is changed, and e.g. `30d` deletes that revision right away. Combine a duration with a number of revisions, e.g. `5, 30d`, to always keep the newest ones. + +#### Selecting the Files + +Exactly one of these selects the files to clean up: + +* `--space `: all files of a space. It can be given several times. The space IDs are accepted as the APIs show them, e.g. `$`. +* `--space --path `: only a file or a folder (recursively) within one space, e.g. `--path /Documents`. +* `--all` (`-a`): all files of all spaces, including disabled spaces, which are skipped. +* `--fs-path ` (`-p`): a file or a folder (recursively) by its path on disk. Only supported by the `posix` driver. + +#### Options + +The strategy and the settings of the background workers default to the configured ones and can be overridden for a run. Only `--queue-full` defaults to `block` without a timeout, whatever is configured: the command finds files faster than the workers clean them up, so dropping the files that find the queue full would skip most of them. + +| Flag | Default | Meaning | +|---|---|---| +| `--strategy`, `-s` | `OC_REVISIONS_CLEANUP_STRATEGY` | The strategy, see above. The command fails if there is none. | +| `--async` | `OC_REVISIONS_CLEANUP_ASYNC` | Clean up files in background workers while looking for more. | +| `--workers` | `OC_REVISIONS_CLEANUP_ASYNC_WORKERS` | The number of background workers, `0` for half the available CPUs, at least 2. | +| `--queue-size` | `OC_REVISIONS_CLEANUP_ASYNC_QUEUE_SIZE` | The number of files that can wait for a worker, `0` for 64 per worker. | +| `--queue-full` | `block` | `block` waits for room in a full queue, `drop` skips the file. | +| `--queue-full-timeout` | `0` | With `block`, skip a file after waiting this long for room, e.g. `10s`. `0` waits as long as it takes. | +| `--dry-run`, `-n` | off | Delete nothing, only print what would be deleted. | +| `--verbose`, `-v` | off | Print every file and every revision, with what was decided about it and why. | +| `--json`, `-j` | off | Print JSON objects, one per line, instead of text. | +| `--timeout`, `-t` | none | Stop the cleanup gracefully after this time, e.g. `30m` or `2h`. | +| `--shuffle` | off | Clean up the spaces, and the files within each folder, in a random order. | + +By default, the command prints a line for every file whose revisions it deleted, that it could not clean up or that it skipped, with the space ID and the path of the file within the space, and a summary: + +```plaintext +25b69780-5f39-43be-a7ac-a9b9e9fe4230 /Documents/report.odt: deleted 3 of 12 revisions (4.2 MiB) +25b69780-5f39-43be-a7ac-a9b9e9fe4230 /Photos/holiday.jpg: skipped: an upload to the file is still being processed +1520 files in 3 spaces: deleted 3 revisions (4.2 MiB) of 1 file, 1 skipped, 0 failed +``` + +With `--verbose`, every revision is listed with the decision (`deleted`, `kept` or `protected`), its time, its size and the reason: + +```plaintext +25b69780-5f39-43be-a7ac-a9b9e9fe4230 /dir1/file1 (node 43ca5408-c77b-4d9a-8a3b-d4470b1253b8): would delete 1 of 3 revisions (1.2 KiB) + protected 2026-10-15T13:24:35+02:00 1.2 KiB protected by "1": one of the 1 newest + kept 2026-10-13T15:24:35+02:00 1.2 KiB kept by "30d": not older than 2026-09-15T15:24:35+02:00 + deleted 2026-03-29T15:24:35+02:00 1.2 KiB deleted by "30d": older than 2026-09-15T15:24:35+02:00 +``` + +With `--json`, every file is an object of the `type` `file`, with the revisions in `details` when `--verbose` is set, and the last line is an object of the `type` `summary`. The sizes are in bytes. The same run as above: + +```json +{"type":"file","space_id":"25b69780-5f39-43be-a7ac-a9b9e9fe4230","node_id":"8a1c0e2f-6b4d-4f7a-9c3e-2d5b7f1a0c94","path":"/Documents/report.odt","dry_run":false,"revisions":12,"deleted":3,"deleted_bytes":4404019,"failed":0} +{"type":"file","space_id":"25b69780-5f39-43be-a7ac-a9b9e9fe4230","node_id":"f3e9d2c1-0a7b-4c8e-b6d5-9e4f1a2b3c7d","path":"/Photos/holiday.jpg","dry_run":false,"skipped":"an upload to the file is still being processed","revisions":0,"deleted":0,"deleted_bytes":0,"failed":0} +{"type":"summary","dry_run":false,"spaces":3,"files":1520,"cleaned":1,"deleted":3,"deleted_bytes":4404019,"skipped":1,"failed":0} +``` + +With `--verbose` as well, a file object lists its revisions in `details`, here from the dry run above: + +```json +{"type":"file","space_id":"25b69780-5f39-43be-a7ac-a9b9e9fe4230","node_id":"43ca5408-c77b-4d9a-8a3b-d4470b1253b8","path":"/dir1/file1","dry_run":true,"revisions":3,"deleted":1,"deleted_bytes":1234,"failed":0,"details":[{"key":"43ca5408-c77b-4d9a-8a3b-d4470b1253b8.REV.2026-10-15T11:24:35.118424081Z","time":"2026-10-15T11:24:35.118424081Z","size":1234,"decision":"protected","reason":"protected by \"1\": one of the 1 newest"},{"key":"43ca5408-c77b-4d9a-8a3b-d4470b1253b8.REV.2026-10-13T13:24:35.118424081Z","time":"2026-10-13T13:24:35.118424081Z","size":1234,"decision":"kept","reason":"kept by \"30d\": not older than 2026-09-15T15:24:35+02:00"},{"key":"43ca5408-c77b-4d9a-8a3b-d4470b1253b8.REV.2026-03-29T13:24:35.118424081Z","time":"2026-03-29T13:24:35.118424081Z","size":1234,"decision":"deleted","reason":"deleted by \"30d\": older than 2026-09-15T15:24:35+02:00"}]} +``` + +A file that failed has an `error`, and so has a revision in `details` that could not be deleted. When `--timeout` stopped the cleanup, the summary has a `stopped` field with the reason. + +Things to keep in mind: + +* **A dry run deletes nothing**, but it still takes the locks of the files, which can create lock files. +* **`--timeout` stops the cleanup gracefully:** the files being cleaned up are finished, the files still waiting for a worker are skipped. It can't interrupt a file that waits for a lock held by another process, though, so the command can run longer than the timeout. Ctrl-C stops the cleanup the same way. +* **Every run starts over from the beginning.** For regular runs with a timeout, e.g. by a cron job, add `--shuffle`, so that every run starts in a different place and the runs eventually reach all files, also in spaces too large for a single run. + +The exit code is `0` when the cleanup is done, `1` if the cleanup or any file failed, and `2` if `--timeout` stopped the cleanup before all files were cleaned up, and nothing failed. + +#### Command Examples + +```bash +# show what a strategy would delete in all spaces, with the reason for every revision +opencloud storage-users revisions cleanup --all --strategy "auto, 90d" --dry-run --verbose + +# keep only the 10 newest revisions of the files in a folder of a space +opencloud storage-users revisions cleanup --space 25b69780-5f39-43be-a7ac-a9b9e9fe4230 --path /Documents --strategy 10 + +# clean up a folder by its path on disk, with the posix driver +opencloud storage-users revisions cleanup --fs-path /var/lib/opencloud/storage/users/projects/25b69780-5f39-43be-a7ac-a9b9e9fe4230/Documents + +# a nightly cron job: the configured strategy, at most one hour, in a random order +opencloud storage-users revisions cleanup --all --timeout 1h --shuffle + +# the same with the short flags +opencloud storage-users revisions cleanup -a -t 1h --shuffle +``` + ## Caching The `storage-users` service caches stat, metadata and uuids of files and folders via the configured store in `STORAGE_USERS_FILEMETADATA_CACHE_STORE` and `STORAGE_USERS_ID_CACHE_STORE`. Possible stores are: diff --git a/services/storage-users/pkg/command/revisions.go b/services/storage-users/pkg/command/revisions.go new file mode 100644 index 0000000000..ac5448ce0c --- /dev/null +++ b/services/storage-users/pkg/command/revisions.go @@ -0,0 +1,580 @@ +package command + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/rs/zerolog" + "github.com/spf13/cobra" + + "github.com/opencloud-eu/opencloud/pkg/config/configlog" + "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config" + "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config/parser" + "github.com/opencloud-eu/opencloud/services/storage-users/pkg/event" + "github.com/opencloud-eu/opencloud/services/storage-users/pkg/revaconfig" + "github.com/opencloud-eu/reva/v2/pkg/events" + "github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup" + "github.com/opencloud-eu/reva/v2/pkg/storagespace" +) + +// The exit codes of the revisions cleanup command +const ( + revisionsCleanupExitFailed = 1 // the cleanup or some of its files failed + revisionsCleanupExitStopped = 2 // stopped by --timeout before all files were cleaned up +) + +// Revisions wraps the commands for the revisions (versions) of files. +func Revisions(cfg *config.Config) *cobra.Command { + revisionsCmd := &cobra.Command{ + Use: "revisions", + Short: "manage the revisions (versions) of files", + } + revisionsCmd.AddCommand(RevisionsCleanup(cfg)) + return revisionsCmd +} + +// RevisionsCleanup deletes old revisions of the files of whole spaces, folders or files. +func RevisionsCleanup(cfg *config.Config) *cobra.Command { + cmd, _ := revisionsCleanupCommand(cfg) + return cmd +} + +// revisionsCleanupCommand returns the revisions cleanup command, and the values of its flags. +func revisionsCleanupCommand(cfg *config.Config) (*cobra.Command, *revisionsCleanupFlags) { + f := &revisionsCleanupFlags{} + cmd := &cobra.Command{ + Use: "cleanup", + Short: "Delete old revisions according to a cleanup strategy", + Long: `Delete old revisions (versions) of files according to a cleanup strategy. + +The storage-users service cleans up the revisions of a file whenever it gets a new one, +with the strategy configured in OC_REVISIONS_CLEANUP_STRATEGY. This command cleans up +the revisions of files that are not changed anymore, or applies a different strategy +once, e.g. after the strategy was changed or to free storage. + +Exactly one of these selects the files to clean up: + --space all files of a space, repeatable; with --path, only that file or + folder within the space + --all all files of all spaces + --fs-path a file or folder by its path on disk ('posix' driver only) + +The strategy and the settings of the background workers default to the configured +ones (OC_REVISIONS_CLEANUP_*) and can be overridden for this run. See the description +of OC_REVISIONS_CLEANUP_STRATEGY for the strategies, e.g. '30d', '10' or '7d, auto, 90d'. +Only --queue-full defaults to 'block' without a timeout, whatever is configured: the +command finds files faster than the workers clean them up, so dropping the files that +find the queue full would skip most of them. + +The command uses the storage directly, with the configuration of the storage-users +service, and can run while the service is running. It needs access to the storage and, +for the 'posix' driver, to NATS. With the single binary, NATS runs within the server: +without a running server, the command gives up after 10 seconds. + +By default, it prints a line for every file it deleted revisions of, could not clean +up or skipped, and a summary. --verbose prints every file with every revision, what +was decided about it and why. --json prints the same as JSON objects, one per line. + +--dry-run deletes nothing and prints what would be deleted. It doesn't change any +file, but it still takes the locks of the files, which can create lock files. + +--timeout stops the cleanup gracefully: the files being cleaned up are finished, the +files still waiting are skipped. It can't interrupt a file that waits for the lock of +another process, though, so the command can run longer than the timeout. As every run +starts over from the beginning, combine --timeout with --shuffle for regular runs, e.g. +by a cron job, so that every run starts in a different place. + +Exit codes: 0 when done, 1 if the cleanup or any file failed, 2 if --timeout stopped +the cleanup before all files were cleaned up (and nothing failed).`, + Example: ` opencloud storage-users revisions cleanup --all --strategy "auto, 90d" --dry-run + opencloud storage-users revisions cleanup --space --path /Documents --strategy 10 + opencloud storage-users revisions cleanup --all --timeout 1h --shuffle`, + PreRunE: func(_ *cobra.Command, _ []string) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + code, err := runRevisionsCleanup(cmd, cfg, f) + if err != nil { + return err + } + if code != 0 { + os.Exit(code) + } + return nil + }, + } + flags := cmd.Flags() + flags.StringSliceVar(&f.spaces, "space", nil, "the id of a space to clean up, repeatable") + flags.StringVar(&f.o.request.Path, "path", "", "a file or folder within the space given by --space, to clean up only that") + flags.StringVarP(&f.o.request.FSPath, "fs-path", "p", "", "a file or folder to clean up, by its path on disk ('posix' driver only)") + flags.BoolVarP(&f.o.request.All, "all", "a", false, "clean up all spaces") + flags.StringVarP(&f.strategy, "strategy", "s", "", "the cleanup strategy, defaults to OC_REVISIONS_CLEANUP_STRATEGY") + flags.BoolVarP(&f.o.request.DryRun, "dry-run", "n", false, "delete nothing, only print what would be deleted") + flags.BoolVarP(&f.o.verbose, "verbose", "v", false, "print every file and every revision, with the decision and its reason") + flags.BoolVarP(&f.o.json, "json", "j", false, "print JSON objects, one per line, instead of text") + flags.DurationVarP(&f.o.timeout, "timeout", "t", 0, "stop the cleanup gracefully after this time, e.g. 30m or 2h; 0 never stops it") + flags.BoolVar(&f.o.request.Shuffle, "shuffle", false, "clean up the spaces and the files within each folder in a random order") + flags.BoolVar(&f.async, "async", false, "clean up files in background workers, while looking for more, defaults to OC_REVISIONS_CLEANUP_ASYNC") + flags.IntVar(&f.workers, "workers", 0, "the number of background workers, 0 for automatic, defaults to OC_REVISIONS_CLEANUP_ASYNC_WORKERS") + flags.IntVar(&f.queueSize, "queue-size", 0, "the number of files that can wait for a worker, 0 for automatic, defaults to OC_REVISIONS_CLEANUP_ASYNC_QUEUE_SIZE") + flags.StringVar(&f.queueFull, "queue-full", string(revisioncleanup.QueueFullBlock), "'block' (wait for room) or 'drop' (skip) files when the queue is full") + flags.DurationVar(&f.queueFullTimeout, "queue-full-timeout", 0, "with --queue-full block, skip a file after waiting this long for room, e.g. 10s; 0 waits as long as it takes") + return cmd, f +} + +// revisionsCleanupFlags holds the values of the flags of the revisions cleanup command: in o +// the ones that are used as they are, the others are turned into options by options. +type revisionsCleanupFlags struct { + o revisionsCleanupOptions + spaces []string + strategy string + async bool + workers int + queueSize int + queueFull string + // the settings of a full queue, which, unlike the others, don't default to the configured + // ones, see the help of the command + queueFullTimeout time.Duration +} + +// revisionsCleanupOptions are the settings of a run of the revisions cleanup command +type revisionsCleanupOptions struct { + request revisioncleanup.Request + timeout time.Duration + verbose bool + json bool +} + +// options builds the options of a run from the flags, with the configured strategy and async +// settings unless the flags override them; changed tells whether a flag was given. +func (f *revisionsCleanupFlags) options(cfg *config.Config, changed func(name string) bool) (revisionsCleanupOptions, error) { + o := f.o + r := &o.request + + for _, space := range f.spaces { + // accept the ids as the APIs show them, e.g. $ + id, err := storagespace.ParseID(space) + if err != nil { + return o, fmt.Errorf("invalid space id %q: %w", space, err) + } + r.SpaceIDs = append(r.SpaceIDs, id.GetSpaceId()) + } + if r.Path != "" && len(r.SpaceIDs) == 0 { + return o, errors.New("--path requires --space") + } + err := r.Validate() + if err != nil { + return o, err + } + if r.FSPath != "" && cfg.Driver != "posix" { + return o, fmt.Errorf("--fs-path is only supported by the 'posix' driver, not by %q", cfg.Driver) + } + + r.Strategy = cfg.RevisionsCleanup.Strategy + if changed("strategy") { + r.Strategy = f.strategy + } + // check the strategy before the storage is opened + switch cleaner, err := revisioncleanup.New(r.Strategy); { + case err != nil: + return o, fmt.Errorf("invalid strategy %q: %w", r.Strategy, err) + case !cleaner.Enabled(): + return o, errors.New("no strategy: the revisions cleanup is disabled, give one with --strategy or configure OC_REVISIONS_CLEANUP_STRATEGY") + } + + async := revisioncleanup.AsyncOptions{ + Enabled: cfg.RevisionsCleanup.Async, + Workers: cfg.RevisionsCleanup.AsyncWorkers, + QueueSize: cfg.RevisionsCleanup.AsyncQueueSize, + // not the configured ones, see revisionsCleanupFlags.queueFullTimeout + QueueFull: revisioncleanup.QueueFullPolicy(f.queueFull), + QueueFullTimeout: f.queueFullTimeout, + } + if changed("async") { + async.Enabled = f.async + } + if changed("workers") { + async.Workers = f.workers + } + if changed("queue-size") { + async.QueueSize = f.queueSize + } + r.Async = &async + + if o.timeout < 0 { + return o, fmt.Errorf("invalid timeout %s", o.timeout) + } + return o, nil +} + +// runRevisionsCleanup runs the revisions cleanup command and returns its exit code. +func runRevisionsCleanup(cmd *cobra.Command, cfg *config.Config, f *revisionsCleanupFlags) (int, error) { + o, err := f.options(cfg, cmd.Flags().Changed) + if err != nil { + return 0, err + } + + // stop gracefully on ctrl-c as well, also while waiting for NATS + ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer stop() + + onDemand, err := revisionsCleanupDriver(ctx, cfg) + if err != nil { + return 0, err + } + // warnings of the cleanup, e.g. about revisions dated in the future; errors are reported + logger := zerolog.New(zerolog.ConsoleWriter{Out: cmd.ErrOrStderr(), TimeFormat: time.RFC3339, NoColor: true}). + With().Timestamp().Logger().Level(zerolog.WarnLevel) + ctx = logger.WithContext(ctx) + + return cleanupRevisions(ctx, onDemand, o, cmd.OutOrStdout()) +} + +// cleanupRevisions runs the cleanup with the given options, prints its results to w and +// returns the exit code of the command. +func cleanupRevisions(ctx context.Context, onDemand revisioncleanup.OnDemand, o revisionsCleanupOptions, w io.Writer) (int, error) { + if o.timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, o.timeout) + defer cancel() + } + + out := newRevisionsCleanupOutput(w, o) + err := onDemand.CleanupRevisionsUnchecked(ctx, o.request, out.report) + switch { + case errors.Is(err, context.DeadlineExceeded): + out.summary(fmt.Sprintf("stopped after the timeout of %s", o.timeout)) + if out.failed() { + return revisionsCleanupExitFailed, nil + } + return revisionsCleanupExitStopped, nil + case errors.Is(err, context.Canceled): + out.summary("interrupted") + return revisionsCleanupExitFailed, nil + case err != nil: + out.summary("") + return 0, err + } + out.summary("") + if out.err != nil { + return 0, fmt.Errorf("could not write the output: %w", out.err) + } + if out.failed() { + return revisionsCleanupExitFailed, nil + } + return 0, nil +} + +// revisionsCleanupDriver creates the storage driver, without the parts of the +// storage-users service that would interfere with the running service. +func revisionsCleanupDriver(ctx context.Context, cfg *config.Config) (revisioncleanup.OnDemand, error) { + newFS, ok := registry.NewFuncs[cfg.Driver] + if !ok { + return nil, fmt.Errorf("unknown storage driver %q", cfg.Driver) + } + drivers := revaconfig.StorageProviderDrivers(cfg) + var stream events.Stream + if cfg.Driver == "posix" { + // neither scan nor watch the file system, that is the service's job: e.g. a second + // inotifywait would stop the one of the service + drivers["posix"] = revaconfig.Posix(cfg, false, false) + // posix refuses to start without an event stream + var err error + stream, err = newEventStream(ctx, cfg, eventStreamTimeout, event.NewStream) + if err != nil { + return nil, fmt.Errorf("could not create the event stream for the posix driver: %w", err) + } + } + driverConfig, ok := drivers[cfg.Driver].(map[string]any) + if !ok { + return nil, fmt.Errorf("unknown storage driver %q", cfg.Driver) + } + // Without async uploads, the driver doesn't join the postprocessing consumer group of the + // service: the events delivered to this short-lived process would be lost. + driverConfig["asyncfileuploads"] = false + + fs, err := newFS(driverConfig, stream, nil) + if err != nil { + return nil, fmt.Errorf("could not initialize the storage driver %q: %w", cfg.Driver, err) + } + onDemand, ok := fs.(revisioncleanup.OnDemand) + if !ok { + return nil, fmt.Errorf("the %q driver can't clean up revisions", cfg.Driver) + } + return onDemand, nil +} + +// eventStreamTimeout is how long the command waits for NATS. The event stream retries +// for up to 15 minutes, which suits the service, but not a command: with the single +// binary, NATS only runs within the server, see +// https://github.com/opencloud-eu/opencloud/issues/3805 +const eventStreamTimeout = 10 * time.Second + +// newEventStream creates the event stream with newStream, but gives up after timeout, or +// when ctx is done (e.g. on ctrl-c), as newStream itself waits for NATS for up to 15 +// minutes and can't be interrupted. newStream then keeps trying in the background until +// the command exits. +func newEventStream[S any](ctx context.Context, cfg *config.Config, timeout time.Duration, newStream func(*config.Config) (S, error)) (S, error) { + type result struct { + stream S + err error + } + done := make(chan result, 1) + go func() { + stream, err := newStream(cfg) + done <- result{stream, err} + }() + + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case r := <-done: + return r.stream, r.err + case <-timer.C: + var none S + return none, fmt.Errorf("could not connect to NATS at %s within %s: the command needs NATS, i.e. a running OpenCloud server or a separate NATS", cfg.Events.Addr, timeout) + case <-ctx.Done(): + var none S + return none, fmt.Errorf("interrupted while connecting to NATS at %s: %w", cfg.Events.Addr, ctx.Err()) + } +} + +// revisionsCleanupOutput prints the results of a revisions cleanup as they come in, and +// sums them up. +type revisionsCleanupOutput struct { + w io.Writer + o revisionsCleanupOptions + enc *json.Encoder + spaces map[string]struct{} + files int + cleaned int // files with deleted revisions + deleted int + bytes uint64 + skipped int + errors int // files that failed, as a whole or for some of their revisions + err error // the first error writing the output +} + +func newRevisionsCleanupOutput(w io.Writer, o revisionsCleanupOptions) *revisionsCleanupOutput { + out := &revisionsCleanupOutput{w: w, o: o, spaces: map[string]struct{}{}} + if o.json { + out.enc = json.NewEncoder(w) + } + return out +} + +// revisionJSON is a revision in the JSON output +type revisionJSON struct { + Key string `json:"key"` + Time time.Time `json:"time"` + Size uint64 `json:"size"` + Decision string `json:"decision"` + Reason string `json:"reason,omitempty"` + Error string `json:"error,omitempty"` +} + +// fileJSON is a file in the JSON output +type fileJSON struct { + Type string `json:"type"` + SpaceID string `json:"space_id"` + NodeID string `json:"node_id"` + Path string `json:"path,omitempty"` + DryRun bool `json:"dry_run"` + Skipped string `json:"skipped,omitempty"` + Error string `json:"error,omitempty"` + Revisions int `json:"revisions"` + Deleted int `json:"deleted"` + DeletedBytes uint64 `json:"deleted_bytes"` + Failed int `json:"failed"` + Details []revisionJSON `json:"details,omitempty"` +} + +// summaryJSON is the summary in the JSON output +type summaryJSON struct { + Type string `json:"type"` + DryRun bool `json:"dry_run"` + Spaces int `json:"spaces"` + Files int `json:"files"` + Cleaned int `json:"cleaned"` + Deleted int `json:"deleted"` + DeletedBytes uint64 `json:"deleted_bytes"` + Skipped int `json:"skipped"` + Failed int `json:"failed"` + Stopped string `json:"stopped,omitempty"` +} + +// report prints the Result of a file, it is called one result at a time. +func (out *revisionsCleanupOutput) report(r revisioncleanup.Result) { + deleted, bytes := r.DeletedCount() + failedRevisions := 0 + for _, rev := range r.Revisions { + if rev.Err != nil { + failedRevisions++ + } + } + failed := r.Err != nil || failedRevisions > 0 + + out.spaces[r.SpaceID] = struct{}{} + out.files++ + out.deleted += deleted + out.bytes += bytes + switch { + case failed: + out.errors++ + case r.Skipped != "": + out.skipped++ + } + if deleted > 0 { + out.cleaned++ + } + + // without --verbose, only what needs attention or changed something + if !out.o.verbose && deleted == 0 && !failed && r.Skipped == "" { + return + } + if out.enc != nil { + out.printJSON(r, deleted, bytes, failedRevisions) + return + } + out.printText(r, deleted, bytes, failedRevisions) +} + +func (out *revisionsCleanupOutput) printJSON(r revisioncleanup.Result, deleted int, bytes uint64, failedRevisions int) { + f := fileJSON{ + Type: "file", + SpaceID: r.SpaceID, + NodeID: r.NodeID, + Path: r.Path, + DryRun: r.DryRun, + Skipped: r.Skipped, + Revisions: len(r.Revisions), + Deleted: deleted, + DeletedBytes: bytes, + Failed: failedRevisions, + } + if r.Err != nil { + f.Error = r.Err.Error() + } + if out.o.verbose { + for _, rev := range r.Revisions { + d := revisionJSON{Key: rev.Key, Time: rev.Time, Size: rev.Size, Decision: string(rev.Decision), Reason: rev.Reason} + if rev.Err != nil { + d.Error = rev.Err.Error() + } + f.Details = append(f.Details, d) + } + } + out.write(out.enc.Encode(f)) +} + +func (out *revisionsCleanupOutput) printText(r revisioncleanup.Result, deleted int, bytes uint64, failedRevisions int) { + file := r.Path + if file == "" || out.o.verbose { + file = strings.TrimSpace(file + " (node " + r.NodeID + ")") + } + line := fmt.Sprintf("%s %s: ", r.SpaceID, file) + switch { + case r.Err != nil: + line += "failed: " + r.Err.Error() + case r.Skipped != "": + line += "skipped: " + r.Skipped + default: + verb := "deleted" + if r.DryRun { + verb = "would delete" + } + line += fmt.Sprintf("%s %d of %s (%s)", verb, deleted, plural(len(r.Revisions), "revision"), formatBytes(bytes)) + if failedRevisions > 0 { + line += fmt.Sprintf(", %d could not be deleted", failedRevisions) + } + } + _, err := fmt.Fprintln(out.w, line) + out.write(err) + + if !out.o.verbose { + for _, rev := range r.Revisions { + if rev.Err != nil { + fmt.Fprintf(out.w, " %s: %s\n", rev.Key, rev.Err) + } + } + return + } + for _, rev := range r.Revisions { + // in the server's time zone, like the times in the reasons + fmt.Fprintf(out.w, " %-9s %s %9s %s\n", rev.Decision, rev.Time.Local().Format(time.RFC3339), formatBytes(rev.Size), rev.Reason) + if rev.Err != nil { + fmt.Fprintf(out.w, " %-9s could not be deleted: %s\n", "", rev.Err) + } + } +} + +// summary prints the totals, stopped tells why the cleanup stopped early, if it did. +func (out *revisionsCleanupOutput) summary(stopped string) { + if out.enc != nil { + out.write(out.enc.Encode(summaryJSON{ + Type: "summary", + DryRun: out.o.request.DryRun, + Spaces: len(out.spaces), + Files: out.files, + Cleaned: out.cleaned, + Deleted: out.deleted, + DeletedBytes: out.bytes, + Skipped: out.skipped, + Failed: out.errors, + Stopped: stopped, + })) + return + } + verb := "deleted" + if out.o.request.DryRun { + verb = "would delete" + } + fmt.Fprintf(out.w, "%s in %s: %s %s (%s) of %s, %d skipped, %d failed\n", + plural(out.files, "file"), plural(len(out.spaces), "space"), verb, plural(out.deleted, "revision"), + formatBytes(out.bytes), plural(out.cleaned, "file"), out.skipped, out.errors) + if stopped != "" { + fmt.Fprintf(out.w, "The cleanup was %s, before all files were cleaned up.\n", stopped) + } +} + +// write records the first error writing the output. +func (out *revisionsCleanupOutput) write(err error) { + if out.err == nil { + out.err = err + } +} + +// failed tells whether the cleanup of any file failed. +func (out *revisionsCleanupOutput) failed() bool { + return out.errors > 0 +} + +// plural returns the count with the noun, e.g. "1 file" or "2 files". +func plural(n int, noun string) string { + if n == 1 { + return "1 " + noun + } + return fmt.Sprintf("%d %ss", n, noun) +} + +// formatBytes formats a size in bytes with a binary unit, e.g. 1.5 MiB. +func formatBytes(b uint64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := uint64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp]) +} diff --git a/services/storage-users/pkg/command/revisions_test.go b/services/storage-users/pkg/command/revisions_test.go new file mode 100644 index 0000000000..c9b90c3741 --- /dev/null +++ b/services/storage-users/pkg/command/revisions_test.go @@ -0,0 +1,327 @@ +package command + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup" + "github.com/test-go/testify/require" + + "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config" +) + +// revisionsCleanupOptionsFor parses the flags of the revisions cleanup command +func revisionsCleanupOptionsFor(t *testing.T, cfg *config.Config, args ...string) (revisionsCleanupOptions, error) { + t.Helper() + cmd, f := revisionsCleanupCommand(cfg) + require.NoError(t, cmd.ParseFlags(args)) + return f.options(cfg, cmd.Flags().Changed) +} + +func revisionsCleanupConfig() *config.Config { + return &config.Config{ + Driver: "decomposed", + RevisionsCleanup: config.RevisionsCleanup{ + Strategy: "auto, 90d", + Async: true, + AsyncWorkers: 4, + AsyncQueueSize: 100, + AsyncQueueFull: "drop", + AsyncQueueFullTimeout: 5 * time.Second, + }, + } +} + +func TestRevisionsCleanupOptions(t *testing.T) { + t.Run("the configuration is the default", func(t *testing.T) { + o, err := revisionsCleanupOptionsFor(t, revisionsCleanupConfig(), "--all") + require.NoError(t, err) + require.True(t, o.request.All) + require.Equal(t, "auto, 90d", o.request.Strategy) + // except for a full queue: the command waits for room as long as it takes + require.Equal(t, &revisioncleanup.AsyncOptions{Enabled: true, Workers: 4, QueueSize: 100, QueueFull: revisioncleanup.QueueFullBlock}, o.request.Async) + require.False(t, o.request.DryRun) + require.False(t, o.request.Shuffle) + require.Zero(t, o.timeout) + }) + + t.Run("flags override the configuration", func(t *testing.T) { + o, err := revisionsCleanupOptionsFor(t, revisionsCleanupConfig(), + "--space", "storage$space1!space1", "--space", "space2", "--strategy", "10", + "--async=false", "--workers", "0", "--queue-size", "7", "--queue-full", "drop", "--queue-full-timeout", "30s", + "--dry-run", "--shuffle", "--timeout", "1h", "--verbose", "--json") + require.NoError(t, err) + require.Equal(t, []string{"space1", "space2"}, o.request.SpaceIDs, "the space ids as the APIs show them are accepted") + require.Equal(t, "10", o.request.Strategy) + require.Equal(t, &revisioncleanup.AsyncOptions{Enabled: false, Workers: 0, QueueSize: 7, QueueFull: revisioncleanup.QueueFullDrop, QueueFullTimeout: 30 * time.Second}, o.request.Async) + require.True(t, o.request.DryRun) + require.True(t, o.request.Shuffle) + require.Equal(t, time.Hour, o.timeout) + require.True(t, o.verbose) + require.True(t, o.json) + }) + + t.Run("short flags", func(t *testing.T) { + o, err := revisionsCleanupOptionsFor(t, revisionsCleanupConfig(), "-anvj", "-s", "10", "-t", "30m") + require.NoError(t, err) + require.True(t, o.request.All) + require.True(t, o.request.DryRun) + require.True(t, o.verbose) + require.True(t, o.json) + require.Equal(t, "10", o.request.Strategy) + require.Equal(t, 30*time.Minute, o.timeout) + + cfg := revisionsCleanupConfig() + cfg.Driver = "posix" + o, err = revisionsCleanupOptionsFor(t, cfg, "-p", "/var/lib/opencloud/users/einstein") + require.NoError(t, err) + require.Equal(t, "/var/lib/opencloud/users/einstein", o.request.FSPath) + }) + + t.Run("a path within a space", func(t *testing.T) { + o, err := revisionsCleanupOptionsFor(t, revisionsCleanupConfig(), "--space", "space1", "--path", "/Documents") + require.NoError(t, err) + require.Equal(t, "/Documents", o.request.Path) + }) + + t.Run("a path on disk with posix", func(t *testing.T) { + cfg := revisionsCleanupConfig() + cfg.Driver = "posix" + o, err := revisionsCleanupOptionsFor(t, cfg, "--fs-path", "/var/lib/opencloud/users/einstein") + require.NoError(t, err) + require.Equal(t, "/var/lib/opencloud/users/einstein", o.request.FSPath) + }) + + for _, tc := range []struct { + name string + strategy string + args []string + err string + }{ + {name: "nothing to clean up", err: "no space"}, + {name: "a path without a space", args: []string{"--path", "/Documents"}, err: "--path requires --space"}, + {name: "all and a space", args: []string{"--all", "--space", "space1"}, err: "not both"}, + {name: "a path in two spaces", args: []string{"--space", "space1", "--space", "space2", "--path", "/Documents"}, err: "exactly one space"}, + {name: "a path on disk and a space", args: []string{"--fs-path", "/data", "--space", "space1"}, err: "can't be combined"}, + {name: "a path on disk without posix", args: []string{"--fs-path", "/data"}, err: "only supported by the 'posix' driver"}, + {name: "the cleanup is disabled", strategy: "disabled", args: []string{"--all"}, err: "--strategy"}, + {name: "an invalid strategy", args: []string{"--all", "--strategy", "whenever"}, err: "invalid strategy"}, + {name: "a negative timeout", args: []string{"--all", "--timeout", "-1s"}, err: "invalid timeout"}, + } { + t.Run("rejects "+tc.name, func(t *testing.T) { + cfg := revisionsCleanupConfig() + if tc.strategy != "" { + cfg.RevisionsCleanup.Strategy = tc.strategy + } + _, err := revisionsCleanupOptionsFor(t, cfg, tc.args...) + require.Error(t, err) + require.Contains(t, err.Error(), tc.err) + }) + } +} + +// fakeOnDemand reports the given results, and then waits for the end of ctx if wait is set +type fakeOnDemand struct { + results []revisioncleanup.Result + wait bool + err error + request revisioncleanup.Request +} + +func (f *fakeOnDemand) CleanupRevisionsUnchecked(ctx context.Context, req revisioncleanup.Request, report func(revisioncleanup.Result)) error { + f.request = req + for _, r := range f.results { + r.DryRun = req.DryRun + report(r) + } + if f.wait { + <-ctx.Done() + return ctx.Err() + } + return f.err +} + +var ( + revisionsCleanupTime = time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + // a file with a deleted and a kept revision + cleanedFile = revisioncleanup.Result{SpaceID: "space1", NodeID: "node1", Path: "/Documents/report.odt", Revisions: []revisioncleanup.RevisionResult{ + {Key: "node1.REV.1", Time: revisionsCleanupTime, Size: 1536, Decision: revisioncleanup.Kept, Reason: "kept by \"auto\""}, + {Key: "node1.REV.2", Time: revisionsCleanupTime.Add(-100 * 24 * time.Hour), Size: 2048, Decision: revisioncleanup.Deleted, Reason: "deleted by \"90d\""}, + }} + // a file with nothing to delete + untouchedFile = revisioncleanup.Result{SpaceID: "space1", NodeID: "node2", Path: "/notes.txt", Revisions: []revisioncleanup.RevisionResult{ + {Key: "node2.REV.1", Time: revisionsCleanupTime, Size: 10, Decision: revisioncleanup.Kept, Reason: "kept by \"auto\""}, + }} + skippedFile = revisioncleanup.Result{SpaceID: "space2", NodeID: "node3", Path: "/upload.bin", Skipped: "an upload to the file is still being processed"} + failedFile = revisioncleanup.Result{SpaceID: "space2", NodeID: "node4", Path: "/broken.txt", Err: errors.New("could not list revisions")} +) + +func runFakeRevisionsCleanup(t *testing.T, f *fakeOnDemand, o revisionsCleanupOptions) (int, string, error) { + t.Helper() + if o.request.Async == nil { + o.request = revisioncleanup.Request{All: true, Strategy: "auto, 90d", DryRun: o.request.DryRun} + } + var out bytes.Buffer + code, err := cleanupRevisions(context.Background(), f, o, &out) + return code, out.String(), err +} + +func TestRevisionsCleanupOutput(t *testing.T) { + t.Run("only the files that changed or need attention, and a summary", func(t *testing.T) { + code, out, err := runFakeRevisionsCleanup(t, &fakeOnDemand{results: []revisioncleanup.Result{cleanedFile, untouchedFile, skippedFile}}, revisionsCleanupOptions{}) + require.NoError(t, err) + require.Equal(t, 0, code) + require.Equal(t, `space1 /Documents/report.odt: deleted 1 of 2 revisions (2.0 KiB) +space2 /upload.bin: skipped: an upload to the file is still being processed +3 files in 2 spaces: deleted 1 revision (2.0 KiB) of 1 file, 1 skipped, 0 failed +`, out) + }) + + t.Run("a dry run", func(t *testing.T) { + o := revisionsCleanupOptions{request: revisioncleanup.Request{DryRun: true}} + code, out, err := runFakeRevisionsCleanup(t, &fakeOnDemand{results: []revisioncleanup.Result{cleanedFile}}, o) + require.NoError(t, err) + require.Equal(t, 0, code) + require.Contains(t, out, "space1 /Documents/report.odt: would delete 1 of 2 revisions") + require.Contains(t, out, "would delete 1 revision (2.0 KiB) of 1 file") + }) + + t.Run("every file and revision with --verbose", func(t *testing.T) { + code, out, err := runFakeRevisionsCleanup(t, &fakeOnDemand{results: []revisioncleanup.Result{cleanedFile, untouchedFile}}, revisionsCleanupOptions{verbose: true}) + require.NoError(t, err) + require.Equal(t, 0, code) + // the times are in the local time zone, like the ones in the reasons + local := func(t time.Time) string { return t.Local().Format(time.RFC3339) } + require.Equal(t, `space1 /Documents/report.odt (node node1): deleted 1 of 2 revisions (2.0 KiB) + kept `+local(revisionsCleanupTime)+` 1.5 KiB kept by "auto" + deleted `+local(revisionsCleanupTime.Add(-100*24*time.Hour))+` 2.0 KiB deleted by "90d" +space1 /notes.txt (node node2): deleted 0 of 1 revision (0 B) + kept `+local(revisionsCleanupTime)+` 10 B kept by "auto" +2 files in 1 space: deleted 1 revision (2.0 KiB) of 1 file, 0 skipped, 0 failed +`, out) + }) + + t.Run("JSON lines", func(t *testing.T) { + code, out, err := runFakeRevisionsCleanup(t, &fakeOnDemand{results: []revisioncleanup.Result{cleanedFile, untouchedFile, failedFile}}, revisionsCleanupOptions{json: true}) + require.NoError(t, err) + require.Equal(t, revisionsCleanupExitFailed, code) + lines := strings.Split(strings.TrimSpace(out), "\n") + require.Len(t, lines, 3, "the untouched file is left out without --verbose") + + var file fileJSON + require.NoError(t, json.Unmarshal([]byte(lines[0]), &file)) + require.Equal(t, fileJSON{Type: "file", SpaceID: "space1", NodeID: "node1", Path: "/Documents/report.odt", Revisions: 2, Deleted: 1, DeletedBytes: 2048}, file) + require.NoError(t, json.Unmarshal([]byte(lines[1]), &file)) + require.Equal(t, "could not list revisions", file.Error) + + var summary summaryJSON + require.NoError(t, json.Unmarshal([]byte(lines[2]), &summary)) + require.Equal(t, summaryJSON{Type: "summary", Spaces: 2, Files: 3, Cleaned: 1, Deleted: 1, DeletedBytes: 2048, Failed: 1}, summary) + }) + + t.Run("JSON lines with the revisions with --verbose", func(t *testing.T) { + _, out, err := runFakeRevisionsCleanup(t, &fakeOnDemand{results: []revisioncleanup.Result{cleanedFile}}, revisionsCleanupOptions{json: true, verbose: true}) + require.NoError(t, err) + var file fileJSON + require.NoError(t, json.Unmarshal([]byte(strings.Split(out, "\n")[0]), &file)) + require.Len(t, file.Details, 2) + require.Equal(t, revisionJSON{Key: "node1.REV.2", Time: revisionsCleanupTime.Add(-100 * 24 * time.Hour), Size: 2048, Decision: "deleted", Reason: "deleted by \"90d\""}, file.Details[1]) + }) +} + +func TestRevisionsCleanupExitCodes(t *testing.T) { + t.Run("a failed file", func(t *testing.T) { + code, out, err := runFakeRevisionsCleanup(t, &fakeOnDemand{results: []revisioncleanup.Result{cleanedFile, failedFile}}, revisionsCleanupOptions{}) + require.NoError(t, err) + require.Equal(t, revisionsCleanupExitFailed, code) + require.Contains(t, out, "space2 /broken.txt: failed: could not list revisions") + }) + + t.Run("a revision that could not be deleted", func(t *testing.T) { + file := cleanedFile + file.Revisions = append([]revisioncleanup.RevisionResult{}, cleanedFile.Revisions...) + file.Revisions[1].Err = errors.New("permission denied") + code, out, err := runFakeRevisionsCleanup(t, &fakeOnDemand{results: []revisioncleanup.Result{file}}, revisionsCleanupOptions{}) + require.NoError(t, err) + require.Equal(t, revisionsCleanupExitFailed, code) + require.Contains(t, out, "deleted 0 of 2 revisions (0 B), 1 could not be deleted\n node1.REV.2: permission denied\n") + }) + + t.Run("the cleanup could not run", func(t *testing.T) { + code, _, err := runFakeRevisionsCleanup(t, &fakeOnDemand{err: errors.New("could not find space space1")}, revisionsCleanupOptions{}) + require.Error(t, err) + require.Contains(t, err.Error(), "could not find space") + require.Equal(t, 0, code, "the error itself makes the command fail") + }) + + t.Run("stopped by the timeout", func(t *testing.T) { + code, out, err := runFakeRevisionsCleanup(t, &fakeOnDemand{results: []revisioncleanup.Result{cleanedFile}, wait: true}, revisionsCleanupOptions{timeout: 10 * time.Millisecond}) + require.NoError(t, err) + require.Equal(t, revisionsCleanupExitStopped, code) + require.Contains(t, out, "The cleanup was stopped after the timeout of 10ms, before all files were cleaned up.") + }) + + t.Run("stopped by the timeout, with a failed file", func(t *testing.T) { + code, _, err := runFakeRevisionsCleanup(t, &fakeOnDemand{results: []revisioncleanup.Result{failedFile}, wait: true}, revisionsCleanupOptions{timeout: 10 * time.Millisecond}) + require.NoError(t, err) + require.Equal(t, revisionsCleanupExitFailed, code) + }) +} + +func TestFormatBytes(t *testing.T) { + for in, want := range map[uint64]string{0: "0 B", 1023: "1023 B", 1024: "1.0 KiB", 1536: "1.5 KiB", 5 << 20: "5.0 MiB", 3 << 40: "3.0 TiB"} { + require.Equal(t, want, formatBytes(in)) + } +} + +func TestNewEventStream(t *testing.T) { + cfg := &config.Config{} + cfg.Events.Addr = "127.0.0.1:9233" + + // blocked stands for a NATS that doesn't answer: newStream returns only when the test ends + blocked := func(t *testing.T) func(*config.Config) (string, error) { + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + return func(*config.Config) (string, error) { + <-release + return "", errors.New("released") + } + } + + t.Run("returns the stream once connected", func(t *testing.T) { + stream, err := newEventStream(context.Background(), cfg, time.Second, func(*config.Config) (string, error) { + return "stream", nil + }) + require.NoError(t, err) + require.Equal(t, "stream", stream) + }) + + t.Run("returns the error of the stream", func(t *testing.T) { + _, err := newEventStream(context.Background(), cfg, time.Second, func(*config.Config) (string, error) { + return "", errors.New("bad credentials") + }) + require.EqualError(t, err, "bad credentials") + }) + + t.Run("gives up after the timeout, and says what it needs", func(t *testing.T) { + start := time.Now() + _, err := newEventStream(context.Background(), cfg, 50*time.Millisecond, blocked(t)) + require.Error(t, err) + require.Contains(t, err.Error(), "could not connect to NATS at 127.0.0.1:9233 within 50ms") + require.Contains(t, err.Error(), "a running OpenCloud server or a separate NATS") + require.True(t, time.Since(start) < 5*time.Second, "it should give up long before 5s") + }) + + t.Run("stops when interrupted", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(50*time.Millisecond, cancel) + _, err := newEventStream(ctx, cfg, time.Minute, blocked(t)) + require.True(t, errors.Is(err, context.Canceled), "expected context.Canceled, got %v", err) + require.Contains(t, err.Error(), "interrupted while connecting to NATS") + }) +} diff --git a/services/storage-users/pkg/command/root.go b/services/storage-users/pkg/command/root.go index db14b09efb..0b357698db 100644 --- a/services/storage-users/pkg/command/root.go +++ b/services/storage-users/pkg/command/root.go @@ -19,6 +19,7 @@ func GetCommands(cfg *config.Config) []*cobra.Command { Uploads(cfg), TrashBin(cfg), Blobstore(cfg), + Revisions(cfg), // infos about this service Health(cfg), diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 81aa42111e..19132a3bdc 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -30,6 +30,7 @@ type Config struct { Events Events `yaml:"events"` FilemetadataCache FilemetadataCache `yaml:"filemetadata_cache"` IDCache IDCache `yaml:"id_cache"` + RevisionsCleanup RevisionsCleanup `yaml:"revisions_cleanup"` MountID string `yaml:"mount_id" env:"STORAGE_USERS_MOUNT_ID" desc:"Mount ID of this storage." introductionVersion:"1.0.0"` ReadOnly bool `yaml:"readonly" env:"STORAGE_USERS_READ_ONLY" desc:"Set this storage to be read-only." introductionVersion:"1.0.0"` UploadExpiration int64 `yaml:"upload_expiration" env:"STORAGE_USERS_UPLOAD_EXPIRATION" desc:"Duration in seconds after which uploads will expire. Note that when setting this to a low number, uploads could be cancelled before they are finished and return a 403 to the user." introductionVersion:"1.0.0"` @@ -226,6 +227,16 @@ type Events struct { // TODO use TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_EVENTS_TLS_ROOT_CA_CERTIFICATE;STORAGE_USERS_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided STORAGE_USERS_EVENTS_TLS_INSECURE will be seen as false." introductionVersion:"1.0.0"` } +// RevisionsCleanup configures the deletion of old revisions, for the 'decomposed', 'decomposeds3' and 'posix' storage drivers +type RevisionsCleanup struct { + Strategy string `yaml:"strategy" env:"OC_REVISIONS_CLEANUP_STRATEGY" desc:"Which old revisions (versions) of a file to delete whenever the file gets a new revision. 'disabled' keeps all revisions. A strategy is a comma-separated list of up to three parts: a number of revisions, which counts from the newest one (e.g. '10'), a duration with one of the units 'w', 'd', 'h', 'm' or 's' (e.g. '30d'; with the uppercase units 'D' and 'W', days and weeks count from midnight, e.g. '1D'), and 'auto', which thins out revisions the older they get. 'X' deletes everything beyond X, e.g. '10', '30d', or '0' to delete all revisions. 'X, auto' keeps everything within X and thins out the older revisions, e.g. '14d, auto'. 'auto, X' deletes everything beyond X and thins out the rest, e.g. 'auto, 90d'. 'X1, X2' keeps everything within X1 and deletes everything beyond X2, e.g. '5, 30d'. 'X1, auto, X2' also thins out in between, e.g. '7d, auto, 90d'. The age of a revision is the time its content was written, i.e. the modification time the file had before it was changed, not the time of that change. Only files that get a new revision are cleaned up, including revisions created from changes on disk with posix. An invalid value makes the storage driver fail at startup." introductionVersion:"%%NEXT%%"` + Async bool `yaml:"async" env:"OC_REVISIONS_CLEANUP_ASYNC" desc:"Run the revision cleanup in background workers, so that it does not slow down the upload, restore or change that created the new revision. When set to false, the cleanup runs right away." introductionVersion:"%%NEXT%%"` + AsyncWorkers int `yaml:"async_workers" env:"OC_REVISIONS_CLEANUP_ASYNC_WORKERS" desc:"Number of background workers for the revision cleanup. A value of 0 determines it automatically: half the available CPUs, respecting container CPU limits, but at least 2." introductionVersion:"%%NEXT%%"` + AsyncQueueSize int `yaml:"async_queue_size" env:"OC_REVISIONS_CLEANUP_ASYNC_QUEUE_SIZE" desc:"Number of files that can wait for a background revision cleanup. A value of 0 determines it automatically: 64 per worker." introductionVersion:"%%NEXT%%"` + AsyncQueueFull string `yaml:"async_queue_full" env:"OC_REVISIONS_CLEANUP_ASYNC_QUEUE_FULL" desc:"What happens when a file gets a new revision while the revision cleanup queue is full. 'drop' skips the cleanup of that file, which is cleaned up with its next revision. 'block' waits for room, up to OC_REVISIONS_CLEANUP_ASYNC_QUEUE_FULL_TIMEOUT." introductionVersion:"%%NEXT%%"` + AsyncQueueFullTimeout time.Duration `yaml:"async_queue_full_timeout" env:"OC_REVISIONS_CLEANUP_ASYNC_QUEUE_FULL_TIMEOUT" desc:"How long 'block' (see OC_REVISIONS_CLEANUP_ASYNC_QUEUE_FULL) waits for room in the revision cleanup queue at most, after which the cleanup of that file is skipped as with 'drop'. A value of 0 waits as long as it takes, so that no cleanup is ever skipped, but then writes can slow down without bound under heavy load. See the Environment Variable Types description for more details." introductionVersion:"%%NEXT%%"` +} + // FilemetadataCache holds cache config type FilemetadataCache struct { Store string `yaml:"store" env:"OC_CACHE_STORE;STORAGE_USERS_FILEMETADATA_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"` diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index 504798445f..a8a277466f 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -173,6 +173,12 @@ func DefaultConfig() *config.Config { Nodes: []string{"127.0.0.1:9233"}, Database: "ids-storage-users", }, + RevisionsCleanup: config.RevisionsCleanup{ + Strategy: "disabled", + Async: true, + AsyncQueueFull: "drop", + AsyncQueueFullTimeout: 5 * time.Second, + }, Tasks: config.Tasks{ PurgeTrashBin: config.PurgeTrashBin{ ProjectDeleteBefore: 30 * 24 * time.Hour, diff --git a/services/storage-users/pkg/revaconfig/drivers.go b/services/storage-users/pkg/revaconfig/drivers.go index 99b53564cd..6aa6e16497 100644 --- a/services/storage-users/pkg/revaconfig/drivers.go +++ b/services/storage-users/pkg/revaconfig/drivers.go @@ -103,6 +103,13 @@ func Posix(cfg *config.Config, enableFSScan, enableFSWatch bool) map[string]any "metadata_prefix": cfg.Drivers.Posix.MetadataPrefix, "multi_tenant_enabled": cfg.Commons.MultiTenantEnabled, "propagator": cfg.Drivers.Posix.Propagator, + + "revisions_cleanup_strategy": cfg.RevisionsCleanup.Strategy, + "revisions_cleanup_async": cfg.RevisionsCleanup.Async, + "revisions_cleanup_async_workers": cfg.RevisionsCleanup.AsyncWorkers, + "revisions_cleanup_async_queue_size": cfg.RevisionsCleanup.AsyncQueueSize, + "revisions_cleanup_async_queue_full": cfg.RevisionsCleanup.AsyncQueueFull, + "revisions_cleanup_async_queue_full_timeout": cfg.RevisionsCleanup.AsyncQueueFullTimeout, "async_propagator_options": map[string]any{ "propagation_delay": cfg.Drivers.Posix.AsyncPropagatorOptions.PropagationDelay, }, @@ -207,6 +214,13 @@ func Decomposed(cfg *config.Config) map[string]any { "max_quota": cfg.Drivers.Decomposed.MaxQuota, "disable_versioning": cfg.Drivers.Decomposed.DisableVersioning, "multi_tenant_enabled": cfg.Commons.MultiTenantEnabled, + + "revisions_cleanup_strategy": cfg.RevisionsCleanup.Strategy, + "revisions_cleanup_async": cfg.RevisionsCleanup.Async, + "revisions_cleanup_async_workers": cfg.RevisionsCleanup.AsyncWorkers, + "revisions_cleanup_async_queue_size": cfg.RevisionsCleanup.AsyncQueueSize, + "revisions_cleanup_async_queue_full": cfg.RevisionsCleanup.AsyncQueueFull, + "revisions_cleanup_async_queue_full_timeout": cfg.RevisionsCleanup.AsyncQueueFullTimeout, "filemetadatacache": map[string]any{ "cache_store": cfg.FilemetadataCache.Store, "cache_nodes": cfg.FilemetadataCache.Nodes, @@ -266,6 +280,13 @@ func DecomposedNoEvents(cfg *config.Config) map[string]any { "max_quota": cfg.Drivers.Decomposed.MaxQuota, "disable_versioning": cfg.Drivers.Decomposed.DisableVersioning, "multi_tenant_enabled": cfg.Commons.MultiTenantEnabled, + + "revisions_cleanup_strategy": cfg.RevisionsCleanup.Strategy, + "revisions_cleanup_async": cfg.RevisionsCleanup.Async, + "revisions_cleanup_async_workers": cfg.RevisionsCleanup.AsyncWorkers, + "revisions_cleanup_async_queue_size": cfg.RevisionsCleanup.AsyncQueueSize, + "revisions_cleanup_async_queue_full": cfg.RevisionsCleanup.AsyncQueueFull, + "revisions_cleanup_async_queue_full_timeout": cfg.RevisionsCleanup.AsyncQueueFullTimeout, "filemetadatacache": map[string]any{ "cache_store": cfg.FilemetadataCache.Store, "cache_nodes": cfg.FilemetadataCache.Nodes, @@ -330,6 +351,13 @@ func DecomposedS3(cfg *config.Config) map[string]any { "disable_versioning": cfg.Drivers.DecomposedS3.DisableVersioning, "multi_tenant_enabled": cfg.Commons.MultiTenantEnabled, "asyncfileuploads": cfg.Drivers.DecomposedS3.AsyncUploads, + + "revisions_cleanup_strategy": cfg.RevisionsCleanup.Strategy, + "revisions_cleanup_async": cfg.RevisionsCleanup.Async, + "revisions_cleanup_async_workers": cfg.RevisionsCleanup.AsyncWorkers, + "revisions_cleanup_async_queue_size": cfg.RevisionsCleanup.AsyncQueueSize, + "revisions_cleanup_async_queue_full": cfg.RevisionsCleanup.AsyncQueueFull, + "revisions_cleanup_async_queue_full_timeout": cfg.RevisionsCleanup.AsyncQueueFullTimeout, "filemetadatacache": map[string]any{ "cache_store": cfg.FilemetadataCache.Store, "cache_nodes": cfg.FilemetadataCache.Nodes, @@ -393,6 +421,13 @@ func DecomposedS3NoEvents(cfg *config.Config) map[string]any { "disable_versioning": cfg.Drivers.DecomposedS3.DisableVersioning, "multi_tenant_enabled": cfg.Commons.MultiTenantEnabled, "lock_cycle_duration_factor": cfg.Drivers.DecomposedS3.LockCycleDurationFactor, + + "revisions_cleanup_strategy": cfg.RevisionsCleanup.Strategy, + "revisions_cleanup_async": cfg.RevisionsCleanup.Async, + "revisions_cleanup_async_workers": cfg.RevisionsCleanup.AsyncWorkers, + "revisions_cleanup_async_queue_size": cfg.RevisionsCleanup.AsyncQueueSize, + "revisions_cleanup_async_queue_full": cfg.RevisionsCleanup.AsyncQueueFull, + "revisions_cleanup_async_queue_full_timeout": cfg.RevisionsCleanup.AsyncQueueFullTimeout, "filemetadatacache": map[string]any{ "cache_store": cfg.FilemetadataCache.Store, "cache_nodes": cfg.FilemetadataCache.Nodes, diff --git a/services/storage-users/pkg/revaconfig/drivers_test.go b/services/storage-users/pkg/revaconfig/drivers_test.go index 940a5b31fa..725ae3c9ce 100644 --- a/services/storage-users/pkg/revaconfig/drivers_test.go +++ b/services/storage-users/pkg/revaconfig/drivers_test.go @@ -1,8 +1,11 @@ package revaconfig import ( + "errors" "testing" + "time" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options" "github.com/test-go/testify/require" "github.com/opencloud-eu/opencloud/pkg/config/envdecode" @@ -26,3 +29,53 @@ func TestMetadataPrefixReachesAllDrivers(t *testing.T) { require.Equal(t, "user.foreign.", driver["metadata_prefix"], name) } } + +func TestRevisionsCleanupReachesAllDrivers(t *testing.T) { + drivers := func(t *testing.T) map[string]map[string]any { + cfg := defaults.FullDefaultConfig() + cfg.Commons = &shared.Commons{GRPCClientTLS: &shared.GRPCClientTLS{}} + if err := envdecode.Decode(cfg); !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) { + require.NoError(t, err) + } + return map[string]map[string]any{ + "posix": Posix(cfg, false, false), + "decomposed": Decomposed(cfg), + "decomposed no events": DecomposedNoEvents(cfg), + "decomposeds3": DecomposedS3(cfg), + "decomposeds3 no events": DecomposedS3NoEvents(cfg), + } + } + + t.Run("defaults keep all revisions", func(t *testing.T) { + for name, driver := range drivers(t) { + o, err := options.New(driver) + require.NoError(t, err, name) + require.Equal(t, "disabled", o.RevisionsCleanupStrategy, name) + require.True(t, o.RevisionsCleanupAsync, name) + require.Equal(t, 0, o.RevisionsCleanupAsyncWorkers, name) + require.Equal(t, 0, o.RevisionsCleanupAsyncQueueSize, name) + require.Equal(t, "drop", o.RevisionsCleanupAsyncQueueFull, name) + require.Equal(t, 5*time.Second, o.RevisionsCleanupAsyncQueueFullTimeout, name) + } + }) + + t.Run("environment variables", func(t *testing.T) { + t.Setenv("OC_REVISIONS_CLEANUP_STRATEGY", "7d, auto, 90d") + t.Setenv("OC_REVISIONS_CLEANUP_ASYNC", "false") + t.Setenv("OC_REVISIONS_CLEANUP_ASYNC_WORKERS", "3") + t.Setenv("OC_REVISIONS_CLEANUP_ASYNC_QUEUE_SIZE", "42") + t.Setenv("OC_REVISIONS_CLEANUP_ASYNC_QUEUE_FULL", "block") + t.Setenv("OC_REVISIONS_CLEANUP_ASYNC_QUEUE_FULL_TIMEOUT", "1m") + for name, driver := range drivers(t) { + // decoded by reva, so that a key it does not know would show up as a default value + o, err := options.New(driver) + require.NoError(t, err, name) + require.Equal(t, "7d, auto, 90d", o.RevisionsCleanupStrategy, name) + require.False(t, o.RevisionsCleanupAsync, name) + require.Equal(t, 3, o.RevisionsCleanupAsyncWorkers, name) + require.Equal(t, 42, o.RevisionsCleanupAsyncQueueSize, name) + require.Equal(t, "block", o.RevisionsCleanupAsyncQueueFull, name) + require.Equal(t, time.Minute, o.RevisionsCleanupAsyncQueueFullTimeout, name) + } + }) +} diff --git a/tests/acceptance/bootstrap/CliContext.php b/tests/acceptance/bootstrap/CliContext.php index 2ac6e976ca..50fe1f831a 100644 --- a/tests/acceptance/bootstrap/CliContext.php +++ b/tests/acceptance/bootstrap/CliContext.php @@ -336,6 +336,165 @@ public function theAdministratorRemovesTheVersionsOfFilesInSpaceUsingSpaceId(str $this->featureContext->setResponse(CliHelper::runCommand($body)); } + /** + * runs the revisions cleanup command with the files to clean up and further flags + * + * @param string $scope the flags that select the files, e.g. "--all" + * @param string $flags e.g. "--strategy 1 --dry-run" + * + * @return void + */ + private function cleanUpFileVersions(string $scope, string $flags): void { + $command = \trim("storage-users revisions cleanup $scope $flags"); + $body = [ + "command" => $command + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @param string $flags + * + * @return void + */ + #[When('the administrator cleans up the file versions using the CLI with flags :flags')] + public function theAdministratorCleansUpTheFileVersionsUsingTheCliWithFlags(string $flags): void { + $this->cleanUpFileVersions("", $flags); + } + + /** + * + * @param string $flags + * + * @return void + */ + #[When('the administrator cleans up the file versions of all spaces using the CLI with flags :flags')] + public function theAdministratorCleansUpTheFileVersionsOfAllSpacesUsingTheCliWithFlags(string $flags): void { + $this->cleanUpFileVersions("--all", $flags); + } + + /** + * + * @param string $space + * @param string $user + * @param string $flags + * + * @return void + */ + #[When('the administrator cleans up the file versions of space :space of user :user using the CLI with flags :flags')] + public function theAdministratorCleansUpTheFileVersionsOfSpaceUsingTheCliWithFlags( + string $space, + string $user, + string $flags + ): void { + $spaceId = $this->spacesContext->getSpaceIdByName($user, $space); + $this->cleanUpFileVersions("--space $spaceId", $flags); + } + + /** + * + * @param string $path + * @param string $space + * @param string $user + * @param string $flags + * + * @return void + */ + #[When('the administrator cleans up the file versions of :path in space :space of user :user using the CLI with flags :flags')] + public function theAdministratorCleansUpTheFileVersionsOfPathInSpaceUsingTheCliWithFlags( + string $path, + string $space, + string $user, + string $flags + ): void { + $spaceId = $this->spacesContext->getSpaceIdByName($user, $space); + $this->cleanUpFileVersions("--space $spaceId --path $path", $flags); + } + + /** + * + * @param string $path + * @param string $user + * @param string $flags + * + * @return void + */ + #[When('the administrator cleans up the file versions of :path of user :user by its path on the POSIX filesystem using the CLI with flags :flags')] + public function theAdministratorCleansUpTheFileVersionsByPathOnPosixUsingTheCliWithFlags( + string $path, + string $user, + string $flags + ): void { + $userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id'); + $fsPath = $this->getUsersStoragePath() . "/$userUuid/" . \ltrim($path, '/'); + $this->cleanUpFileVersions("--fs-path $fsPath", $flags); + } + + /** + * + * @param int $exitCode + * + * @return void + */ + #[Then('the command should exit with code :exitCode')] + public function theCommandShouldExitWithCode(int $exitCode): void { + $response = $this->featureContext->getResponse(); + $this->featureContext->theHTTPStatusCodeShouldBe(200, '', $response); + + $jsonResponse = $this->featureContext->getJsonDecodedResponse($response); + Assert::assertSame( + $exitCode, + $jsonResponse["exitCode"], + "Expected exit code $exitCode, but got " . $jsonResponse["exitCode"] + . ". Message: " . $jsonResponse["message"] + ); + } + + /** + * checks the output of a command that prints JSON objects, one per line, e.g. with --json, + * for an object with a "type" and the given values + * + * @param string $type + * @param TableNode $table with the columns "key" and "value" + * + * @return void + */ + #[Then('the JSON output of the command should contain an object of type :type with:')] + public function theJsonOutputOfTheCommandShouldContainAnObjectOfTypeWith(string $type, TableNode $table): void { + $response = $this->featureContext->getResponse(); + $message = $this->featureContext->getJsonDecodedResponse($response)["message"]; + $expected = []; + foreach ($table->getHash() as $row) { + $expected[$row['key']] = $this->featureContext->substituteInLineCodes($row['value']); + } + + // other lines, e.g. log messages, are not JSON objects + foreach (\preg_split('/\r?\n/', $message) as $line) { + $object = \json_decode(\trim($line), true); + if (!\is_array($object) || ($object["type"] ?? null) !== $type) { + continue; + } + $matches = true; + foreach ($expected as $key => $value) { + $actual = $object[$key] ?? null; + if (\is_bool($actual)) { + $actual = $actual ? "true" : "false"; + } + if ((string)$actual !== $value) { + $matches = false; + break; + } + } + if ($matches) { + return; + } + } + Assert::fail( + "No JSON object of type '$type' with " . \json_encode($expected) . " in the output:\n$message" + ); + } + /** * * @return void diff --git a/tests/acceptance/bootstrap/FilesVersionsContext.php b/tests/acceptance/bootstrap/FilesVersionsContext.php index 29d24b31d9..943923d1c1 100644 --- a/tests/acceptance/bootstrap/FilesVersionsContext.php +++ b/tests/acceptance/bootstrap/FilesVersionsContext.php @@ -84,6 +84,36 @@ public function userGetsFileVersions(string $user, string $file): void { $this->featureContext->setResponse($this->getFileVersions($user, $file)); } + /** + * for changes that happen in the background, e.g. the revision cleanup after postprocessing + * + * @param string $file + * @param string $user + * @param int $number + * + * @return void + * @throws Exception + */ + #[Then('the number of versions of file :file for user :user should eventually be :number')] + public function theNumberOfVersionsOfFileForUserShouldEventuallyBe(string $file, string $user, int $number): void { + $actualNumber = null; + for ($attempt = 0; $attempt < 20; $attempt++) { + $response = $this->getFileVersions($user, $file); + $this->featureContext->theHTTPStatusCodeShouldBe(207, '', $response); + $responseXmlObject = HttpRequestHelper::getResponseXml($response, __METHOD__); + $actualNumber = \count($responseXmlObject->xpath("//d:getlastmodified")); + if ($actualNumber === $number) { + return; + } + \usleep(500000); + } + Assert::assertEquals( + $number, + $actualNumber, + "Expected number of versions of file '$file' was '$number', but got '$actualNumber' after 10 seconds" + ); + } + /** * * @param string $file diff --git a/tests/acceptance/features/cliCommands/cleanupFileVersions.feature b/tests/acceptance/features/cliCommands/cleanupFileVersions.feature new file mode 100644 index 0000000000..09c325f39e --- /dev/null +++ b/tests/acceptance/features/cliCommands/cleanupFileVersions.feature @@ -0,0 +1,172 @@ +@env-config +Feature: clean up old file versions according to a cleanup strategy + As an administrator + I want to delete old file versions according to a cleanup strategy + So that files that are not changed anymore don't keep all their old versions + + # The strategies are written without spaces, as the CLI wrapper splits commands on spaces; + # "2,auto,10" is the same as "2, auto, 10". + + Background: + Given user "Alice" has been created with default attributes + + + Scenario: keep only the newest version of every file in all spaces + Given user "Alice" has uploaded file with content "version 1" to "textfile.txt" + And user "Alice" has uploaded file with content "version 2" to "textfile.txt" + And user "Alice" has uploaded file with content "version 3" to "textfile.txt" + And user "Alice" has uploaded file with content "version 4" to "textfile.txt" + And user "Alice" has uploaded file with content "version 1" to "anotherFile.txt" + And user "Alice" has uploaded file with content "version 2" to "anotherFile.txt" + When the administrator cleans up the file versions of all spaces using the CLI with flags "--strategy 1" + Then the command should be successful + And the command output should contain "/textfile.txt: deleted 2 of 3 revisions" + And the command output should not contain "/anotherFile.txt" + And the command output should contain "deleted 2 revisions" + When user "Alice" gets the number of versions of file "textfile.txt" + Then the HTTP status code should be "207" + And the number of versions should be "1" + When user "Alice" gets the number of versions of file "anotherFile.txt" + Then the HTTP status code should be "207" + And the number of versions should be "1" + And the content of file "textfile.txt" for user "Alice" should be "version 4" + + + Scenario: a dry run only prints what would be deleted + Given user "Alice" has uploaded file with content "version 1" to "textfile.txt" + And user "Alice" has uploaded file with content "version 2" to "textfile.txt" + And user "Alice" has uploaded file with content "version 3" to "textfile.txt" + When the administrator cleans up the file versions of all spaces using the CLI with flags "-s 0 --dry-run" + Then the command should be successful + And the command output should contain "/textfile.txt: would delete 2 of 2 revisions" + When user "Alice" gets the number of versions of file "textfile.txt" + Then the HTTP status code should be "207" + And the number of versions should be "2" + + + Scenario: clean up only a folder within the personal space + Given user "Alice" has created folder "folder" + And user "Alice" has uploaded file with content "version 1" to "folder/inFolder.txt" + And user "Alice" has uploaded file with content "version 2" to "folder/inFolder.txt" + And user "Alice" has uploaded file with content "version 3" to "folder/inFolder.txt" + And user "Alice" has uploaded file with content "version 1" to "outside.txt" + And user "Alice" has uploaded file with content "version 2" to "outside.txt" + And user "Alice" has uploaded file with content "version 3" to "outside.txt" + When the administrator cleans up the file versions of "/folder" in space "Personal" of user "Alice" using the CLI with flags "-s 0" + Then the command should be successful + And the command output should contain "/folder/inFolder.txt: deleted 2 of 2 revisions" + When user "Alice" gets the number of versions of file "folder/inFolder.txt" + Then the HTTP status code should be "207" + And the number of versions should be "0" + When user "Alice" gets the number of versions of file "outside.txt" + Then the HTTP status code should be "207" + And the number of versions should be "2" + + + Scenario: clean up only a project space + Given the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API + And using spaces DAV path + And user "Alice" has created a space "projectSpace" with the default quota using the Graph API + And user "Alice" has uploaded file with content "version 1" to "personal.txt" + And user "Alice" has uploaded file with content "version 2" to "personal.txt" + And user "Alice" has uploaded file with content "version 3" to "personal.txt" + And user "Alice" has uploaded a file inside space "projectSpace" with content "version 1" to "project.txt" + And user "Alice" has uploaded a file inside space "projectSpace" with content "version 2" to "project.txt" + And user "Alice" has uploaded a file inside space "projectSpace" with content "version 3" to "project.txt" + And we save it into "PROJECT_FILEID" + When the administrator cleans up the file versions of space "projectSpace" of user "Alice" using the CLI with flags "-s 0" + Then the command should be successful + And the command output should contain "/project.txt: deleted 2 of 2 revisions" + When user "Alice" gets the number of versions of file "project.txt" using file-id "<>" + Then the HTTP status code should be "207" + And the number of versions should be "0" + When user "Alice" gets the number of versions of file "personal.txt" + Then the HTTP status code should be "207" + And the number of versions should be "2" + + + Scenario: print the results as JSON + Given user "Alice" has uploaded file with content "version 1" to "textfile.txt" + And user "Alice" has uploaded file with content "version 2" to "textfile.txt" + And user "Alice" has uploaded file with content "version 3" to "textfile.txt" + When the administrator cleans up the file versions of all spaces using the CLI with flags "-s 0 --json" + Then the command should be successful + And the JSON output of the command should contain an object of type "file" with: + | key | value | + | path | /textfile.txt | + | dry_run | false | + | revisions | 2 | + | deleted | 2 | + | failed | 0 | + And the JSON output of the command should contain an object of type "summary" with: + | key | value | + | deleted | 2 | + | failed | 0 | + + + Scenario: a cleanup without a strategy fails when the configured one is disabled + Given user "Alice" has uploaded file with content "version 1" to "textfile.txt" + And user "Alice" has uploaded file with content "version 2" to "textfile.txt" + When the administrator cleans up the file versions using the CLI with flags "--all" + Then the command should exit with code "1" + And the command output should contain "no strategy" + When user "Alice" gets the number of versions of file "textfile.txt" + Then the HTTP status code should be "207" + And the number of versions should be "1" + + + Scenario: a cleanup stopped by its timeout exits with code 2 + Given user "Alice" has uploaded file with content "version 1" to "textfile.txt" + And user "Alice" has uploaded file with content "version 2" to "textfile.txt" + When the administrator cleans up the file versions of all spaces using the CLI with flags "-s 0 --timeout 1ns" + Then the command should exit with code "2" + And the command output should contain "stopped after the timeout of 1ns" + When user "Alice" gets the number of versions of file "textfile.txt" + Then the HTTP status code should be "207" + And the number of versions should be "1" + + + Scenario: a strategy without spaces after its commas + Given user "Alice" has uploaded file with content "version 1" to "textfile.txt" + And user "Alice" has uploaded file with content "version 2" to "textfile.txt" + And user "Alice" has uploaded file with content "version 3" to "textfile.txt" + And user "Alice" has uploaded file with content "version 4" to "textfile.txt" + When the administrator cleans up the file versions of all spaces using the CLI with flags "-s auto,1" + Then the command should be successful + And the command output should contain "/textfile.txt: deleted 2 of 3 revisions" + When user "Alice" gets the number of versions of file "textfile.txt" + Then the HTTP status code should be "207" + And the number of versions should be "1" + + @skipOnOpencloud-decomposed-Storage @skipOnOpencloud-decomposeds3-Storage + Scenario: clean up a folder given by its path on the POSIX filesystem + Given user "Alice" has created folder "folder" + And user "Alice" has uploaded file with content "version 1" to "folder/inFolder.txt" + And user "Alice" has uploaded file with content "version 2" to "folder/inFolder.txt" + And user "Alice" has uploaded file with content "version 3" to "folder/inFolder.txt" + And user "Alice" has uploaded file with content "version 1" to "outside.txt" + And user "Alice" has uploaded file with content "version 2" to "outside.txt" + And user "Alice" has uploaded file with content "version 3" to "outside.txt" + When the administrator cleans up the file versions of "folder" of user "Alice" by its path on the POSIX filesystem using the CLI with flags "-s 0" + Then the command should be successful + And the command output should contain "/folder/inFolder.txt: deleted 2 of 2 revisions" + When user "Alice" gets the number of versions of file "folder/inFolder.txt" + Then the HTTP status code should be "207" + And the number of versions should be "0" + When user "Alice" gets the number of versions of file "outside.txt" + Then the HTTP status code should be "207" + And the number of versions should be "2" + + + Scenario: the configured strategy cleans up a file whenever it gets a new version + Given the following configs have been set: + | config | value | + | OC_REVISIONS_CLEANUP_STRATEGY | 2 | + | OC_REVISIONS_CLEANUP_ASYNC | false | + And user "Alice" has uploaded file with content "version 1" to "textfile.txt" + And user "Alice" has uploaded file with content "version 2" to "textfile.txt" + And user "Alice" has uploaded file with content "version 3" to "textfile.txt" + And user "Alice" has uploaded file with content "version 4" to "textfile.txt" + And user "Alice" has uploaded file with content "version 5" to "textfile.txt" + Then the number of versions of file "textfile.txt" for user "Alice" should eventually be "2" + And the content of file "textfile.txt" for user "Alice" should be "version 5" diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/basic/basic.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/basic/basic.go index 7b7bec0727..45decb911a 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/basic/basic.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/basic/basic.go @@ -52,5 +52,5 @@ func (s *strategy) AddWWWAuthenticate(w http.ResponseWriter, r *http.Request, re // fall back to hostname if not configured realm = r.Host } - w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Basic realm="%s"`, realm)) + w.Header().Add("WWW-Authenticate", auth.Challenge("Basic", realm)) } diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/bearer/bearer.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/bearer/bearer.go index 62168019f0..3ed45596cf 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/bearer/bearer.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/bearer/bearer.go @@ -63,5 +63,5 @@ func (s *strategy) AddWWWAuthenticate(w http.ResponseWriter, r *http.Request, re // fall back to hostname if not configured realm = r.Host } - w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Bearer realm="%s"`, realm)) + w.Header().Add("WWW-Authenticate", auth.Challenge("Bearer", realm)) } diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/dav.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/dav.go index 30dd16ab9c..2b1c0bc91b 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/dav.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/dav.go @@ -20,7 +20,6 @@ package ocdav import ( "context" - "fmt" "net/http" "path" "path/filepath" @@ -35,6 +34,7 @@ import ( "github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/errors" "github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net" "github.com/opencloud-eu/reva/v2/pkg/appctx" + "github.com/opencloud-eu/reva/v2/pkg/auth" ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" "github.com/opencloud-eu/reva/v2/pkg/rhttp/router" @@ -438,7 +438,7 @@ func handleBasicAuth(ctx context.Context, selector pool.Selectable[gatewayv1beta } func authChallenge(scheme, realm string) string { - return fmt.Sprintf("%s realm=\"%s\", charset=\"UTF-8\"", scheme, realm) + return auth.Challenge(scheme, realm, `charset="UTF-8"`) } func handleSignatureAuth(ctx context.Context, selector pool.Selectable[gatewayv1beta1.GatewayAPIClient], token, sig, expiration string) (*gatewayv1beta1.AuthenticateResponse, error) { diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go index da869a905b..f707a2b1ff 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go @@ -1538,6 +1538,8 @@ func mdToPropResponse(ctx context.Context, pf *XML, md *provider.ResourceInfo, p types.WriteString("" + strconv.Itoa(int(conversions.ShareTypeUser)) + "") case "2": // provider.GranteeType_GRANTEE_TYPE_GROUP types.WriteString("" + strconv.Itoa(int(conversions.ShareTypeGroup)) + "") + case "4": // conversions.ShareTypeGuest, guest shares are stored as user grants + types.WriteString("" + strconv.Itoa(int(conversions.ShareTypeGuest)) + "") default: sublog.Debug().Interface("shareType", shareType).Msg("unknown share type, ignoring") } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/auth/challenge.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/auth/challenge.go new file mode 100644 index 0000000000..b4091811f0 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/auth/challenge.go @@ -0,0 +1,14 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package auth + +import "strings" + +var quotedStringEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) + +// Challenge builds a WWW-Authenticate challenge for the given scheme with the +// realm escaped as a quoted-string. Additional params are appended without escaping. +func Challenge(scheme, realm string, params ...string) string { + return strings.Join(append([]string{scheme + ` realm="` + quotedStringEscaper.Replace(realm) + `"`}, params...), ", ") +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/conversions/main.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/conversions/main.go index 699dd89961..bf013d438e 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/conversions/main.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/conversions/main.go @@ -61,6 +61,9 @@ const ( // ShareTypeSpaceMembershipGroup represents an action regarding group type space members ShareTypeSpaceMembershipGroup ShareType = 8 + // ShareTypeGuest represents a guest share + ShareTypeGuest ShareType = 4 + // ShareWithUserTypeUser represents a normal user ShareWithUserTypeUser ShareWithUserType = 0 diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/decomposed.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/decomposed.go index a49bfb2c90..e5e2a771c4 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/decomposed.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/decomposed.go @@ -16,6 +16,10 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. +// Package decomposed implements the "decomposed" storage driver: decomposedfs with +// a blobstore on the local disk. It only creates the blobstore and hands over to +// decomposedfs.NewDefault, see pkg/storage/pkg/decomposedfs/README.md for how the +// decomposedfs based drivers are structured. package decomposed import ( diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposeds3/decomposeds3.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposeds3/decomposeds3.go index eabf69b42e..a4de391307 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposeds3/decomposeds3.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposeds3/decomposeds3.go @@ -16,6 +16,10 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. +// Package decomposeds3 implements the "decomposeds3" storage driver: decomposedfs with +// a blobstore in an S3 bucket. It only creates the blobstore and hands over to +// decomposedfs.NewDefault, see pkg/storage/pkg/decomposedfs/README.md for how the +// decomposedfs based drivers are structured. package decomposeds3 import ( diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup/lookup.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup/lookup.go index 284d2531ba..56259011bd 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup/lookup.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup/lookup.go @@ -134,6 +134,51 @@ func (lu *Lookup) IDsForPath(ctx context.Context, path string) (string, string, return spaceID, nodeID, nil } +// SpacePathFromFSPath resolves a path on disk, as an admin sees it, to the space it is in and its +// path relative to the space root ("" for the space root itself). Relative paths are taken +// relative to the working directory; symlinks are not resolved. The file or folder must be +// known to the id cache, i.e. assimilated. +// +// It checks no permissions, like the other lookups: it only resolves the path. +func (lu *Lookup) SpacePathFromFSPath(ctx context.Context, p string) (string, string, error) { + abs, err := filepath.Abs(p) + if err != nil { + return "", "", err + } + if _, ok := relWithin(lu.InternalRoot(), abs); !ok { + return "", "", errtypes.BadRequest(fmt.Sprintf("%q is not within the storage root %q", abs, lu.InternalRoot())) + } + spaceID, _, err := lu.IDsForPath(ctx, abs) + if err != nil { + return "", "", errtypes.NotFound(fmt.Sprintf("%q is not known to the storage: %v", abs, err)) + } + spaceRoot, err := lu.GetCachedID(ctx, spaceID, spaceID) + if err != nil { + return "", "", errors.Wrapf(err, "could not find the root of space %q", spaceID) + } + rel, ok := relWithin(spaceRoot, abs) + if !ok { + return "", "", errtypes.InternalError(fmt.Sprintf("%q is not within the root %q of its space %q", abs, spaceRoot, spaceID)) + } + if rel == "." { + rel = "" + } + return spaceID, rel, nil +} + +// relWithin returns the path of p relative to root, "." for root itself, and whether p is root +// or within it. Both paths must be absolute. +// +// It relies on filepath.IsLocal rather than on a check for a ".." prefix, which would also +// reject names that only start with "..", e.g. "..foo". +func relWithin(root, p string) (string, bool) { + rel, err := filepath.Rel(root, p) + if err != nil || !filepath.IsLocal(rel) { + return "", false + } + return rel, true +} + // NodeFromPath returns the node for the given path func (lu *Lookup) NodeIDFromParentAndName(ctx context.Context, parent *node.Node, name string) (string, error) { parentPath, err := lu.GetCachedID(ctx, parent.SpaceID, parent.ID) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go index 3588b93719..898967c0c5 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go @@ -16,6 +16,11 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. +// Package posix implements the "posix" storage driver, which keeps a regular, +// browsable directory tree on disk and picks up changes made to it directly. +// It reuses the decomposedfs core with its own lookup, tree, trashbin and blobstore +// and wraps the result, see pkg/storage/pkg/decomposedfs/README.md for how it +// builds itself and how it delegates to and overrides the shared code. package posix import ( @@ -49,6 +54,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper" "github.com/opencloud-eu/reva/v2/pkg/storage/utils/middleware" @@ -64,8 +70,19 @@ type posixFS struct { tree *tree.Tree um usermapper.Mapper + // the inner Decomposedfs, for the on-demand revision cleanup: storage.FS is the + // middleware wrapping it, which doesn't pass optional interfaces through + revisions revisioncleanup.OnDemand + // resolves Request.FSPath for the on-demand revision cleanup + fsPaths fsPathResolver } +type fsPathResolver interface { + SpacePathFromFSPath(ctx context.Context, p string) (string, string, error) +} + +var _ revisioncleanup.OnDemand = (*posixFS)(nil) + // New returns an implementation to of the storage.FS interface that talk to // a local filesystem. func NewDefault(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (storage.FS, error) { @@ -186,10 +203,14 @@ func New(o *options.Options, stream events.Stream, cache, historyCache *idcache. UserMapper: um, DisableVersioning: o.DisableVersioning, Trashbin: trashbin, + // the tree already created the revision cleanup for its own use in assimilation + RevisionsCleanup: tp.RevisionsCleanup(), } dfs, err := decomposedfs.New(&o.Options, aspects, log) if err != nil { + // the tree already started the revision cleanup, stop its workers again + _ = tp.RevisionsCleanup().Shutdown(context.Background()) return nil, err } @@ -228,6 +249,10 @@ func New(o *options.Options, stream events.Stream, cache, historyCache *idcache. mw := middleware.NewFS(dfs, hooks...) fs.FS = mw + if onDemand, ok := dfs.(revisioncleanup.OnDemand); ok { + fs.revisions = onDemand + } + fs.fsPaths = lu fs.tree = tp fs.um = um @@ -240,6 +265,29 @@ func (fs *posixFS) WithDisabledSpaces() storage.FS { return &f } +// CleanupRevisionsUnchecked cleans up revisions on demand, see revisioncleanup.OnDemand. It goes to +// the inner Decomposedfs directly, bypassing the middleware: its space group hooks only matter for +// creating files, the cleanup only deletes revisions. +// +// A path on disk (Request.FSPath) is resolved here to its space and its path within the space, +// which the Decomposedfs doesn't know about. +func (fs *posixFS) CleanupRevisionsUnchecked(ctx context.Context, req revisioncleanup.Request, report func(revisioncleanup.Result)) error { + if fs.revisions == nil { + return errtypes.NotSupported("on-demand revision cleanup") + } + if req.FSPath != "" { + if err := req.Validate(); err != nil { + return err + } + spaceID, path, err := fs.fsPaths.SpacePathFromFSPath(ctx, req.FSPath) + if err != nil { + return err + } + req.SpaceIDs, req.Path, req.FSPath = []string{spaceID}, path, "" + } + return fs.revisions.CleanupRevisionsUnchecked(ctx, req, report) +} + // WarmupIDCache allows triggering a posix fs scan and id cache warmup manually. func (fs *posixFS) WarmupIDCache(root string, assimilate, onlyDirty bool) error { return fs.tree.WarmupIDCache(root, assimilate, onlyDirty) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation.go index 0db4e62a02..015dfd2e99 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation.go @@ -40,6 +40,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/reva/v2/pkg/appctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/events" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher" @@ -831,6 +832,12 @@ assimilate: currentPath := currentNode.InternalPath() stat, err := os.Stat(currentPath) if err == nil { + // Named after the mtime of '.CURRENT', i.e. the time the content it preserves was + // copied there, when the previous change was picked up (normally right after that + // content was written), not after the time of this change. The revision cleanup + // takes the age of a revision from this name (see revisioncleanup.revisionTime); + // whether it should rather be the time of this change is discussed in + // https://github.com/opencloud-eu/opencloud/issues/3735. revisionPath := t.lookup.VersionPath(n.SpaceID, n.ID, stat.ModTime().UTC().Format(time.RFC3339Nano)) err = os.Rename(currentPath, revisionPath) @@ -838,6 +845,28 @@ assimilate: t.log.Error().Err(err).Str("path", path).Str("revisionPath", revisionPath).Msg("could not create revision") return } + + // A change on disk has just created a revision, apply the revision cleanup + // strategy (see package revisioncleanup). This is the only trigger for those + // revisions: they are created by the rename above and never go through + // CreateRevision or the upload code, where the other triggers are. + // It runs when this goroutine is done, after the new content was copied to + // '.CURRENT' below (or that failed): '.CURRENT' is missing from the rename above + // until then, and a change on disk in between would find no '.CURRENT' to create + // a revision from, so the cleanup must not make that window any longer. + // The cleanup takes the node lock itself. If the assimilation that started this + // goroutine still holds it, the cleanup just waits, as nothing waits for this + // goroutine there is no deadlock. + // context.Background() carries no logger, pass t.log on or the cleanup's + // logging would be lost. + // The cleanup is handed the ids, not n: n is still used by the assimilation + // that started this goroutine, and a node caches state without locking. + // HandleRevisionCreatedByID reads a node of its own only to clean it up: not + // at all if the cleanup is disabled, and in the background only in the worker. + defer func() { + ctx := appctx.WithLogger(context.Background(), t.log) + t.revisionsCleanup.HandleRevisionCreatedByID(ctx, n.SpaceID, n.ID) + }() } // Copy the new version to the current version diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/revisions.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/revisions.go index 17db1d4640..5667694f26 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/revisions.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/revisions.go @@ -155,6 +155,13 @@ func (tp *Tree) ListRevisions(ctx context.Context, ref *provider.Reference) (rev return nil, errtypes.NotFound(f) } + return tp.ListRevisionsUnchecked(ctx, n) +} + +// ListRevisionsUnchecked lists all revisions of the given node without checking permissions. +// ListRevisions checks the permissions of the requesting user before delegating here, +// the revision cleanup calls it directly as it is not acting on behalf of a user. +func (tp *Tree) ListRevisionsUnchecked(ctx context.Context, n *node.Node) (revisions []*provider.FileVersion, err error) { revisions = []*provider.FileVersion{} versionGlob := tp.lookup.VersionPath(n.SpaceID, n.ID, "*") if items, err := filepath.Glob(versionGlob); err == nil { @@ -237,6 +244,14 @@ func (tp *Tree) DownloadRevision(ctx context.Context, ref *provider.Reference, r } revNode := node.NewBaseNode(spaceID, revisionKey, tp.lookup) + // the revision's metadata may come from a stale cache entry, after another process + // deleted the revision (see Decomposedfs.RestoreRevision) + if _, err := os.Stat(revNode.InternalPath()); err != nil { + if os.IsNotExist(err) { + return nil, nil, errtypes.NotFound(revisionKey) + } + return nil, nil, err + } _, blobsize, err := tp.lookup.ReadBlobIDAndSizeAttr(ctx, revNode, nil) if err != nil { return nil, nil, errors.Wrapf(err, "Decomposedfs: could not read blob id and size for revision '%s' of node '%s'", kp[1], n.ID) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go index 072c96a9b7..0f4150fbc5 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go @@ -27,6 +27,7 @@ import ( "os" "path/filepath" "strings" + "syscall" "time" "github.com/google/uuid" @@ -55,6 +56,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper" "github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates" @@ -116,6 +118,9 @@ type Tree struct { // files that failed to assimilate, keyed by path, see updateFile() assimilationFailures *assimilation.Failures + // deletes old revisions after a change on disk created a new one, disabled without a strategy + revisionsCleanup *revisioncleanup.Service + es events.Stream log *zerolog.Logger } @@ -148,6 +153,24 @@ func New(lu node.PathLookup, bs node.Blobstore, um usermapper.Mapper, trashbin * } t.idResolver = t.lookup t.assimilateFunc = t.assimilate + // The tree creates the revision cleanup itself, as assimilation needs it for + // revisions created from changes on disk, and the watcher started below may + // assimilate right away. posix.New passes the same instance (with the same background + // workers) on to Decomposedfs through the aspects (see RevisionsCleanup), which also + // stops the workers on Shutdown. An invalid strategy or invalid async settings fail + // startup rather than silently keeping or deleting revisions the admin did not expect. + revisionsCleanup, err := revisioncleanup.NewServiceFromOptions(&o.Options, t.lookup, t, log) + if err != nil { + return nil, errors.Wrap(err, "tree: invalid revisions cleanup configuration") + } + t.revisionsCleanup = revisionsCleanup + // stop its workers again if the tree can't be set up after all + started := false + defer func() { + if !started { + _ = revisionsCleanup.Shutdown(context.Background()) + } + }() if err := t.checkStorage(); err != nil { return nil, errors.Wrap(err, "tree: unfit storage '"+o.Root+"'") } @@ -219,6 +242,7 @@ func New(lu node.PathLookup, bs node.Blobstore, um usermapper.Mapper, trashbin * }() } + started = true return t, nil } @@ -548,6 +572,26 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) // ListFolder lists the content of a folder node func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, error) { + return t.listFolder(ctx, n, true) +} + +var _ revisioncleanup.Walker = (*Tree)(nil) + +// ListFolderKnownUnchecked lists the children of n that are known to the storage, for +// maintenance, see revisioncleanup.Walker. Known children are the files and folders the storage +// has registered (assimilated), i.e. whose path resolves to a node id in the id cache. Unknown +// ones exist on disk but are not registered yet, e.g. files just created on disk directly. +// Unlike ListFolder, it checks no permissions, and it skips the unknown children instead of +// assimilating them, so that it changes nothing: they are missing from its result until they +// are assimilated, e.g. by the watcher or the next ListFolder. +func (t *Tree) ListFolderKnownUnchecked(ctx context.Context, n *node.Node) ([]*node.Node, error) { + return t.listFolder(ctx, n, false) +} + +// listFolder lists the children of n. For users, it leaves out the ones denied to the user +// in ctx and assimilates the ones not known yet; otherwise, for maintenance, it lists all +// known children, see ListFolderKnownUnchecked. +func (t *Tree) listFolder(ctx context.Context, n *node.Node, forUser bool) ([]*node.Node, error) { ctx, span := tracer.Start(ctx, "ListFolder") defer span.End() dir := n.InternalPath() @@ -623,8 +667,21 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro // silently turned into an assimilation attempt. return errors.Wrap(err, "tree: error resolving ids for "+path) } + if !forUser { + // A listing for maintenance (ListFolderKnownUnchecked, e.g. the walk of the + // on-demand revision cleanup) must not change anything, so it skips the + // item instead of assimilating it. Assimilation registers the item, with a + // new id and metadata, and with enable_fs_revisions it can even create a + // revision (e.g. for a file whose id is missing from the id cache) and run + // the revision cleanup with the driver's own strategy, which a dry run must + // not do either. The item has no revisions the storage knows of yet, so + // there is nothing to clean up; it is listed once it has been assimilated, + // e.g. by the watcher, a scan or a listing for a user. + t.log.Debug().Str("path", path).Msg("skipping an item that is unknown to the storage (not assimilated yet)") + continue + } // we don't know about this node yet, assimilate it on the fly - t.log.Info().Err(err).Str("path", path).Msg("encountered unknown entity while listing the directory. Assimilate.") + t.log.Info().Err(err).Str("path", path).Msg("encountered an item unknown to the storage while listing the directory, assimilating it") err = t.assimilateFunc(scanItem{Path: path}) if err != nil { t.log.Error().Err(err).Str("path", path).Msg("failed to assimilate node") @@ -643,8 +700,10 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro continue } - // prevent listing denied resources - if !child.IsDenied(ctx) { + // prevent listing denied resources, for users only: a listing for maintenance + // (ListFolderKnownUnchecked) has no user in the context, IsDenied would panic + // without one, and it must see all children, also the ones denied to users + if !forUser || !child.IsDenied(ctx) { if child.SpaceRoot == nil { child.SpaceRoot = n.SpaceRoot } @@ -770,6 +829,36 @@ func (t *Tree) DeleteBlob(node *node.Node) error { return t.blobstore.Delete(node) } +// RevisionsCleanup returns the revision cleanup, a disabled one without a strategy. +// It is used to hand the same instance to Decomposedfs via aspects.Aspects, so that +// uploads, restores and changes on disk are all cleaned up by the same strategy and +// share the same background workers. +func (t *Tree) RevisionsCleanup() *revisioncleanup.Service { + return t.revisionsCleanup +} + +var _ revisioncleanup.SpaceScoper = (*Tree)(nil) + +// ScopeToSpace switches the current goroutine to the group of the space with use_space_groups, +// as the space group hooks of the storage driver's middleware do for user requests, see +// revisioncleanup.SpaceScoper. The returned context carries the group, for the goroutines that +// listFolder starts. Without use_space_groups, it does nothing. +func (t *Tree) ScopeToSpace(ctx context.Context, spaceID string) (context.Context, func() error, error) { + if !t.options.UseSpaceGroups { + return ctx, func() error { return nil }, nil + } + fi, err := os.Stat(t.lookup.InternalPath(spaceID, spaceID)) + if err != nil { + return ctx, nil, errors.Wrapf(err, "could not stat the root of space %q", spaceID) + } + gid := fi.Sys().(*syscall.Stat_t).Gid + unscope, err := t.userMapper.ScopeUserByIds(-1, int(gid)) + if err != nil { + return ctx, nil, errors.Wrap(err, "failed to scope user") + } + return context.WithValue(ctx, decomposedfs.CtxKeySpaceGID, gid), unscope, nil +} + // BuildSpaceIDIndexEntry returns the entry for the space id index func (t *Tree) BuildSpaceIDIndexEntry(spaceID string) string { return spaceID diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/README.md b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/README.md new file mode 100644 index 0000000000..16bec12720 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/README.md @@ -0,0 +1,357 @@ +# decomposedfs and the drivers built on it + +This package is the shared core of three storage drivers: + +| driver | package | where file content lives | where metadata lives | +|----------------|---------------------------------|-----------------------------------------------|----------------------------------------------| +| `decomposed` | `pkg/storage/fs/decomposed` | blobs on local disk | node files in a private directory tree | +| `decomposeds3` | `pkg/storage/fs/decomposeds3` | blobs in an S3 bucket | node files in a private directory tree | +| `posix` | `pkg/storage/fs/posix` | the user's files themselves, as they appear on disk | extended attributes on those files | + +`decomposed` and `decomposeds3` "decompose" a file tree into nodes, metadata and blobs, so the files are not recognizable on disk. +`posix` keeps a normal, browsable directory tree, which may also be changed directly on disk, and keeps its own data in extended attributes and hidden directories. +Despite these differences, all three share most of their logic (the CS3 API, uploads, spaces, shares/grants, locks, revisions, quotas) through this package. + +> **Note:** `pkg/storage/utils/decomposedfs` is an older copy of this package. It is still used by the legacy `ocis` and `s3ng` drivers. +> New work goes into `pkg/storage/pkg/decomposedfs` (this package); see commit 6d021d812, "Add a new version of the decomposedfs lib which we'll use moving forward". + +## Architecture overview + +### How requests reach the storage + +```mermaid +flowchart TB + subgraph services["reva services"] + SP["storageprovider
CS3 gRPC: metadata, spaces,
shares, revisions, trash"] + DP["dataprovider
HTTP: TUS uploads, downloads"] + end + REG[("fs registry
registry.NewFuncs")] + SP -->|"getFS(driver)"| REG + DP -->|"getFS(driver)"| REG + + subgraph entry["driver entry points · pkg/storage/fs"] + DEC["decomposed.New
creates local blobstore"] + DS3["decomposeds3.New
creates S3 blobstore"] + PNEW["posix.NewDefault → posix.New
creates id caches, posix lookup,
tree, trashbin, blobstore"] + end + REG -.->|"decomposed"| DEC + REG -.->|"decomposeds3"| DS3 + REG -.->|"posix"| PNEW + + ND["decomposedfs.NewDefault
creates decomposedfs lookup,
tree, trashbin"] + DEC --> ND + DS3 --> ND + + subgraph wrap["posix only: wrapping"] + PFS["posixFS
forwards optional interfaces,
adds WarmupIDCache"] + MW["middleware.FS
space group hooks"] + PFS --> MW + end + + NEW["decomposedfs.New(options, aspects)
common constructor"] + ND -->|"aspects"| NEW + PNEW -->|"aspects"| NEW + + DFS["Decomposedfs
implements storage.FS"] + NEW --> DFS + PNEW ==>|"returned to the services"| PFS + MW ==> DFS + ND ==>|"returned to the services"| DFS + + classDef shared fill:#dbeafe,stroke:#1d4ed8,color:#0b1b3f + classDef decomposed fill:#dcfce7,stroke:#15803d,color:#0b2e17 + classDef posix fill:#ffedd5,stroke:#c2410c,color:#3b1606 + classDef caller fill:#f5f3ff,stroke:#6d28d9,color:#1e1035 + class ND,NEW,DFS shared + class DEC,DS3 decomposed + class PNEW,PFS,MW posix + class SP,DP,REG caller +``` + +### Components and extension points + +```mermaid +flowchart TB + subgraph core["shared core · pkg/storage/pkg/decomposedfs"] + DFS["Decomposedfs
CS3 logic, spaces, grants,
locks, quotas, revisions"] + UPS["upload session store
TUS sessions, Finalize,
postprocessing"] + DFS --> UPS + end + + subgraph aspects["aspects.Aspects · extension points"] + direction LR + ILU[/"node.PathLookup"/] + ITR[/"node.Tree"/] + ITB[/"trashbin.Trashbin"/] + IBS[/"node.Blobstore"/] + end + DFS --> ILU + DFS --> ITR + DFS --> ITB + UPS --> ITR + ITR --> IBS + + subgraph dfam["decomposed / decomposeds3"] + DLU["decomposedfs/lookup
id → path by computation"] + DTR["decomposedfs/tree"] + DTB["DecomposedfsTrashbin"] + BLOC["decomposed/blobstore"] + BS3["decomposeds3/blobstore"] + DISK[("local disk
root/spaces/…/nodes, blobs")] + S3[("S3 bucket")] + DLU --> DISK + DTR --> DISK + DTB --> DISK + BLOC --> DISK + BS3 --> S3 + end + + subgraph pfam["posix · pkg/storage/fs/posix"] + PLU["posix lookup
id ↔ path via id cache"] + PTR["posix tree"] + PTB["posix trashbin"] + BPX["posix/blobstore
content = the file itself"] + NATS[("NATS KV
id cache")] + PDISK[("local disk / shared fs
real files, .oc-nodes, .Trash")] + WATCH(["fs watcher
inotify, GPFS, CephFS, NATS"]) + PLU --> NATS + PTR --> PDISK + PTB --> PDISK + BPX --> PDISK + WATCH -->|"changes on disk:
assimilation, bypasses Decomposedfs"| PTR + end + + ILU -.-> DLU + ITR -.-> DTR + ITB -.-> DTB + IBS -.->|"decomposed"| BLOC + IBS -.->|"decomposeds3"| BS3 + ILU -.-> PLU + ITR -.-> PTR + ITB -.-> PTB + IBS -.-> BPX + + classDef shared fill:#dbeafe,stroke:#1d4ed8,color:#0b1b3f + classDef iface fill:#ffffff,stroke:#1d4ed8,stroke-dasharray:4 3,color:#0b1b3f + classDef decomposed fill:#dcfce7,stroke:#15803d,color:#0b2e17 + classDef posix fill:#ffedd5,stroke:#c2410c,color:#3b1606 + classDef external fill:#f3f4f6,stroke:#4b5563,color:#111827 + class DFS,UPS shared + class ILU,ITR,ITB,IBS iface + class DLU,DTR,DTB,BLOC,BS3 decomposed + class PLU,PTR,PTB,BPX posix + class DISK,S3,NATS,PDISK,WATCH external +``` + +How to read them: + +- **Colours:** blue is the shared core in this package, green the code used only by `decomposed`/`decomposeds3`, orange the code used only by `posix`, grey the external systems, and purple the callers. +- **First diagram:** + - dotted arrows are lookups by driver name in the registry; + - thin arrows show what a constructor calls or builds; + - thick arrows show the object that is handed back to the services and the chain a request then travels through. +- **Second diagram:** + - solid arrows are calls at runtime; + - dotted arrows lead from an interface to the implementation each driver plugs in through `aspects.Aspects`. +- **Two paths into the storage:** requests from the services enter through `Decomposedfs` for all drivers (posix wraps it in `posixFS` and `middleware.FS` first). Changes made directly on disk enter only through the posix watcher and the tree's assimilation, which bypasses `Decomposedfs` and the upload code entirely. +- **Not shown:** the shared helpers `permissions`, `spaceidindex`, `node`, `metadata`, `tree/propagator` and `revisioncleanup`, and the NATS event stream used for async postprocessing. They are described in the sections below. + +## From configuration to a running driver + +### Registration + +Every driver registers a constructor in the storage driver registry (`pkg/storage/fs/registry`) from an `init()` function: + +```go +// pkg/storage/fs/decomposed/decomposed.go +func init() { registry.Register("decomposed", New) } + +// pkg/storage/fs/decomposeds3/decomposeds3.go +func init() { registry.Register("decomposeds3", New) } + +// pkg/storage/fs/posix/posix.go +func init() { registry.Register("posix", NewDefault) } +``` + +`pkg/storage/fs/loader` imports all driver packages for their side effect, so importing the loader makes every driver available. + +### Who creates the driver + +Two services look up the configured driver by name in `registry.NewFuncs` and call its constructor with the driver's config map and the event stream: + +- the CS3 **storage provider** gRPC service (`internal/grpc/services/storageprovider`, `getFS`): metadata operations, listing, spaces, shares, locks, revisions, trash +- the **data provider** HTTP service (`internal/http/services/dataprovider`, `getFS`): up- and downloads, e.g. TUS uploads via the data transfer handlers in `pkg/rhttp/datatx` + +Each service creates its own instance of the driver from the same configuration, even when both run in the same process. +So anything a driver keeps in memory is not shared between the two, and shared state has to live on disk, in the metadata cache or in NATS. + +## How a driver builds itself + +The central type is `Decomposedfs` (`decomposedfs.go`). It implements `storage.FS` and the optional interfaces (TUS uploads, upload session listing, ...). +It does not touch the storage layout itself, though. Everything layout specific is delegated to a set of components that each driver supplies in an `aspects.Aspects` struct: + +```go +// aspects/aspects.go +type Aspects struct { + Lookup node.PathLookup // resolves ids and references to nodes and paths + Tree node.Tree // creates, moves, deletes nodes; blobs; revisions; propagation + Trashbin trashbin.Trashbin // lists, restores and purges trashed items + Permissions permissions.Permissions // assembles CS3 permissions from grants + EventStream events.Stream // NATS events, e.g. for async postprocessing + DisableVersioning bool + UserMapper usermapper.Mapper // maps users to unix ids (posix with space groups) + RevisionsCleanup *revisioncleanup.Service // deletes old revisions (in background workers by default), disabled without a strategy +} +``` + +`decomposedfs.New(options, aspects, log)` is the common constructor all drivers end up calling. It: + +1. calls `aspects.Tree.Setup()` to create the storage layout, +2. sets up the space indexes (`spaceidindex`: by user, group, mail, type) under `/indexes`, +3. creates the `Decomposedfs` from the aspects, +4. creates the upload session store (`upload.NewSessionStore`), which gets the same aspects, +5. calls `Trashbin.Setup(fs)` so the trashbin can call back into the storage, +6. with async uploads enabled, subscribes to the postprocessing events and starts `Decomposedfs.Postprocessing` consumers. + +### decomposed and decomposeds3: `NewDefault` + +Both drivers only create their blobstore and then hand over to `decomposedfs.NewDefault`: + +``` +decomposed.New(config) decomposeds3.New(config) + options.New(config) parseConfig(config), check S3 settings + blobstore.New(root) (local disk) blobstore.New(endpoint, bucket, ...) (S3) + \ / + `---> decomposedfs.NewDefault(config, blobstore, stream, log) + options.New(config) + lookup.New(metadata backend: messagepack (default) or xattrs) + permissions.NewPermissions(...) + tree.New(lookup, blobstore, options, permissions, id cache) + revisioncleanup.NewServiceFromOptions(options, lookup, tree) (starts the cleanup workers) + Aspects{Lookup, Tree, Permissions, Trashbin: &DecomposedfsTrashbin{}, ...} + decomposedfs.New(options, aspects) +``` + +The blobstore is the only thing that differs between the two, everything else comes from this package: +`lookup` (`decomposedfs/lookup`), `tree` (`decomposedfs/tree`) and the trashbin `DecomposedfsTrashbin` (`recycle.go`). + +### posix: its own components around the same core + +`posix.NewDefault` sets up the NATS key-value id caches posix depends on, then calls `posix.New`. That function replaces almost every aspect with a posix specific implementation: + +``` +posix.NewDefault(config) + options.New(config) posix options embed the decomposedfs options; + defaults: metadata backend "hybrid", scan debounce, inotify stats + idcache (NATS KV) id -> path cache, plus a history cache for moved/deleted ids + posix.New(options, stream, idcache, historyCache) + usermapper unix mapper with use_space_groups, null mapper otherwise + posix lookup.New(...) metadata backend "hybrid" (default) or "xattrs" + permissions.NewPermissions (shared with decomposedfs) + posix trashbin.New + Setup freedesktop.org style '.Trash' per space + posix blobstore.New(root) writes the content into the actual file + posix tree.New(...) creates the revision cleanup (and its workers), starts the fs + watcher and the initial scan, if configured + Aspects{...posix components...} + decomposedfs.New(options, aspects) the shared core, as above + middleware.NewFS(dfs, hooks...) optional space group scoping hooks + posixFS{FS: middleware, tree, um} the storage.FS that is returned +``` + +The result is layered like this: + +```mermaid +flowchart TD + SP[storageprovider / dataprovider] --> PFS[posixFS] + PFS -->|embeds storage.FS| MW[middleware.FS
space group hooks] + MW --> DFS[Decomposedfs
shared CS3 logic, uploads, spaces, grants] + DFS --> LU[posix lookup] + DFS --> TR[posix tree] + DFS --> TB[posix trashbin] + DFS --> PM[permissions] + DFS --> US[upload session store] + TR --> BS[posix blobstore] + TR --> W[watcher + assimilation] +``` + +For `decomposed` and `decomposeds3`, `NewDefault` returns the `Decomposedfs` directly, with no wrapper or middleware. + +## Delegation and overriding + +### Decomposedfs delegates layout specifics to the aspects + +Most `Decomposedfs` methods follow the same pattern: resolve a reference with the lookup, check permissions, then ask the tree to do the work. For example: + +- `CreateDir`, `Move`, `Delete`, `ListFolder` → `fs.lu.NodeFromResource` → `fs.p.AssemblePermissions` → `fs.tp.CreateDir/Move/Delete/ListFolder` +- `ListRevisions`, `DownloadRevision` → `fs.tp.ListRevisions/DownloadRevision` +- `ListRecycle`, `RestoreRecycleItem`, `PurgeRecycleItem`, `EmptyRecycle` → `fs.trashbin` +- size and etag propagation after changes → `fs.tp.Propagate` +- uploads → `fs.sessionStore` → `tp.InitNewNode`, `tp.CreateRevision`, `tp.WriteBlob` + +The interfaces that define those contracts live in `node/node.go` (`Tree`, `PathLookup`, `Blobstore`), `trashbin/trashbin.go` and `permissions/`. +They are the place to look when a feature needs driver specific behaviour. +Typically a method is added to the interface and implemented in both `decomposedfs/tree` and `fs/posix/tree`. +`Tree.ListRevisionsUnchecked` is a recent example, used by the revision cleanup (see `revisioncleanup`). + +Before adding such a method, check whether the shared code already handles both layouts. +`Node` methods work through the lookup, so they resolve paths correctly for every driver. +For example, deleting a single revision needs no driver-specific code: `Node.DeleteRevisionUnchecked` removes the revision and returns its blob id, and `Tree.DeleteBlob` deletes the blob in whichever blobstore the driver uses. +`revisioncleanup.PurgeRevisionUnchecked` combines the two for all drivers. + +Shared building blocks that both trees use instead of re-implementing them: + +- `node` – the `Node` type, its attributes, locks, grants and processing status +- `metadata` – the metadata backends (`messagepack`, `xattrs`, `hybrid`) and the file metadata cache +- `tree/propagator` – tree size and mtime propagation (used by both trees) +- `upload` – TUS upload sessions, `Finalize`, postprocessing integration +- `permissions` – CS3 permission assembly from grants and the permissions service +- `revisioncleanup` – applying the revision cleanup strategy, right away or in background workers; documented in [`revisioncleanup/README.md`](revisioncleanup/README.md) + +### What differs per component + +| aspect | decomposed / decomposeds3 | posix | +|------------|---------------------------|-------| +| **lookup** (`InternalPath`) | computed from the ids: `/spaces//nodes/`; children are found through symlinks named after the child in the parent's node directory | the real path of the file, looked up in the NATS KV id cache (id → path); `IDsForPath` goes the other way | +| **metadata** | `messagepack` (`.mpk` file next to the node, default) or `xattrs` | `hybrid` (xattrs, large values spill over into `.mpk` files in `.oc-nodes`, default) or `xattrs`; the node id is stored in an xattr on the file | +| **blobstore** | local blob directory or S3 bucket, addressed by blob id | no separate blobs: `Upload` moves or copies the uploaded file into the real file path; `Delete` is a no-op | +| **tree** | creates/moves nodes by manipulating node files and symlinks | creates/moves real files and directories; additionally watches the file system and *assimilates* external changes (new ids, metadata, revisions with `enable_fs_revisions`) | +| **revisions** | `.REV.` node file next to the node, content in a blob | full copy in `/.oc-nodes/.REV.`; with `enable_fs_revisions` a `.CURRENT` copy is kept to create revisions from changes on disk | +| **trash** | deleted nodes are renamed in place to `.T.`, and a symlink in `/spaces//trash` points to them | freedesktop.org style `/.Trash/{files,info}` with `.trashinfo` files | +| **spaces** | space roots at `/spaces/`, derived from the id | space roots are directories at the paths given by `personalspacepath_template` and `generalspacepath_template` (evaluated by the shared `CreateStorageSpace`, but only the posix lookup places space roots there) | +| **wrapping** | `Decomposedfs` returned as is | `posixFS` → `middleware.FS` → `Decomposedfs` | + +### How posix overrides and extends `Decomposedfs` + +`posixFS` embeds a `storage.FS` (the middleware wrapping `Decomposedfs`), so every `storage.FS` method is forwarded automatically. +Go only promotes the methods of the embedded *interface type*, though, not those of the concrete type behind it. +That means optional interfaces that `Decomposedfs` implements in addition to `storage.FS` would be invisible on `posixFS`. +`posixFS` therefore forwards them explicitly with type assertions, e.g. `UseIn` (`storage.ComposableFS`), `NewUpload`/`GetUpload` (`tusd.DataStore`), `ListUploadSessions` and `WithDisabledSpaces`. +It also adds posix-only methods such as `WarmupIDCache`. +When `Decomposedfs` gains a new optional interface, posix only supports it once a forwarding method is added to `posixFS`. + +Other places where posix hooks into the shared code: + +- **Middleware hooks** (`pkg/storage/utils/middleware`): with `use_space_groups`, every call first resolves the space's unix group and scopes the process to it, so files are created with the right group ownership. +- **Trashbin setup:** `Trashbin.Setup(fs)` is called twice for posix: in `posix.New` with the `posixFS` wrapper (not filled in yet at that point), and in `decomposedfs.New` with the inner `Decomposedfs`. The posix trashbin keeps the first reference and ignores the second, but currently does not use it otherwise. `DecomposedfsTrashbin` on the other hand requires a `*Decomposedfs` and uses it to implement trash operations. +- **Driver specific behaviour inside the shared code** is selected through the aspects, not through type switches. For example, the upload store calls `tp.WriteBlob`, which in posix moves the upload into the real file and updates `.CURRENT`, while in decomposedfs it uploads a blob. + +### Changes on disk (posix only) + +`decomposed` and `decomposeds3` assume they are the only writers. +posix allows changes to the files on disk, and picks them up through the watcher configured with `watch_fs`/`watch_type`: inotify, GPFS, CephFS or NATS. +It can also scan the whole tree at startup (`scan_fs`). +Detected changes are queued (`ScanDebouncer`) and *assimilated* (`tree/assimilation.go`): new files get an id and metadata, moved files keep their id via the id cache, and sizes are propagated. +With `enable_fs_revisions` a revision is created from the previous `.CURRENT` copy. +This path bypasses `Decomposedfs` and the upload code completely, so any feature that hooks into uploads (e.g. revision cleanup, postprocessing) needs a separate trigger in assimilation if it should also apply to changes on disk. + +The inotify watcher starts `inotifywait` with `KillOthers`, which runs `pkill inotifywait`. +Only one watching posix instance per host is therefore possible, which matters for tests (see `fs/posix/tree/export_test.go`). + +## Where to start reading + +- `decomposedfs.go` – `NewDefault`, `New`, the `Decomposedfs` struct, `Postprocessing` +- `aspects/aspects.go` – the extension points +- `node/node.go` – `Node`, and the `Tree`, `PathLookup` and `Blobstore` interfaces +- `upload/` – `store.go` (`CreateNodeForUpload`, `updateExistingNode`), `upload.go` (`FinishUpload*`, `Finalize`, `Cleanup`) +- `pkg/storage/fs/posix/posix.go` – how posix assembles its aspects and wraps the result +- `pkg/storage/fs/posix/tree/assimilation.go` – how changes on disk are picked up diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects/aspects.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects/aspects.go index 69ebcc93f9..c7d295f713 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects/aspects.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects/aspects.go @@ -22,6 +22,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/events" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/trashbin" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper" ) @@ -35,4 +36,9 @@ type Aspects struct { EventStream events.Stream DisableVersioning bool UserMapper usermapper.Mapper + // RevisionsCleanup deletes old revisions after new ones have been created, right away or in + // background workers; disabled without a strategy, and nil disables it as well. It is + // passed in here rather than created by Decomposedfs because posix needs the same instance + // (and the same workers) in its own tree, for revisions created from changes on disk. + RevisionsCleanup *revisioncleanup.Service } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/commit_upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/commit_upload.go index 462889799d..0a2b641613 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/commit_upload.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/commit_upload.go @@ -45,11 +45,18 @@ func (fs *Decomposedfs) MarkProcessing(ctx context.Context, ref *provider.Refere if err != nil { return err } - defer func() { + // released before the revision cleanup, or on any return before that + unlocked := false + unlockNode := func() { + if unlocked { + return + } + unlocked = true if err := unlock(); err != nil { appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not close mark-processing lock") } - }() + } + defer unlockNode() if !processing { if !n.IsProcessing(ctx) { @@ -59,7 +66,15 @@ func (fs *Decomposedfs) MarkProcessing(ctx context.Context, ref *provider.Refere if id != sessionID { return nil } - return n.RemoveXattr(ctx, prefixes.StatusPrefix) + if err := n.RemoveXattr(ctx, prefixes.StatusPrefix); err != nil { + return err + } + // The upload of this session is done: apply the revision cleanup strategy (see package + // revisioncleanup), which skipped the node while it was marked. It never fails clearing + // the mark. The node lock is released first, for the reasons given in RestoreRevision. + unlockNode() + fs.revisionsCleanup.HandleRevisionCreated(ctx, n) + return nil } if n.IsProcessing(ctx) { diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go index 22e4d65c45..020f0489a6 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go @@ -16,6 +16,10 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. +// Package decomposedfs is the shared core of the decomposed, decomposeds3 and posix +// storage drivers. Each driver supplies its storage layout through aspects.Aspects, +// see README.md in this directory for how the drivers are built and how they +// delegate to and override the shared code. package decomposedfs import ( @@ -56,6 +60,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/spaceidindex" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/timemanager" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/trashbin" @@ -129,6 +134,10 @@ type Decomposedfs struct { stream events.Stream sessionStore SessionStore + // deletes old revisions after uploads and restores created new ones, nil if + // disabled. Shared with the upload session store, see aspects.Aspects. + revisionsCleanup *revisioncleanup.Service + UserCache *ttlcache.Cache userSpaceIndex *spaceidindex.Index groupSpaceIndex *spaceidindex.Index @@ -183,6 +192,13 @@ func NewDefault(m map[string]interface{}, bs node.Blobstore, es events.Stream, l store.TLSRootCA(o.IDCache.TLSRootCACertificate), ), log) + // fail at startup on an invalid strategy or async settings rather than silently keeping + // or deleting revisions the admin did not expect. The posix driver does the same in tree.New. + revisionsCleanup, err := revisioncleanup.NewServiceFromOptions(o, lu, tp, log) + if err != nil { + return nil, errors.Wrap(err, "invalid revisions cleanup configuration") + } + aspects := aspects.Aspects{ Lookup: lu, Tree: tp, @@ -190,9 +206,16 @@ func NewDefault(m map[string]interface{}, bs node.Blobstore, es events.Stream, l EventStream: es, DisableVersioning: o.DisableVersioning, Trashbin: &DecomposedfsTrashbin{}, + RevisionsCleanup: revisionsCleanup, } - return New(o, aspects, log) + fs, err := New(o, aspects, log) + if err != nil { + // stop the workers of the revision cleanup again + _ = revisionsCleanup.Shutdown(context.Background()) + return nil, err + } + return fs, nil } // New returns an implementation of the storage.FS interface that talks to @@ -245,20 +268,21 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor } fs := &Decomposedfs{ - tp: aspects.Tree, - lu: aspects.Lookup, - trashbin: aspects.Trashbin, - o: o, - p: aspects.Permissions, - um: aspects.UserMapper, - chunkHandler: chunking.NewChunkHandler(filepath.Join(o.Root, "uploads")), - stream: aspects.EventStream, - UserCache: ttlcache.NewCache(), - userSpaceIndex: userSpaceIndex, - groupSpaceIndex: groupSpaceIndex, - mailSpaceIndex: mailSpaceIndex, - spaceTypeIndex: spaceTypeIndex, - log: log, + tp: aspects.Tree, + lu: aspects.Lookup, + trashbin: aspects.Trashbin, + o: o, + p: aspects.Permissions, + um: aspects.UserMapper, + chunkHandler: chunking.NewChunkHandler(filepath.Join(o.Root, "uploads")), + stream: aspects.EventStream, + UserCache: ttlcache.NewCache(), + revisionsCleanup: aspects.RevisionsCleanup, + userSpaceIndex: userSpaceIndex, + groupSpaceIndex: groupSpaceIndex, + mailSpaceIndex: mailSpaceIndex, + spaceTypeIndex: spaceTypeIndex, + log: log, } fs.sessionStore = upload.NewSessionStore(fs, aspects, o.Root, o.AsyncFileUploads, o.Tokens, log) if err = fs.trashbin.Setup(fs); err != nil { @@ -405,6 +429,18 @@ func (fs *Decomposedfs) handlePostprocessingEvent(ctx context.Context, event eve } } + // Apply the revision cleanup strategy (see package revisioncleanup) for async + // uploads that replaced an existing file and thus created a revision. It only + // runs now that postprocessing finished: until then the upload may still roll + // back to the revision recorded in versionID, or write its blob into a revision + // node created by a newer upload. Failed uploads are skipped, as they just + // restored their previous revision instead of creating a new one. + // Cleanup itself skips the node if another upload is still processing it. + // Pass sublog on, so that the cleanup logs with the event's context. + if !failed && isVersion { + fs.revisionsCleanup.HandleRevisionCreated(appctx.WithLogger(ctx, &sublog), n) + } + if err := events.Publish( ctx, fs.stream, @@ -493,7 +529,7 @@ func (fs *Decomposedfs) handlePostprocessingEvent(ctx context.Context, event eve sublog.Debug().Msg("node is not stuck, ignoring") return } - _, deleteErr = n.DeleteRevision(ctx, "") + _, deleteErr = n.DeleteRevisionUnchecked(ctx, "") } else { versionID := time.Unix(int64(ev.Timestamp.Seconds), int64(ev.Timestamp.Nanos)).UTC().Format(time.RFC3339Nano) deleteErr = fs.deleteRevisionFile(ctx, n, n.ID+node.RevisionIDDelimiter+versionID) @@ -661,7 +697,8 @@ func (fs *Decomposedfs) WithDisabledSpaces() storage.FS { // Shutdown shuts down the storage func (fs *Decomposedfs) Shutdown(ctx context.Context) error { - return nil + // stop the background revision cleanup workers, see revisioncleanup.Service.Shutdown + return fs.revisionsCleanup.Shutdown(ctx) } // GetQuota returns the quota available diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go index 79f6623371..795b853c4e 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go @@ -40,6 +40,7 @@ import ( "github.com/google/uuid" "github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider" "github.com/opencloud-eu/reva/v2/pkg/appctx" + "github.com/opencloud-eu/reva/v2/pkg/conversions" ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/mime" @@ -144,10 +145,35 @@ type Tree interface { CreateRevision(ctx context.Context, n *Node, version string) (string, error) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string, openReaderFunc func(md *provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error) + // ListRevisionsUnchecked lists all revisions of n without checking permissions. + // ListRevisions performs the permission check for user requests and then + // delegates to it. The unchecked variant exists for system housekeeping like + // the revision cleanup (see package revisioncleanup), which may run e.g. in + // async postprocessing where the context has no user with suitable permissions. + ListRevisionsUnchecked(ctx context.Context, n *Node) ([]*provider.FileVersion, error) Propagate(ctx context.Context, node *Node, sizeDiff int64) (err error) } +// ValidateRevisionKey makes sure that revisionKey denotes a revision of n. +// +// Callers that turn a full revision key into a version for Node.DeleteRevisionUnchecked, like +// revisioncleanup.PurgeRevisionUnchecked, use it to guard against keys that would hit something +// else, as Node.DeleteRevisionUnchecked does not check permissions: +// - a key of another node, which would delete that node's revision +// - the bare '.REV.' prefix, or the node id itself: they would turn into an +// empty version, for which Node.DeleteRevisionUnchecked deletes the node's *current* revision, +// i.e. reverts the node to its latest revision or purges the node if there is none +// - path separators, as the key is turned into a file path +func ValidateRevisionKey(n *Node, revisionKey string) error { + if !strings.HasPrefix(revisionKey, n.ID+RevisionIDDelimiter) || + len(revisionKey) == len(n.ID+RevisionIDDelimiter) || + strings.ContainsAny(revisionKey, `/\`) { + return errtypes.BadRequest(fmt.Sprintf("invalid revision key %q for node %q", revisionKey, n.ID)) + } + return nil +} + // PathLookup defines the interface for the lookup component type PathLookup interface { WithDisabledSpaces() PathLookup @@ -952,8 +978,8 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi // share indicator if _, ok := fieldMaskKeysMap["share-types"]; returnAllFields || ok { - granteeTypes := n.getGranteeTypes(ctx) - if len(granteeTypes) > 0 { + granteeTypes, hasGuestShares := n.getGranteeTypes(ctx) + if len(granteeTypes) > 0 || hasGuestShares { // TODO add optional property to CS3 ResourceInfo to transport grants? var s strings.Builder first := true @@ -965,6 +991,12 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi } s.WriteString(strconv.Itoa(int(t))) } + if hasGuestShares { + if !first { + s.WriteString(",") + } + s.WriteString(strconv.Itoa(int(conversions.ShareTypeGuest))) + } ri.Opaque = utils.AppendPlainToOpaque(ri.Opaque, "share-types", s.String()) } } @@ -1369,17 +1401,25 @@ func (n *Node) ListGrants(ctx context.Context) ([]*provider.Grant, error) { return grants, nil } -func (n *Node) getGranteeTypes(ctx context.Context) []provider.GranteeType { +// getGranteeTypes returns the grantee types of the node's grants. The second +// return value reports whether the node has guest shares, which are stored as +// user grants with a mail ACE prefix. +func (n *Node) getGranteeTypes(ctx context.Context) ([]provider.GranteeType, bool) { types := []provider.GranteeType{} + hasGuestShares := false if g, err := n.ListGrantees(ctx); err == nil { hasUserShares, hasGroupShares := false, false for i := range g { switch { - case !hasUserShares && (strings.HasPrefix(g[i], prefixes.GrantUserAcePrefix) || strings.HasPrefix(g[i], prefixes.GrantMailAcePrefix)): + case !hasGuestShares && strings.HasPrefix(g[i], prefixes.GrantMailAcePrefix): + hasGuestShares = true + case !hasUserShares && strings.HasPrefix(g[i], prefixes.GrantUserAcePrefix): hasUserShares = true case !hasGroupShares && strings.HasPrefix(g[i], prefixes.GrantGroupAcePrefix): hasGroupShares = true - case hasUserShares && hasGroupShares: + } + + if hasUserShares && hasGroupShares && hasGuestShares { break } } @@ -1390,7 +1430,7 @@ func (n *Node) getGranteeTypes(ctx context.Context) []provider.GranteeType { types = append(types, provider.GranteeType_GRANTEE_TYPE_GROUP) } } - return types + return types, hasGuestShares } // ancestorGrants returns the active grants on the node and all its ancestors. The grants on the @@ -1648,13 +1688,15 @@ func (n *Node) RevertUpload(ctx context.Context, versionID string) error { return nil } -// DeleteRevision deletes a revision of the node and returns the blob id of the +// DeleteRevisionUnchecked deletes a revision of the node and returns the blob id of the // deleted revision, so that the caller can delete the blob. If versionID is // empty the node's current revision is deleted: the latest stored revision is // restored onto the node (or the node is purged when there is none) and the // processing flag is removed, unstalling the node. No blob is returned in that // case. -func (n *Node) DeleteRevision(ctx context.Context, versionID string) (string, error) { +// +// It checks no permissions, Decomposedfs.DeleteRevision is the variant that does. +func (n *Node) DeleteRevisionUnchecked(ctx context.Context, versionID string) (string, error) { if versionID != "" { return n.deleteRevision(ctx, n.ID+RevisionIDDelimiter+versionID) } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options/options.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options/options.go index f2827eccc2..93fc8dc6f4 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options/options.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options/options.go @@ -96,6 +96,67 @@ type Options struct { DisableVersioning bool `mapstructure:"disable_versioning"` + // RevisionsCleanupStrategy defines which old revisions of a file are deleted after a new + // revision has been created, e.g. "auto", "14d, auto", "auto, 90d" or "7d, auto, 90d". + // Leaving it empty or setting it to "disabled" keeps all revisions. + // + // The supported forms, where X is a duration with a unit suffix (w, d, h, m, s) or + // a plain number of revisions, which always refers to the newest revisions: + // - "auto": thin out revisions the older they get, keeping e.g. one per 10 + // seconds in the last minute, one per minute in the last hour, one per hour in + // the last day, one per day in the last 30 days and one per week beyond + // - "X": delete everything beyond X, e.g. "10" or "30d"; "0" deletes all revisions, + // which, unlike disable_versioning, also removes the ones accumulated so far + // - "X, auto": keep everything within X, apply "auto" to the older revisions + // - "auto, X": delete everything beyond X, apply "auto" to the rest + // - "X1, X2": keep everything within X1, delete everything beyond X2; one must be + // a number and the other a duration, e.g. "5, 30d" or "7d, 10" + // - "X1, auto, X2": keep everything within X1, delete everything beyond X2 and + // apply "auto" in between, e.g. "7d, auto, 90d" + // - "disabled": never delete any revisions + // + // A number of revisions to keep counts the ones within X1 too: "7d, 10" keeps at + // most 10 revisions, unless more than 10 are younger than 7 days. + // + // The age of a revision is the time its content was written, i.e. the modification time + // the file had before it was changed, not the time of that change, which turned the + // content into a revision. A revision created today from content that was unchanged for + // two months is two months old, so e.g. "30d" deletes it right away; combine + // durations with a number of revisions, e.g. "5, 30d", to always keep the newest ones. + // + // A duration in days or weeks with an uppercase unit (D, W) is aligned to midnight in the + // server's local time zone: "1d" means the last 24 hours, "1D" since yesterday 00:00, + // "0D" since today 00:00, e.g. "1D, auto, 30d". + // + // See revisions.NewRevisionCleanupStrategy in pkg/storage/revisions for the exact + // grammar and pkg/storage/pkg/decomposedfs/revisioncleanup for when it is applied. + // It is shared by all decomposedfs based drivers, including posix, and also applies + // to revisions posix creates from changes on disk (enable_fs_revisions). + // An invalid value makes the driver fail at startup. + RevisionsCleanupStrategy string `mapstructure:"revisions_cleanup_strategy"` + + // RevisionsCleanupAsync runs the revision cleanup in the background instead of as part of + // the upload, restore or change on disk that created the new revision, so that those are not + // slowed down by it. Defaults to true. + RevisionsCleanupAsync bool `mapstructure:"revisions_cleanup_async"` + // RevisionsCleanupAsyncWorkers is the number of background workers for the revision cleanup. + // 0, the default, determines it automatically: half the available CPUs (which respects + // container CPU limits), but at least 2. + RevisionsCleanupAsyncWorkers int `mapstructure:"revisions_cleanup_async_workers"` + // RevisionsCleanupAsyncQueueSize is the number of nodes that can wait for a background + // revision cleanup. 0, the default, determines it automatically from the number of workers. + RevisionsCleanupAsyncQueueSize int `mapstructure:"revisions_cleanup_async_queue_size"` + // RevisionsCleanupAsyncQueueFull defines what happens when a new revision is created while + // the queue is full: "drop" (the default) skips the cleanup of that node and logs it, the + // node is cleaned up again when it gets its next revision; "block" waits until there is + // room, for at most RevisionsCleanupAsyncQueueFullTimeout, so that no cleanup is skipped + // unless the queue stays full that long, at the cost of slowing down writes under heavy + // load. + RevisionsCleanupAsyncQueueFull string `mapstructure:"revisions_cleanup_async_queue_full"` + // RevisionsCleanupAsyncQueueFullTimeout is how long "block" waits for room in a full queue, + // before the cleanup of the node is skipped as with "drop". 0 waits as long as it takes. + RevisionsCleanupAsyncQueueFullTimeout time.Duration `mapstructure:"revisions_cleanup_async_queue_full_timeout"` + MountID string `mapstructure:"mount_id"` MultiTenantEnabled bool `mapstructure:"multi_tenant_enabled"` @@ -121,7 +182,12 @@ type TokenOptions struct { // New returns a new Options instance for the given configuration func New(m map[string]interface{}) (*Options, error) { - o := &Options{} + o := &Options{ + // defaults for options that are not false/0/"" by default; mapstructure only + // overwrites the fields that are present in m + RevisionsCleanupAsync: true, + RevisionsCleanupAsyncQueueFullTimeout: 5 * time.Second, + } if err := mapstructure.Decode(m, o); err != nil { err = errors.Wrap(err, "error decoding conf") return nil, err diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup.go new file mode 100644 index 0000000000..b74a8a8bdb --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup.go @@ -0,0 +1,186 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package decomposedfs + +import ( + "context" + "fmt" + "maps" + "math/rand/v2" + "os" + "slices" + "strings" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/pkg/errors" + + "github.com/opencloud-eu/reva/v2/pkg/errtypes" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup" +) + +var _ revisioncleanup.OnDemand = (*Decomposedfs)(nil) + +// CleanupRevisionsUnchecked cleans up the revisions of the files of whole spaces, or of a file or +// folder within a space, on demand, see revisioncleanup.OnDemand. It is meant for command line +// tools: it checks no permissions and needs no user in the context. +// +// With Request.All, a space that can't be cleaned up doesn't stop the others: a disabled space +// (in the trash) is reported as skipped, any other one as failed. A space requested by its id +// fails the request instead. +// +// Paths on disk (Request.FSPath) are not supported: the decomposed layout stores files under +// their ids. The posix driver resolves them before it calls this. +// +// The strategy and the AsyncOptions default to the configured ones (the +// revisions_cleanup_* options) and can be overridden in the request; without AsyncOptions in +// the request, the run waits for room in a full queue as long as it takes, see +// revisioncleanup.Request. +func (fs *Decomposedfs) CleanupRevisionsUnchecked(ctx context.Context, req revisioncleanup.Request, report func(revisioncleanup.Result)) error { + if err := req.Validate(); err != nil { + return err + } + if req.FSPath != "" { + return errtypes.NotSupported("cleaning up revisions by a path on disk, only the posix driver supports it") + } + w, ok := fs.tp.(revisioncleanup.Walker) + if !ok { + return errtypes.NotSupported("cleaning up revisions on demand, the tree can't list folders for maintenance") + } + if report == nil { + report = func(revisioncleanup.Result) {} + } + + strategy := req.Strategy + if strategy == "" { + strategy = fs.o.RevisionsCleanupStrategy + } + cleaner, err := revisioncleanup.New(strategy) + if err != nil { + return errors.Wrap(err, "invalid revisions cleanup strategy") + } + if !cleaner.Enabled() { + return errors.New("no revisions cleanup strategy configured or given") + } + o := revisioncleanup.RunOptions{Async: revisioncleanup.AsyncOptionsFromOptions(fs.o), DryRun: req.DryRun, Shuffle: req.Shuffle} + // a run queues the files as fast as it finds them: dropping them, or giving up after + // the timeout meant for uploads, would skip most of them + o.Async.QueueFull, o.Async.QueueFullTimeout = revisioncleanup.QueueFullBlock, 0 + if req.Async != nil { + o.Async = *req.Async + } + + roots, err := fs.revisionCleanupRoots(ctx, req, report) + if err != nil { + return err + } + return revisioncleanup.Run(ctx, fs.lu, w, roots, cleaner, o, report, fs.log) +} + +// errSpaceDisabled is the error for a space that is disabled, i.e. in the trash +var errSpaceDisabled = errors.New("the space is disabled") + +// revisionCleanupRoots resolves the scope of the request to the nodes to clean up below. With +// Request.All, it reports the spaces that can't be cleaned up and leaves them out. +func (fs *Decomposedfs) revisionCleanupRoots(ctx context.Context, req revisioncleanup.Request, report func(revisioncleanup.Result)) ([]*node.Node, error) { + // a space given more than once is cleaned up once + spaceIDs := uniqueInOrder(req.SpaceIDs) + if req.All { + ids, err := fs.allSpaceIDs() + if err != nil { + return nil, err + } + spaceIDs = ids + } + // Request.Shuffle randomizes the order in two places: here, the order of the spaces, and in + // revisioncleanup.Run, the order of the files and folders within each folder it walks + // through (RunOptions.Shuffle). + if req.Shuffle { + spaceIDs = slices.Clone(spaceIDs) + rand.Shuffle(len(spaceIDs), func(i, j int) { spaceIDs[i], spaceIDs[j] = spaceIDs[j], spaceIDs[i] }) + } + + roots := make([]*node.Node, 0, len(spaceIDs)) + for _, spaceID := range spaceIDs { + n, err := fs.revisionCleanupRoot(ctx, spaceID, req.Path) + switch { + case err == nil: + roots = append(roots, n) + case !req.All: + return nil, err + case errors.Is(err, errSpaceDisabled): + report(revisioncleanup.Result{SpaceID: spaceID, NodeID: spaceID, DryRun: req.DryRun, Skipped: errSpaceDisabled.Error()}) + default: + // one space that can't be cleaned up doesn't stop the others + report(revisioncleanup.Result{SpaceID: spaceID, NodeID: spaceID, DryRun: req.DryRun, Err: err}) + } + } + return roots, nil +} + +// revisionCleanupRoot resolves the space, or the file or folder at path within it. +func (fs *Decomposedfs) revisionCleanupRoot(ctx context.Context, spaceID, path string) (*node.Node, error) { + var n *node.Node + var err error + if path == "" { + n, err = fs.lu.NodeFromSpaceID(ctx, spaceID) + } else { + n, err = fs.lu.NodeFromResource(ctx, &provider.Reference{ + ResourceId: &provider.ResourceId{SpaceId: spaceID, OpaqueId: spaceID}, + Path: "./" + strings.TrimLeft(path, "/"), + }) + } + if err == nil && n != nil && n.Exists { + return n, nil + } + // the lookups don't find disabled spaces, tell them apart from missing ones + if root, rerr := node.ReadNode(ctx, fs.lu, spaceID, spaceID, "", true, nil, false); rerr == nil && root.Exists && root.IsDisabled(ctx) { + return nil, fmt.Errorf("space %q: %w", spaceID, errSpaceDisabled) + } + switch { + case err != nil: + return nil, errors.Wrapf(err, "could not find space %q", spaceID) + case path != "": + return nil, fmt.Errorf("no file or folder %q in space %q", path, spaceID) + default: + return nil, fmt.Errorf("no space %q", spaceID) + } +} + +// allSpaceIDs lists the ids of all spaces from the space type index, of all space types, +// without permission checks. +func (fs *Decomposedfs) allSpaceIDs() ([]string, error) { + spaceTypes, err := fs.spaceTypeIndex.Indexes() + if err != nil { + return nil, errors.Wrap(err, "could not list the space types") + } + // a space can be in the index of more than one type, collect each once + unique := map[string]struct{}{} + for _, spaceType := range spaceTypes { + spaces, err := fs.spaceTypeIndex.Load(spaceType) + if err != nil { + if os.IsNotExist(err) { + continue // no spaces of this type + } + return nil, errors.Wrapf(err, "could not read the index of %q spaces", spaceType) + } + for spaceID := range spaces { + unique[spaceID] = struct{}{} + } + } + return slices.Sorted(maps.Keys(unique)), nil +} + +// uniqueInOrder returns ids without duplicates, in the order of their first occurrence. +func uniqueInOrder(ids []string) []string { + seen := make(map[string]struct{}, len(ids)) + unique := make([]string, 0, len(ids)) + for _, id := range ids { + if _, ok := seen[id]; !ok { + seen[id] = struct{}{} + unique = append(unique, id) + } + } + return unique +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/README.md b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/README.md new file mode 100644 index 0000000000..bc08472796 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/README.md @@ -0,0 +1,152 @@ +# Revision cleanup + +Every change to a file keeps the previous content as a revision (version). +Without a cleanup, revisions accumulate forever and use up storage. +The revision cleanup deletes old revisions according to a configurable strategy, whenever a file gets a new revision. + +It works for the `decomposed`, `decomposeds3` and `posix` storage drivers. +With posix it also covers the revisions created from changes made directly on disk (`enable_fs_revisions`). +For how those drivers are structured, see [`../README.md`](../README.md); for implementation details, see the package documentation in `cleanup.go`. + +## Configuration + +The options are part of the decomposedfs options and therefore apply to all three drivers. + +| Option | Default | Meaning | +|---|---|---| +| `revisions_cleanup_strategy` | `""` (disabled) | Which revisions to delete, see below. Empty or `disabled` keeps all revisions. | +| `revisions_cleanup_async` | `true` | Run the cleanup in background workers, so that it doesn't slow down the upload, restore or change that created the new revision. `false` runs it right away. | +| `revisions_cleanup_async_workers` | `0` (automatic) | Number of background workers. Automatic: half the available CPUs (respecting container CPU limits), at least 2. | +| `revisions_cleanup_async_queue_size` | `0` (automatic) | Number of files that can wait for a background cleanup. Automatic: 64 per worker. | +| `revisions_cleanup_async_queue_full` | `drop` | What happens when the queue is full: `drop` skips the cleanup of that file (logged), it is cleaned up with its next revision; `block` waits for room, up to `revisions_cleanup_async_queue_full_timeout`. | +| `revisions_cleanup_async_queue_full_timeout` | `5s` | How long `block` waits for room at most, after which the cleanup of that file is skipped as with `drop`. `0` waits as long as it takes, so that no cleanup is ever skipped, but then writes can slow down without bound under heavy load. | + +An invalid value in any of these options makes the storage driver fail at startup. + +## Strategies + +A strategy is a comma separated list of up to three parts. +Each `X` is either + +- a plain number, which is a **number of revisions**, always counting from the newest one, or +- a number with a unit suffix, which is a **duration**: `w` weeks, `d` days, `h` hours, `m` minutes, `s` seconds. + Durations in days or weeks can use an uppercase unit, `D` or `W`, to count from midnight (server time) instead of from now: + `1d` means the last 24 hours, `1D` means since yesterday 00:00, `0D` since today 00:00. + +| Strategy | Meaning | Example | +|---|---|---| +| `auto` | Thin out revisions the older they get (see below) | `auto` | +| `X` | Delete everything beyond `X`, no thinning | `10` keeps the 10 newest revisions, `30d` deletes revisions older than 30 days, `0` deletes all revisions | +| `X, auto` | Keep everything within `X`, thin out the older revisions | `14d, auto` | +| `auto, X` | Delete everything beyond `X`, thin out the rest | `auto, 90d`, `auto, 50` | +| `X1, X2` | Keep everything within `X1`, delete everything beyond `X2`; one must be a number and the other a duration | `5, 30d` deletes revisions older than 30 days but always keeps the 5 newest; `7d, 10` keeps at most 10 revisions but never deletes revisions younger than 7 days | +| `X1, auto, X2` | Keep everything within `X1`, delete everything beyond `X2`, thin out in between | `7d, auto, 90d`, `2, auto, 50`, `1D, auto, 30d` | +| `disabled` | Never delete revisions (same as empty) | `disabled` | + +Rules worth knowing: + +- **`auto` thins out revisions** by keeping one revision per time slot. The slots get longer the older the revisions are: + + | Age of the revision | Kept | + |---|---| + | last minute | one per 10 seconds | + | last hour | one per minute | + | last 24 hours | one per hour | + | last 30 days | one per day | + | older | one per week | + +- **A number of revisions is a total**, the protected revisions included: `7d, 10` keeps 10 revisions in total, not 10 on top of the ones from the last 7 days. Revisions within the protected range are never deleted, though, even if there are more of them. +- **In `auto, N`, the number `N` applies first, then `auto` thins out what is left.** So `auto, 10` keeps *at most* 10 revisions, usually fewer, and sometimes fewer than `auto` alone: if the 10 newest revisions were all saved within the same minute yesterday, `auto, 10` keeps a single revision, while `auto` keeps one of those plus one per day before. It does *not* mean "thin out first, then keep the 10 newest of what is left". The same goes for a number in `X1, auto, N`. +- **Two values of the same kind are rejected** in `X1, X2`, because one of them would have no effect: `5, 10` is the same as `10`, and `7d, 30d` is the same as `30d`. The error message names the equivalent single value. +- **In `X1, auto, X2` with two values of the same kind, `X1` must be smaller than `X2`**, otherwise `auto` would never apply. +- **`0` deletes all revisions.** Unlike `disable_versioning`, which only stops creating new revisions, this also removes the ones that have accumulated, over time (see [When the cleanup runs](#when-the-cleanup-runs) below). + +## How the age of a revision is measured + +A revision's age is the time **its content was written**, i.e. the modification time the file had before it was changed. +That is also the time the revision is named after on disk, and the date the version list shows for it. +It is **not** the time the file was changed, which turned that content into a revision; that time is not recorded anywhere. +Revisions are never modified after they were created; the point is only which of the two times the cleanup looks at. + +For files that change often, both dates are close together. For files that sit unchanged for a long time, they are far apart: + +1. On August 14, a document is saved. +2. Nobody touches it for two months. +3. On October 14, it is changed. A revision is created to keep the previous content, dated August 14. +4. The cleanup runs right after the change. The revision counts as 61 days old. + +With a strategy that deletes by duration, e.g. `30d`, `auto, 30d` or `7d, 30d`, that revision is deleted right away, and the content the document had until that morning is gone. +To always keep the most recent versions of a file regardless of their age, combine the duration with a number of revisions, e.g. `5, 30d`, or use `auto`, which keeps one revision per week even for old content. + +## When the cleanup runs + +The cleanup of a file runs after the file got a new revision: + +- after an upload replaced an existing file, once the upload is complete (with async postprocessing: once postprocessing finished successfully), +- after a revision was restored; the revision holding the content from before the restore is always kept, +- with posix and `enable_fs_revisions`, after a change on disk created a revision. + +The cleanup skips a file while an upload to it is still being processed, as that upload may still need its revisions. +Only the latest upload of a file is tracked, though: if an older upload is still being processed after a newer one finished, the cleanup runs anyway. +That is safe: if it deletes the revision the older upload writes its content into, the older upload creates it again when it finishes, and the revision the older upload would roll back to after a failed postprocessing must not be restored anyway once a newer upload finished, as that would discard the newer upload's content. + +Files that are never changed again keep their revisions. To clean those up as well, or to apply a different strategy once, the cleanup can also run on demand for whole spaces, folders or files: in OpenCloud with the command `opencloud storage-users revisions cleanup`. + +## Metrics + +The cleanup exports Prometheus metrics, all named `reva_revision_cleanup_*`. + +| Metric | Type | Meaning | +|---|---|---| +| `reva_revision_cleanup_enabled` | gauge | 1 if a strategy is configured, 0 otherwise | +| `reva_revision_cleanup_revisions_deleted_total` | counter | Deleted revisions | +| `reva_revision_cleanup_bytes_deleted_total` | counter | Total size of the deleted revisions | +| `reva_revision_cleanup_purge_failures_total` | counter | Revisions that could not be deleted, which are retried with the file's next revision, and revisions whose blob could not be deleted afterwards, which is left behind (logged with its blob id) | +| `reva_revision_cleanup_queue_full_total{outcome}` | counter | Requests that found the queue full, by `outcome`: `blocked` (the writer waited) or `dropped` (cleanup skipped); a writer that waited until the timeout of `block` counts as both | +| `reva_revision_cleanup_queue_length` | gauge | Files waiting for a background cleanup, including writers waiting for room in a full queue | +| `reva_revision_cleanup_queue_capacity` | gauge | Size of the queue | +| `reva_revision_cleanup_workers` | gauge | Background workers | +| `reva_revision_cleanup_running` | gauge | Files being cleaned up by workers right now | + +Things to keep in mind: + +- **The queue holds files, not revisions.** A file waits at most once, however many revisions it got in the meantime, and how many of its revisions get deleted is only decided when a worker processes it. +- **One process can have several queues.** The storageprovider and the dataprovider each create their own storage driver, and with it their own cleanup. The queue and worker gauges are the sum over all of them. + Because a sum can hide one queue being full while another one is empty, `queue_full_total` counts every request that found its queue full. + It is the most direct sign that there are too few workers or the queue is too small. +- **On-demand cleanups only count their deletions.** A cleanup of whole spaces, folders or files on demand uses a queue and workers of its own, which the queue, worker and `enabled` metrics leave out, as they describe the cleanup that runs whenever a file gets a new revision. The revisions it deletes are counted in `revisions_deleted_total`, `bytes_deleted_total` and `purge_failures_total`. + +Example queries: + +```promql +# revisions deleted per second, and storage freed per second +rate(reva_revision_cleanup_revisions_deleted_total[5m]) +rate(reva_revision_cleanup_bytes_deleted_total[5m]) + +# how full the queues are (0..1, can exceed 1 while writers wait for room) +reva_revision_cleanup_queue_length / reva_revision_cleanup_queue_capacity + +# requests hitting a full queue, worth an alert if it doesn't stay at 0 +sum by (outcome) (rate(reva_revision_cleanup_queue_full_total[5m])) + +# how busy the workers are (0..1) +reva_revision_cleanup_running / reva_revision_cleanup_workers + +# failed deletions, worth an alert +increase(reva_revision_cleanup_purge_failures_total[1h]) > 0 +``` + +## Logging + +- At startup with background workers: one info line `revision cleanup: running in the background` with the strategy, workers, queue size, queue full policy and timeout. +- Per cleaned file: one info line `revision cleanup: purged old revisions` with the strategy (`config`), the number and size of the deleted revisions, and how many revisions the file had. Nothing is logged for files where nothing was deleted. +- Every deleted revision is logged at debug level (`revision cleanup: purged revision`), with its `size` and the `reason`: which rule of the strategy deleted it, and the boundary it was evaluated with. This is particularly useful to understand the time slots of `auto`. Examples: + - `deleted by "30d": older than 2026-09-14T10:00:00Z` + - `deleted by "10": beyond the 10 newest, counting the 3 protected ones` + - `deleted by "auto" (last hour): its slot of 1 minute starting 2026-10-14T09:05:00Z already keeps a newer revision` +- A revision dated in the future is logged as a warning (`revision cleanup: revision dated in the future, it counts as the newest one`), every time its file is cleaned up, and its reason ends in `dated in the future (...), so it counts as the newest`. Its time is the modification time of its content, which comes from the client or from the change on disk, e.g. a client with a wrong clock. Such a revision takes the first places of a number of revisions to keep, and no duration deletes it before its date has passed, which can make the cleanup delete the actual newest revisions instead: check the client or the clock. +- With `drop`, or when `block` gives up after its timeout, a skipped cleanup is logged as a warning; on shutdown, the number of abandoned queued cleanups is logged. + +## Limitations + +- **Shutdown:** cleanups still waiting in the queue are abandoned when the service stops. They are picked up again with the files' next revisions. diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/cleanup.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/cleanup.go new file mode 100644 index 0000000000..283b1f478d --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/cleanup.go @@ -0,0 +1,498 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +// Package revisioncleanup applies a revisions.RevisionCleanupStrategy to the +// revisions of a node, independently of the storage driver that holds them. +// +// # How the storage drivers plug in +// +// All decomposedfs based drivers (decomposed, decomposeds3, posix) share the +// Decomposedfs struct, which implements the CS3 API, the upload sessions and +// RestoreRevision. What differs per driver is the on-disk layout, which each +// driver supplies as a node.Tree through aspects.Aspects: +// +// - decomposed and decomposeds3 use decomposedfs/tree.Tree. A revision is an +// empty node file next to the node, '.REV.', whose +// metadata references the content as a blob in the blobstore (BlobIDAttr). +// The two drivers differ only in the blobstore (local disk or S3). +// - posix uses fs/posix/tree.Tree. A revision is a full copy of the content in +// the '.oc-nodes' directory of the space (lookup.VersionPath), the posix +// blobstore does not hold any separate blobs. +// +// In both cases the revision key is '.REV.', with an +// optional '.' suffix when CreateRevision had to rename a colliding revision. +// +// See pkg/storage/pkg/decomposedfs/README.md for a more detailed overview of the +// driver structure. +// +// The shared code only knows node.Tree. Listing revisions is driver specific +// (node.Tree.ListRevisionsUnchecked), but deleting one is not: Node.DeleteRevisionUnchecked handles +// both layouts, and node.Tree.DeleteBlob deletes the blob in whichever blobstore the +// driver uses. PurgeRevisionUnchecked combines the two. So both the decision about *what* to +// delete and the deletion itself are made here, once for all drivers. +// Neither ListRevisionsUnchecked nor PurgeRevisionUnchecked check permissions: this is system +// housekeeping, which runs e.g. in async postprocessing where there is no suitable +// user in the context. +// +// # When the cleanup runs +// +// Service.HandleRevisionCreated is called after a new revision has been created and has +// settled: +// +// - after a successful synchronous upload (upload.DecomposedFsSession.FinishUploadDecomposed), +// - after successful async postprocessing (Decomposedfs.Postprocessing), +// - in the upload pipeline: when Decomposedfs.MarkProcessing clears the processing mark +// of an upload, +// - after Decomposedfs.RestoreRevision, +// - in posix, when a change on disk turned the '.CURRENT' copy into a revision +// (assimilation). Those revisions are created by a plain rename and never go +// through CreateRevision or the upload code, hence the separate trigger. +// +// Besides, the revisions of whole spaces, folders or single files can be cleaned up on demand, +// e.g. by a command line tool, through the OnDemand interface implemented by the storage drivers +// (see Request and Run). That cleanup also reaches files that never get a new revision anymore. +// +// It deliberately does not run inside Tree.CreateRevision, because revisions are +// not settled at that point while uploads are in flight: +// +// - An upload whose postprocessing is still pending writes its blob into a +// revision node that a newer upload created for it in the meantime +// (see DecomposedFsSession.Finalize, "uploading to revision node, that was +// created for us by another upload"). +// - A failed upload rolls back by restoring the revision recorded in its +// session's MetaData["versionID"]. +// +// The 'auto' strategy keeps only one revision per 10 seconds in the last minute, +// so with rapid uploads a cleanup inside CreateRevision could delete a revision +// that one of those uploads still needs. Instead, Cleanup does nothing at all +// while the node is marked as processing. +// +// # Running in the background +// +// By default (option revisions_cleanup_async) HandleRevisionCreated only queues the node, and +// background workers run Cleaner.Cleanup, so that the cleanup does not slow down the +// upload, restore or change on disk that triggered it (see Service). This also +// shortens the time the node lock is held by those operations. The number of workers, +// the queue size and what happens when the queue is full (block or drop) are +// configurable (AsyncOptions). With revisions_cleanup_async disabled, HandleRevisionCreated runs the +// cleanup right away, in the caller's goroutine. +// +// # Metrics +// +// The cleanup exports Prometheus metrics (see metrics.go), all named +// reva_revision_cleanup_*: counters of deleted revisions, deleted bytes, failed deletions +// and requests that found the queue full, and gauges of the queue length and capacity, the +// workers, the running cleanups and whether the cleanup is enabled. The queue and worker +// gauges are summed over all Services of the process. +// +// Overlapping uploads: the processing status only tracks the latest upload of a node, so a +// cleanup triggered by a newer upload doesn't see an older one that is still in +// postprocessing. That is safe: if the cleanup deletes the revision the older upload writes +// its content into, the older upload creates it again when it finishes; and the revision +// the older upload would roll back to after failed postprocessing must not be restored once +// a newer upload has finished, as that would discard the newer upload's content. +package revisioncleanup + +import ( + "context" + "fmt" + "slices" + "strings" + "time" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + + "github.com/opencloud-eu/reva/v2/pkg/appctx" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" + "github.com/opencloud-eu/reva/v2/pkg/storage/revisions" +) + +// BlobDeleter is the part of node.Tree needed to purge a revision. +type BlobDeleter interface { + DeleteBlob(n *node.Node) error +} + +// RevisionStore is the part of node.Tree needed to clean up revisions. +// It is a separate interface so that the posix tree can pass itself from within +// its own package without the cleanup depending on the full node.Tree. +type RevisionStore interface { + ListRevisionsUnchecked(ctx context.Context, n *node.Node) ([]*provider.FileVersion, error) + BlobDeleter +} + +// SpaceScoper is implemented by trees whose files may only be accessible to the group of their +// space, i.e. posix with use_space_groups. The cleanup runs on behalf of the system, e.g. in +// background workers, so nothing has switched to that group for it, unlike for user requests, +// where the storage driver's middleware does. +type SpaceScoper interface { + // ScopeToSpace switches the current goroutine to the group of the space, and returns a + // context carrying the group, for the goroutines the tree starts itself, e.g. to list a + // folder, and a function that switches back. The goroutine stays on its OS thread until then. + ScopeToSpace(ctx context.Context, spaceID string) (context.Context, func() error, error) +} + +// scopeToSpace switches to the group of the space of n if rs is a SpaceScoper, see there. It +// returns ctx and a no-op otherwise. +func scopeToSpace(ctx context.Context, rs any, spaceID string) (context.Context, func() error, error) { + scoper, ok := rs.(SpaceScoper) + if !ok { + return ctx, func() error { return nil }, nil + } + return scoper.ScopeToSpace(ctx, spaceID) +} + +// PurgeRevisionUnchecked physically removes the revision of n with the given key, including its +// blob, metadata and lockfile, without checking permissions. Callers that act on behalf +// of a user must check permissions themselves. +// +// It works for all decomposedfs based drivers: +// - decomposed, decomposeds3: a revision is the empty node file '.REV.' +// with its metadata and lockfile, plus the blob holding the content, referenced by +// BlobIDAttr in the revision's metadata. +// - posix: a revision is a full copy of the content in the '.oc-nodes' directory of the +// space, with its metadata and lockfile. Revisions created from changes on disk by +// assimilation have the same layout. The posix blobstore does not hold separate blobs, +// its Delete is a no-op. +// +// Node.DeleteRevisionUnchecked removes everything but the blob, under the node's metadata lock +// (which is reentrant, callers may hold it), and returns the blob id, so that the blob can be +// deleted here. It is a no-op for a revision that does not exist (anymore), e.g. when it was purged +// concurrently. +// +// The key is validated first (node.ValidateRevisionKey), as it is turned into the version that +// Node.DeleteRevisionUnchecked expects, and an empty version would make +// Node.DeleteRevisionUnchecked revert or even purge the node itself instead of deleting a revision. +// +// The blob of a revision is never shared with the current version or other revisions: +// every upload gets a new blob id, CreateRevision moves the reference of the previous +// blob to the revision, and RestoreRevision moves it back while removing the revision +// without deleting its blob. So deleting it here cannot affect anything else. +// +// Decomposedfs.deleteRevisionFile, used by the DeleteFileVersions API and the DeleteRevision +// event, does the same without the key validation. It is left as it is to keep that code +// identical to upstream. +func PurgeRevisionUnchecked(ctx context.Context, bd BlobDeleter, n *node.Node, revisionKey string) error { + blobID, err := removeRevisionUnchecked(ctx, n, revisionKey) + if err != nil { + return err + } + return deleteBlob(bd, n.SpaceID, blobID) +} + +// removeRevisionUnchecked removes the revision of n with the given key, but not its blob, and +// returns the blob id, see PurgeRevisionUnchecked. It checks no permissions. +func removeRevisionUnchecked(ctx context.Context, n *node.Node, revisionKey string) (string, error) { + if err := node.ValidateRevisionKey(n, revisionKey); err != nil { + return "", err + } + return n.DeleteRevisionUnchecked(ctx, strings.TrimPrefix(revisionKey, n.ID+node.RevisionIDDelimiter)) +} + +// deleteBlob deletes the blob of a removed revision. An empty blob id means there is no blob: +// a zero byte revision, or no such revision. +func deleteBlob(bd BlobDeleter, spaceID, blobID string) error { + if blobID == "" { + return nil + } + // DeleteBlob only needs the blob coordinates, not a full node + return bd.DeleteBlob(&node.Node{BaseNode: node.BaseNode{SpaceID: spaceID}, BlobID: blobID}) +} + +// Cleaner deletes old revisions according to a cleanup strategy. +// A disabled Cleaner, created by New without a strategy, never deletes anything; see +// Enabled. A nil *Cleaner behaves the same, as a safeguard, but New never returns one. +// +// It is stateless apart from the parsed strategy and safe for concurrent use. +// The storage drivers use it through a Service, created from their options by +// NewServiceFromOptions and passed on through aspects.Aspects.RevisionsCleanup. +type Cleaner struct { + // nil if the cleanup is disabled + strategy revisions.RevisionCleanupStrategy + // the specification the strategy was parsed from, logged so that support can tell + // which configuration was in effect from the log output alone + config string +} + +// New creates a Cleaner from a strategy specification as understood by +// revisions.NewRevisionCleanupStrategy. An empty or "disabled" specification +// returns a disabled Cleaner. +// +// A disabled Cleaner, rather than nil, lets the call sites call Cleanup unconditionally, +// without the risk of a nil pointer, while it still skips all the work, including taking +// the node lock, when the feature is off, which is the default. +func New(config string) (*Cleaner, error) { + config = strings.TrimSpace(config) + if config == "" || config == "disabled" { + return &Cleaner{config: "disabled"}, nil + } + strategy, err := revisions.NewRevisionCleanupStrategy(config) + if err != nil { + return nil, err + } + return &Cleaner{strategy: strategy, config: config}, nil +} + +// Enabled tells whether c cleans up anything, i.e. whether it was created with a strategy. +// It is false for a disabled Cleaner, and for a nil one. +func (c *Cleaner) Enabled() bool { + return c != nil && c.strategy != nil +} + +// keptOnRequest is the reason for revisions that the operation triggering the cleanup asked +// to keep, see Cleanup. +const keptOnRequest = "protected: kept on request of the operation that triggered the cleanup (e.g. the state from before a restore)" + +// Cleanup deletes the revisions of n that the strategy does not want to keep. +// Revisions with a key in keep are never deleted. +// +// It locks the metadata of n (the lock is reentrant, callers may already hold it) +// and does nothing while an upload to n is still being processed, as that upload +// may still need existing revisions to write its blob into or to roll back to. +// See the package documentation for the details. The blobs of the deleted revisions are +// deleted after the lock is released. If rs is a SpaceScoper, it switches to the group of the +// space of n first. +// +// Errors are logged, and reported in the Result, but not returned: a failed cleanup must +// not fail the operation that triggered it. Callers that don't report on the cleanup can +// ignore the Result. +// +// It checks no permissions: it is housekeeping on behalf of the system, e.g. after async +// postprocessing, where there is no suitable user in the context. Callers decide which nodes +// to clean up. +func (c *Cleaner) Cleanup(ctx context.Context, lu node.PathLookup, rs RevisionStore, n *node.Node, keep ...string) Result { + return c.run(ctx, lu, rs, n, false, keep) +} + +// DryRun decides like Cleanup, but deletes nothing: the Result tells which revisions would +// be deleted, kept or protected, and why. It neither counts metrics nor logs purges. +// +// Like Cleanup, it checks no permissions. +func (c *Cleaner) DryRun(ctx context.Context, lu node.PathLookup, rs RevisionStore, n *node.Node, keep ...string) Result { + return c.run(ctx, lu, rs, n, true, keep) +} + +// The result is named, so that the deletion of the blobs, which runs deferred, can report its +// errors in it. +func (c *Cleaner) run(ctx context.Context, lu node.PathLookup, rs RevisionStore, n *node.Node, dryRun bool, keep []string) (result Result) { + if !c.Enabled() || n == nil { + return Result{DryRun: dryRun, Skipped: "the revision cleanup is disabled"} + } + result = Result{SpaceID: n.SpaceID, NodeID: n.ID, DryRun: dryRun} + log := appctx.GetLogger(ctx).With().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Logger() + + ctx, unscope, err := scopeToSpace(ctx, rs, n.SpaceID) + if err != nil { + log.Error().Err(err).Msg("revision cleanup: could not switch to the group of the space") + result.Err = fmt.Errorf("could not switch to the group of the space: %w", err) + return result + } + defer func() { _ = unscope() }() + + // The blobs of the purged revisions are deleted once the node lock is released (this defer + // runs after the one releasing it): their revisions are gone by then, so nothing refers to + // them anymore, and deleting a blob can be slow, e.g. one request per blob with + // decomposeds3, which would block uploads to the node all along. + type pendingBlob struct { + key string // of the revision the blob belonged to + blobID string + size uint64 // of the revision + } + var blobs []pendingBlob + defer func() { + for _, b := range blobs { + if err := deleteBlob(rs, n.SpaceID, b.blobID); err != nil { + // the revision is gone, but not its blob: that's a failure, not a deletion + PurgeFailures.Inc() + log.Error().Err(err).Str("revision", b.key).Str("blobid", b.blobID).Msg("revision cleanup: could not delete the blob of a purged revision") + for i := range result.Revisions { + if result.Revisions[i].Key == b.key { + result.Revisions[i].Err = fmt.Errorf("the revision was deleted, but not its blob %q: %w", b.blobID, err) + } + } + continue + } + RevisionsDeleted.Inc() + BytesDeleted.Add(float64(b.size)) + } + }() + + // Holding the node lock serializes the cleanup with uploads to the same node: + // those create revisions and mark the node as processing while holding it, so + // the processing check below cannot race with a new upload starting. + // Revisions have their own lockfiles, purging them while holding the node lock + // does not deadlock. A dry run takes the lock as well, to decide on the same + // consistent state a real cleanup would see. + unlock, err := lu.MetadataBackend().Lock(n) + if err != nil { + log.Error().Err(err).Msg("revision cleanup: could not lock node") + result.Err = fmt.Errorf("could not lock node: %w", err) + return result + } + defer func() { _ = unlock() }() + + // Read the status from the backend and not via n.IsProcessing(): n caches its + // attributes, and after an upload the processing flag is removed by + // session.Cleanup through a different *node.Node instance, so the cache of n + // would still say "processing" and the cleanup would never run. + status, err := lu.MetadataBackend().Get(ctx, n, prefixes.StatusPrefix) + switch { + case err != nil && !metadata.IsAttrUnset(err): + log.Error().Err(err).Msg("revision cleanup: could not read node status") + result.Err = fmt.Errorf("could not read node status: %w", err) + return result + case strings.HasPrefix(string(status), node.ProcessingStatus): + log.Debug().Msg("revision cleanup: node is processing, skipping") + result.Skipped = "an upload to the file is still being processed" + return result + } + + versions, err := rs.ListRevisionsUnchecked(ctx, n) + if err != nil { + log.Error().Err(err).Msg("revision cleanup: could not list revisions") + result.Err = fmt.Errorf("could not list revisions: %w", err) + return result + } + if len(versions) == 0 { + return result + } + + revs := make([]revisions.Revision, len(versions)) + for i, v := range versions { + revs[i] = fileVersionRevision{version: v, mtime: revisionTime(v)} + } + + // All revisions, including the ones in keep, are handed to the strategy, as + // count based strategies ("keep the n newest") need to see the complete set to + // decide correctly. keep is only applied to the strategy's deletion plan. + purgeErrs := map[string]error{} + purged := 0 + var purgedBytes uint64 + plan, err := revisions.ExecuteCleanupStrategy(time.Now(), c.strategy, revs, ctx, &log, func(rev revisions.Revision, reason string) error { + if dryRun || slices.Contains(keep, rev.Key()) { + return nil + } + blobID, err := removeRevisionUnchecked(ctx, n, rev.Key()) + if err != nil { + PurgeFailures.Inc() + purgeErrs[rev.Key()] = err + return fmt.Errorf("could not purge revision %q: %w", rev.Key(), err) + } + blobs = append(blobs, pendingBlob{key: rev.Key(), blobID: blobID, size: rev.Size()}) + purged++ + purgedBytes += rev.Size() + // the reason explains which rule deleted the revision, e.g. which slot of the 'auto' + // strategy, see revisions.Plan + log.Debug().Str("revision", rev.Key()).Uint64("size", rev.Size()).Str("reason", reason).Msg("revision cleanup: purged revision") + return nil + }) + if plan == nil { + // the strategy itself failed + log.Error().Err(err).Msg("revision cleanup: failed") + result.Err = err + return result + } + result.Revisions = revisionResults(revs, plan, keep, purgeErrs) + + // One summary per node at info level, so that admins can see the cleanup at work + // without the per revision debug logging. Nothing is logged when nothing was purged, + // which is the common case for nodes with few revisions. + if purged > 0 { + log.Info().Str("config", c.config).Int("purged", purged).Uint64("purgedbytes", purgedBytes). + Int("revisions", len(versions)).Msg("revision cleanup: purged old revisions") + } + if err != nil { + // a revision that could not be purged is still listed and will be retried + // by the next cleanup of this node + log.Error().Err(err).Msg("revision cleanup: failed") + } + return result +} + +// revisionResults turns the plan of the strategy into one RevisionResult per revision, in the +// order of revs (newest first). +func revisionResults(revs []revisions.Revision, plan revisions.Plan, keep []string, purgeErrs map[string]error) []RevisionResult { + decisions := map[string]Decision{} + for _, rev := range plan.Deleted() { + decisions[rev.Key()] = Deleted + } + for _, rev := range plan.Retained() { + decisions[rev.Key()] = Kept + } + for _, rev := range plan.Safeguarded() { + decisions[rev.Key()] = Protected + } + + results := make([]RevisionResult, 0, len(revs)) + sorted := slices.Clone(revs) + revisions.SortRevisionsCanonically(sorted) + for _, rev := range sorted { + r := RevisionResult{ + Key: rev.Key(), + Time: rev.LastModificationTime(), + Size: rev.Size(), + Decision: decisions[rev.Key()], + Reason: plan.Reason(rev.Key()), + Err: purgeErrs[rev.Key()], + } + if r.Decision == Deleted && slices.Contains(keep, rev.Key()) { + r.Decision = Protected + r.Reason = keptOnRequest + } + results = append(results, r) + } + return results +} + +// revisionTime determines when a revision was last modified from the timestamp in its key, +// '.REV.' with an optional '.' suffix for colliding revisions. +// The mtime of the revision file is used as a fallback. +// +// This is the age the strategies work with, and it is the age of the revision's *content*: +// revisions are named after the mtime of the content they preserve (see +// DecomposedFsStore.updateExistingNode, the posix assimilation and RestoreRevision), not after +// the time the revision was created, i.e. when that content was replaced. The time a revision +// was created is not recorded anywhere. For a file that was unchanged for two months and is +// changed today, the new revision therefore counts as two months old, and a strategy deleting +// by duration, e.g. "30d", deletes it right away. See "How the age of a revision is measured" +// in README.md. +// For changes on disk with posix, the name is the time the content was copied to '.CURRENT', +// when that change was picked up, normally right after the content was written (see the posix +// assimilation). +// Whether the age should rather be the time a revision was created, i.e. when its content was +// replaced, is discussed in https://github.com/opencloud-eu/opencloud/issues/3735. +// +// The key is preferred over the file mtime because the file mtime is not reliable: +// the upload code only sets it to the mtime of the old content (os.Chtimes in +// DecomposedFsStore.updateExistingNode) after CreateRevision has returned, and the +// FileVersion.Mtime from ListRevisionsUnchecked has only a precision of seconds. +// The '.' suffix is appended by CreateRevision when a revision with the same +// timestamp already exists, it is not part of the timestamp and is stripped. +func revisionTime(v *provider.FileVersion) time.Time { + if _, ts, ok := strings.Cut(v.Key, node.RevisionIDDelimiter); ok { + if t, err := time.Parse(time.RFC3339Nano, ts); err == nil { + return t + } + if i := strings.LastIndex(ts, "."); i > 0 { + if t, err := time.Parse(time.RFC3339Nano, ts[:i]); err == nil { + return t + } + } + } + return time.Unix(int64(v.Mtime), 0) +} + +// fileVersionRevision adapts a CS3 FileVersion, as returned by +// node.Tree.ListRevisionsUnchecked, to the revisions.Revision interface the +// cleanup strategies work on. +type fileVersionRevision struct { + version *provider.FileVersion + mtime time.Time +} + +var _ revisions.Revision = fileVersionRevision{} + +func (f fileVersionRevision) Key() string { return f.version.Key } +func (f fileVersionRevision) LastModificationTime() time.Time { return f.mtime } +func (f fileVersionRevision) Size() uint64 { return f.version.Size } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/metrics.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/metrics.go new file mode 100644 index 0000000000..b9caf099e7 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/metrics.go @@ -0,0 +1,123 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisioncleanup + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Prometheus metrics of the revision cleanup, e.g. for Grafana dashboards. +// +// They are registered once per process, at package level: a process can create several +// storage drivers, and with them several Services (e.g. the storageprovider and the +// dataprovider each create their own driver), and registering a metric twice panics. +// The gauges describing the queue and the workers are therefore the sum over all Services of +// the process: each Service adds its share when it starts and removes it when it shuts down. +// As a sum can hide one queue being full while another one is empty, QueueFull counts every +// request that found its queue full, which is the most direct sign that the number of workers +// or the queue size is too small. +var ( + // RevisionsDeleted counts the revisions deleted by the cleanup. + RevisionsDeleted = promauto.NewCounter(prometheus.CounterOpts{ + Name: "reva_revision_cleanup_revisions_deleted_total", + Help: "Number of old file revisions deleted by the revision cleanup", + }) + + // BytesDeleted counts the size of the revisions deleted by the cleanup. + BytesDeleted = promauto.NewCounter(prometheus.CounterOpts{ + Name: "reva_revision_cleanup_bytes_deleted_total", + Help: "Total size in bytes of the old file revisions deleted by the revision cleanup", + }) + + // PurgeFailures counts the revisions that could not be deleted. They are retried by the + // next cleanup of their node, i.e. when the node gets its next revision. + PurgeFailures = promauto.NewCounter(prometheus.CounterOpts{ + Name: "reva_revision_cleanup_purge_failures_total", + Help: "Number of old file revisions the revision cleanup failed to delete, completely or only their blob", + }) + + // QueueFull counts the cleanup requests that found the queue full, by outcome: "blocked" + // (the caller waited for room, queue full policy "block") or "dropped" (the cleanup was + // skipped until the node's next revision, queue full policy "drop"). A caller that waited + // in vain until the timeout of "block" counts as both: blocked when it starts to wait, and + // dropped when it gives up. + QueueFull = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "reva_revision_cleanup_queue_full_total", + Help: "Number of revision cleanup requests that found the queue full, by outcome (blocked or dropped)", + }, []string{"outcome"}) + + // QueueLength is the number of nodes waiting for a background cleanup, including the + // ones whose callers wait for room in a full queue. A node waits at most once (requests + // are de-duplicated per node), how many of its revisions get deleted is only known once a + // worker processes it. + QueueLength = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "reva_revision_cleanup_queue_length", + Help: "Number of files waiting for a background revision cleanup, summed over all queues of the process", + }) + + // QueueCapacity is the size of the background cleanup queues. + QueueCapacity = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "reva_revision_cleanup_queue_capacity", + Help: "Number of files that can wait for a background revision cleanup, summed over all queues of the process", + }) + + // Workers is the number of background cleanup workers, 0 when the cleanup runs synchronously. + Workers = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "reva_revision_cleanup_workers", + Help: "Number of background revision cleanup workers, summed over all queues of the process", + }) + + // Running is the number of nodes being cleaned up by background workers right now. + Running = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "reva_revision_cleanup_running", + Help: "Number of files whose revisions are being cleaned up by background workers right now", + }) + + // Enabled is 1 if a revision cleanup strategy is configured, 0 otherwise. + Enabled = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "reva_revision_cleanup_enabled", + Help: "Whether the revision cleanup is enabled (1) or not (0)", + }) +) + +// labels of QueueFull +const ( + queueFullBlocked = "blocked" + queueFullDropped = "dropped" +) + +// serviceMetrics are the metrics a Service updates about itself: its configuration, its queue +// and its workers. +type serviceMetrics struct { + enabled, workers, queueCapacity, queueLength, running prometheus.Gauge + queueFull *prometheus.CounterVec +} + +// driverMetrics are the exported metrics, updated by the Services of the storage drivers. +var driverMetrics = serviceMetrics{ + enabled: Enabled, + workers: Workers, + queueCapacity: QueueCapacity, + queueLength: QueueLength, + running: Running, + queueFull: QueueFull, +} + +// unregisteredMetrics returns metrics that are not registered, for the Service of an on-demand +// Run: the exported ones describe the cleanup of the storage driver, which a run of its own, +// with its own queue and workers, must not distort, should it ever run in the same process. +// The deletions of a run are counted all the same (RevisionsDeleted, BytesDeleted and +// PurgeFailures), as the Cleaner counts them. +func unregisteredMetrics() *serviceMetrics { + gauge := func() prometheus.Gauge { return prometheus.NewGauge(prometheus.GaugeOpts{Name: "unregistered"}) } + return &serviceMetrics{ + enabled: gauge(), + workers: gauge(), + queueCapacity: gauge(), + queueLength: gauge(), + running: gauge(), + queueFull: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "unregistered"}, []string{"outcome"}), + } +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/ondemand.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/ondemand.go new file mode 100644 index 0000000000..cc80f7a25e --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/ondemand.go @@ -0,0 +1,266 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisioncleanup + +import ( + "context" + "errors" + "fmt" + "math/rand/v2" + "path" + "sync" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/rs/zerolog" + + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" +) + +// Request describes an on-demand cleanup of revisions, e.g. by a command line tool, as opposed +// to the cleanup that runs whenever a file gets a new revision. +// +// The strategy and the AsyncOptions default to the configuration of the storage driver, and +// can be overridden for this run. Without AsyncOptions of its own, though, a run waits for +// room in a full queue for as long as it takes (QueueFullBlock without a timeout), whatever +// the queue full policy of the storage driver: a run queues the files as fast as it finds +// them, so dropping them whenever the queue is full would skip most of them. +type Request struct { + // SpaceIDs are the spaces to clean up. Exactly one of SpaceIDs, All and FSPath must be given. + SpaceIDs []string + // All cleans up all spaces. + All bool + // Path narrows the cleanup down to a file or a folder (recursively) within the space, + // relative to the space root. It requires exactly one space in SpaceIDs. + Path string + // FSPath is a file or folder (recursively) given by its path on disk, as an admin sees it. + // Only drivers that keep files under their own names on disk support it, i.e. posix, which + // resolves it to the space and the path within it. + FSPath string + + // Strategy overrides the configured revisions cleanup strategy for this run, see + // revisions.NewRevisionCleanupStrategy. + Strategy string + // Async overrides the configured AsyncOptions for this run. Without it, the configured + // ones are used, but with QueueFullBlock and no timeout. + Async *AsyncOptions + // DryRun only decides which revisions would be deleted, without deleting anything. + DryRun bool + // Shuffle cleans up the spaces, and the files and folders within each folder, in a random + // order instead of the same order every time. A run that is stopped early through its + // context (e.g. by a timeout) starts over from the beginning the next time; shuffled, + // repeated runs start in different places, and eventually reach all files, also within a + // space that is too large for a single run. + Shuffle bool +} + +// Validate checks the scope of the request. +func (r Request) Validate() error { + switch { + case r.FSPath != "" && (r.All || len(r.SpaceIDs) > 0 || r.Path != ""): + return errors.New("a path on disk can't be combined with spaces or a path within a space") + case r.FSPath != "": + return nil + case r.All && len(r.SpaceIDs) > 0: + return errors.New("either all spaces or specific spaces can be cleaned up, not both") + case !r.All && len(r.SpaceIDs) == 0: + return errors.New("no space to clean up given") + case r.Path != "" && len(r.SpaceIDs) != 1: + return errors.New("a path requires exactly one space") + } + return nil +} + +// OnDemand is implemented by the storage drivers that can clean up revisions on demand. +// +// CleanupRevisionsUnchecked cleans up the revisions of all files within the scope of the request +// and hands the Result of every file to report, one at a time, including the files that were +// skipped. It returns once all cleanups are done. It returns an error if the request is invalid or +// the cleanup could not be run; errors with single files are only reported. +// +// The end of ctx, e.g. a timeout, stops the cleanup gracefully: the files being cleaned up +// are finished, and the files already queued are reported as skipped. It then returns ctx's +// error. +type OnDemand interface { + CleanupRevisionsUnchecked(ctx context.Context, req Request, report func(Result)) error +} + +// Walker is the part of a tree needed to clean up the revisions of whole folders or spaces. +type Walker interface { + RevisionStore + // ListFolderKnownUnchecked lists the children of a folder that are known to the storage, for + // maintenance. Unlike ListFolder: + // - it checks no permissions, so it needs no user in the context, and it also lists the + // files and folders that are denied to users; + // - it lists only the known children, the files and folders the storage has registered + // with a node id, and never assimilates (registers) unknown ones, which exist on disk + // but are not registered yet. Only posix has unknown files today, as files can be + // created on disk directly; in the other trees, every child is known. + // So it changes nothing, and a cleanup, a dry run in particular, changes nothing but the + // revisions it deletes. Unknown files have no revisions the storage knows of anyway, and a + // later run covers them once they are registered. + ListFolderKnownUnchecked(ctx context.Context, n *node.Node) ([]*node.Node, error) +} + +// RunOptions are the settings of an on-demand Run. +type RunOptions struct { + // Async are the AsyncOptions of the run's own Service. + Async AsyncOptions + // DryRun only decides which revisions would be deleted, without deleting anything. + DryRun bool + // Shuffle walks the files and folders within each folder in a random order, see + // Request.Shuffle. The order of the roots is the caller's. + Shuffle bool +} + +// Run cleans up the revisions of the given roots, files or folders (recursively, e.g. space roots), +// with the given Cleaner and RunOptions, and hands the Result of every file to report, one at a +// time. It is the common implementation of OnDemand.CleanupRevisionsUnchecked for the storage +// drivers. +// +// It uses a Service of its own, independent of the storage driver's, so that the Cleaner and +// the AsyncOptions of the run apply: files are queued as they are found, while the workers +// already clean up, and Run waits for all cleanups before it returns. Files dropped by a full +// queue (queue full policy "drop") are reported as skipped. That Service leaves the metrics +// of the storage driver's queue and workers alone; the deleted revisions are counted. +// +// It returns an error if a cleanup could not be run at all, or ctx's error if it ends. Errors +// with single files or folders are only reported. When ctx ends, Run stops gracefully: it +// finishes the cleanups that are running, reports the files that are still queued as skipped, +// and doesn't look at the remaining ones. +// +// It checks no permissions: the Walker lists folders without them, and neither do the +// cleanups. +func Run(ctx context.Context, lu node.PathLookup, w Walker, roots []*node.Node, cleaner *Cleaner, o RunOptions, report func(Result), log *zerolog.Logger) error { + if !cleaner.Enabled() { + return errors.New("no revisions cleanup strategy configured or given") + } + if report == nil { + report = func(Result) {} + } + // the workers report concurrently, report gets the results one at a time + var reportMu sync.Mutex + // the paths of the files queued by the walk, until their results are reported: the walk + // knows them for free, while looking them up for every result would read all the parents + paths := map[string]*queuedPath{} + serialized := func(result Result) { + reportMu.Lock() + defer reportMu.Unlock() + key := result.SpaceID + "/" + result.NodeID + if p, ok := paths[key]; ok { + if result.Path == "" { + result.Path = p.path + } + if p.count--; p.count <= 0 { + delete(paths, key) + } + } + report(result) + } + remember := func(n *node.Node, path string) { + reportMu.Lock() + defer reportMu.Unlock() + key := n.SpaceID + "/" + n.ID + if p, ok := paths[key]; ok { + p.count++ // the same file in more than one root + return + } + paths[key] = &queuedPath{path: path, count: 1} + } + + s, err := newService(cleaner, lu, w, o.Async, log, true, o.DryRun, serialized) + if err != nil { + return err + } + // stop the workers also when ctx ended; they are idle by then unless ctx ended + defer func() { _ = s.Shutdown(context.WithoutCancel(ctx)) }() + + for _, root := range roots { + if err := walkScoped(ctx, lu, w, root, s, o, serialized, remember); err != nil { + return err + } + } + return s.Wait(ctx) +} + +// walkScoped walks root, see walk, switched to the group of its space if w is a SpaceScoper, +// so that the folders can be listed, see SpaceScoper. A space whose group can't be switched to +// is reported and left out. +func walkScoped(ctx context.Context, lu node.PathLookup, w Walker, root *node.Node, s *Service, o RunOptions, report func(Result), remember func(n *node.Node, path string)) error { + ctx, unscope, err := scopeToSpace(ctx, w, root.SpaceID) + if err != nil { + report(Result{SpaceID: root.SpaceID, NodeID: root.ID, DryRun: o.DryRun, Err: fmt.Errorf("could not switch to the group of the space: %w", err)}) + return nil + } + defer func() { _ = unscope() }() + return walk(ctx, w, root, rootPath(ctx, lu, root), s, o, report, remember) +} + +// queuedPath is the path of a queued file, see Run +type queuedPath struct { + path string + count int +} + +// rootPath returns the path of root within its space, "/" for the space root, or "" if it +// can't be determined. +func rootPath(ctx context.Context, lu node.PathLookup, root *node.Node) string { + if root.SpaceRoot == nil { + return "" + } + p, err := lu.Path(ctx, root, node.NoCheck) + if err != nil { + return "" + } + return p +} + +// walk queues the cleanup of n, if it is a file, or of all files below it, if it is a folder. +// rootPath is the path of n within its space, the paths of the files below it are derived +// from it and handed to remember before they are queued. +func walk(ctx context.Context, w Walker, n *node.Node, rootPath string, s *Service, o RunOptions, report func(Result), remember func(n *node.Node, path string)) error { + type entry struct { + n *node.Node + path string + } + stack := []entry{{n: n, path: rootPath}} + for len(stack) > 0 { + if err := ctx.Err(); err != nil { + return err + } + current := stack[len(stack)-1] + stack = stack[:len(stack)-1] + + switch current.n.Type(ctx) { + case provider.ResourceType_RESOURCE_TYPE_FILE: + remember(current.n, current.path) + s.HandleRevisionCreated(ctx, current.n) + case provider.ResourceType_RESOURCE_TYPE_CONTAINER: + children, err := w.ListFolderKnownUnchecked(ctx, current.n) + if err != nil && ctx.Err() != nil { + return ctx.Err() // stopped while listing, not an error of the folder + } + if err != nil { + report(Result{SpaceID: current.n.SpaceID, NodeID: current.n.ID, Path: current.path, DryRun: o.DryRun, Err: fmt.Errorf("could not list the folder at %q: %w", current.n.InternalPath(), err)}) + continue + } + // The second part of Request.Shuffle, after the order of the spaces (see + // Decomposedfs.CleanupRevisionsUnchecked): the order of the children of this folder, + // files and folders alike. As the walk takes them from a stack, depth first, this also + // changes the order in which the subfolders are walked through. + if o.Shuffle { + rand.Shuffle(len(children), func(i, j int) { children[i], children[j] = children[j], children[i] }) + } + for _, child := range children { + childPath := "" + if current.path != "" { + childPath = path.Join(current.path, child.Name) + } + stack = append(stack, entry{n: child, path: childPath}) + } + default: + // references, symlinks and the like have no revisions + } + } + return nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/result.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/result.go new file mode 100644 index 0000000000..ac8b34c8c1 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/result.go @@ -0,0 +1,65 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisioncleanup + +import "time" + +// Decision is what the cleanup decided about a revision. +type Decision string + +const ( + // Deleted revisions are deleted by the strategy (or would be, in a dry run). + Deleted Decision = "deleted" + // Kept revisions are not deleted by the strategy, but not protected either: a later + // cleanup may delete them as they get older. + Kept Decision = "kept" + // Protected revisions are safeguarded by the strategy, or were requested to be kept by + // the operation that triggered the cleanup (e.g. the state from before a restore). + Protected Decision = "protected" +) + +// RevisionResult is the outcome of a cleanup for one revision. +type RevisionResult struct { + Key string + // Time is the time the cleanup takes the age of the revision from, see revisionTime. + Time time.Time + Size uint64 + // Decision and Reason explain what the strategy decided and why, see revisions.Plan. + Decision Decision + Reason string + // Err is set when deleting a revision failed. + Err error +} + +// Result is the outcome of a cleanup for one node, with an entry for each of its revisions, +// newest first. It is used to report on a cleanup, e.g. in the verbose output of a command. +type Result struct { + SpaceID string + NodeID string + // Path is the path of the file (or folder) within its space, e.g. "/Documents/report.odt". + // Only on-demand runs (Run) set it, from the folders they walk through; it is empty when + // the path is not known, e.g. for a space that could not be cleaned up. + Path string + // DryRun is set when nothing was deleted, only decided. + DryRun bool + // Skipped explains why the node was not cleaned up, e.g. because an upload to it is still + // being processed. Revisions is empty then. + Skipped string + // Err is set when the cleanup failed as a whole, e.g. when the revisions could not be + // listed. Errors of single revisions are in Revisions. + Err error + Revisions []RevisionResult +} + +// DeletedCount returns the number and the total size of the revisions that were deleted +// (or would be, in a dry run), not counting the ones whose deletion failed. +func (r Result) DeletedCount() (count int, bytes uint64) { + for _, rev := range r.Revisions { + if rev.Decision == Deleted && rev.Err == nil { + count++ + bytes += rev.Size + } + } + return count, bytes +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/service.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/service.go new file mode 100644 index 0000000000..63bc0e366f --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup/service.go @@ -0,0 +1,689 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisioncleanup + +import ( + "context" + "fmt" + "runtime" + "slices" + "strings" + "sync" + "time" + + "github.com/rs/zerolog" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/opencloud-eu/reva/v2/pkg/appctx" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options" +) + +var tracer = otel.Tracer("github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup") + +// QueueFullPolicy defines what happens when a cleanup is scheduled while the queue is full. +type QueueFullPolicy string + +const ( + // QueueFullBlock makes the caller wait until there is room in the queue, so that no + // cleanup is ever skipped, at the cost of slowing down writes under heavy load. + QueueFullBlock QueueFullPolicy = "block" + // QueueFullDrop skips the cleanup and logs it, so that writes never wait for the + // cleanup. The node is cleaned up again when it gets its next revision. + QueueFullDrop QueueFullPolicy = "drop" +) + +// queueSizePerWorker is the number of queue slots per worker when the queue size is +// determined automatically. +const queueSizePerWorker = 64 + +// AsyncOptions configures how a Service runs the cleanup. They are passed as +// parameters rather than read from the driver options, so that other callers, like a +// command line tool, can use their own settings. +type AsyncOptions struct { + // Enabled runs the cleanup in background workers. Otherwise HandleRevisionCreated runs it + // right away, in the caller's goroutine. + Enabled bool + // Workers is the number of background workers, 0 determines it automatically: + // half the available CPUs, but at least 2. + Workers int + // QueueSize is the number of nodes that can wait for a cleanup, 0 determines it + // automatically from the number of workers. + QueueSize int + // QueueFull is the policy when the queue is full, "" means QueueFullDrop. + QueueFull QueueFullPolicy + // QueueFullTimeout bounds how long QueueFullBlock waits for room in a full queue: after + // that, the cleanup of the node is skipped, as with QueueFullDrop. 0 waits as long as it + // takes. + QueueFullTimeout time.Duration +} + +// resolve validates the options and determines the automatic values. +func (o AsyncOptions) resolve() (workers int, queueSize int, policy QueueFullPolicy, err error) { + switch { + case o.Workers < 0: + return 0, 0, "", fmt.Errorf("invalid number of revision cleanup workers %d", o.Workers) + case o.Workers == 0: + // GOMAXPROCS rather than NumCPU, as it respects container CPU limits (cgroups) + workers = max(2, runtime.GOMAXPROCS(0)/2) + default: + workers = o.Workers + } + + switch { + case o.QueueSize < 0: + return 0, 0, "", fmt.Errorf("invalid revision cleanup queue size %d", o.QueueSize) + case o.QueueSize == 0: + queueSize = workers * queueSizePerWorker + default: + queueSize = o.QueueSize + } + + if o.QueueFullTimeout < 0 { + return 0, 0, "", fmt.Errorf("invalid revision cleanup queue full timeout %s", o.QueueFullTimeout) + } + + switch o.QueueFull { + case "": + policy = QueueFullDrop + case QueueFullBlock, QueueFullDrop: + policy = o.QueueFull + default: + return 0, 0, "", fmt.Errorf("invalid revision cleanup queue full policy %q, must be %q or %q", o.QueueFull, QueueFullBlock, QueueFullDrop) + } + return workers, queueSize, policy, nil +} + +// Service is the revision cleanup of a storage driver. Callers report that a new revision +// was created (HandleRevisionCreated), and the Service decides whether, when and how the +// old revisions are cleaned up; callers don't need to know whether that happens right away +// or in background workers (see AsyncOptions). It binds a Cleaner to the driver's lookup +// and tree, so that callers only need to name the node. +// A disabled Service, created without a strategy, never cleans up anything; see Enabled. +// A nil *Service behaves the same, as a safeguard, but NewService never returns one. +// +// In asynchronous mode, the cleanup runs in background workers fed by a bounded queue, so +// that it doesn't slow down the upload, restore or change on disk that created the new +// revision. Queued requests only carry the space and node id, and the worker reads the +// node again: a *node.Node must not be shared between goroutines, as its lock-held flag is +// per goroutine and its cached attributes go stale. +// +// Requests are de-duplicated per node: a node that is already waiting in the queue is not +// queued again (the revisions to keep are merged into the waiting request). A node whose +// cleanup is already running is not queued either, as another worker would only wait for +// the node lock: when that cleanup is done, the node is queued once more, at the end of the +// queue, as new revisions may have been created in the meantime, with the revisions to keep +// of all the requests that came in until then. A node therefore never occupies more than +// one worker, and a single busy file can't keep the workers from the others. +type Service struct { + cleaner *Cleaner + lu node.PathLookup + rs RevisionStore + log *zerolog.Logger + + // set for the Service of an on-demand Run: only decide, don't delete, and hand the Result + // of every cleanup to report + dryRun bool + report func(Result) + // the metrics this Service updates, see m + metrics *serviceMetrics + // set for the Service of an on-demand Run: it leaves the metrics of the storage driver's + // Service alone (see unregisteredMetrics), and it can be stopped through the caller's + // context: a caller blocked on a full queue gives up when its context ends, and a cleanup + // in synchronous mode always finishes its file, see HandleRevisionCreated + onDemand bool + + async bool + policy QueueFullPolicy + process func(ctx context.Context, spaceID, nodeID string, keep []string) + queue chan *request + done chan struct{} + wg sync.WaitGroup + + // how long a caller waits for room in a full queue with QueueFullBlock, 0 for as long as + // it takes + queueFullTimeout time.Duration + + // the numbers this Service added to the Workers and QueueCapacity gauges + workers int + queueSize int + + mu sync.Mutex + queued map[string]*request // the requests waiting in the queue, by node; QueueLength counts them + // the nodes being cleaned up by a worker, with the request to queue once more when it + // is done, nil if there is none; QueueLength counts those requests as well + running map[string]*request + closed bool + // pending counts the requests from being queued until their cleanup has finished (or they + // were dropped or abandoned); idle is closed whenever pending is 0, see Wait + pending int + idle chan struct{} +} + +type request struct { + ctx context.Context // see detach + // link points to the span of the request that triggered the cleanup, see detach + link trace.Link + spaceID string + nodeID string + keep []string +} + +func (r *request) key() string { + return r.spaceID + "/" + r.nodeID +} + +// merge adds the revisions to keep of other to r, s.mu must be held. +func (r *request) merge(other *request) { + for _, k := range other.keep { + if !slices.Contains(r.keep, k) { + r.keep = append(r.keep, k) + } + } +} + +// NewServiceFromOptions creates the Service of a storage driver from its options: the cleanup +// strategy (RevisionsCleanupStrategy) and the RevisionsCleanupAsync* settings. +// It returns a disabled Service when no strategy is configured, and an error for invalid +// settings. +func NewServiceFromOptions(o *options.Options, lu node.PathLookup, rs RevisionStore, log *zerolog.Logger) (*Service, error) { + cleaner, err := New(o.RevisionsCleanupStrategy) + if err != nil { + return nil, err + } + return NewService(cleaner, lu, rs, AsyncOptionsFromOptions(o), log) +} + +// AsyncOptionsFromOptions returns the AsyncOptions configured in the driver options +// (the RevisionsCleanupAsync* settings). +func AsyncOptionsFromOptions(o *options.Options) AsyncOptions { + return AsyncOptions{ + Enabled: o.RevisionsCleanupAsync, + Workers: o.RevisionsCleanupAsyncWorkers, + QueueSize: o.RevisionsCleanupAsyncQueueSize, + QueueFull: QueueFullPolicy(o.RevisionsCleanupAsyncQueueFull), + QueueFullTimeout: o.RevisionsCleanupAsyncQueueFullTimeout, + } +} + +// NewService creates a Service that cleans up with the given Cleaner, and starts its +// workers if opts.Enabled. With a disabled (or nil) Cleaner, it returns a disabled Service, +// which starts no workers and never cleans up anything. +// The options are validated in any case, so that invalid settings are reported even while +// the cleanup is disabled. +func NewService(cleaner *Cleaner, lu node.PathLookup, rs RevisionStore, opts AsyncOptions, log *zerolog.Logger) (*Service, error) { + return newService(cleaner, lu, rs, opts, log, false, false, nil) +} + +// newService creates a Service, see NewService; onDemand creates the Service of an on-demand +// Run, with dryRun it only decides without deleting, and report, if not nil, gets the Result +// of every cleanup. +func newService(cleaner *Cleaner, lu node.PathLookup, rs RevisionStore, opts AsyncOptions, log *zerolog.Logger, onDemand, dryRun bool, report func(Result)) (*Service, error) { + workers, queueSize, policy, err := opts.resolve() + if err != nil { + return nil, err + } + if log == nil { + log = &zerolog.Logger{} + } + if !cleaner.Enabled() { + return &Service{cleaner: cleaner, log: log}, nil + } + s := &Service{cleaner: cleaner, lu: lu, rs: rs, log: log, onDemand: onDemand, dryRun: dryRun, report: report} + if onDemand { + s.metrics = unregisteredMetrics() + } + s.m().enabled.Set(1) + if opts.Enabled { + s.queueFullTimeout = opts.QueueFullTimeout + s.startWorkers(workers, queueSize, policy, s.cleanupNode) + log.Info().Str("config", cleaner.config).Int("workers", workers).Int("queuesize", queueSize). + Str("queuefull", string(policy)).Dur("queuefulltimeout", opts.QueueFullTimeout). + Msg("revision cleanup: running in the background") + } + return s, nil +} + +// Enabled tells whether s cleans up anything, i.e. whether its Cleaner has a strategy. +// It is false for a disabled Service, and for a nil one. +func (s *Service) Enabled() bool { + return s != nil && s.cleaner.Enabled() +} + +// startWorkers switches the Service to asynchronous mode. process does the work for one +// node; it is a parameter so that tests can observe the queueing without a storage. +func (s *Service) startWorkers(workers, queueSize int, policy QueueFullPolicy, process func(ctx context.Context, spaceID, nodeID string, keep []string)) { + s.async = true + s.policy = policy + s.process = process + s.queue = make(chan *request, queueSize) + s.done = make(chan struct{}) + s.queued = make(map[string]*request) + s.running = make(map[string]*request) + s.idle = make(chan struct{}) + close(s.idle) + s.workers = workers + s.queueSize = queueSize + s.m().workers.Add(float64(workers)) + s.m().queueCapacity.Add(float64(queueSize)) + for range workers { + s.wg.Add(1) + go s.work() + } +} + +// HandleRevisionCreated handles the event that a new revision of n was created and has +// settled: it cleans up the old revisions of n according to the strategy, never deleting +// the revisions with a key in keep. In asynchronous mode it only queues n and returns, +// see Service and AsyncOptions. +// Errors are logged, never returned: a failed cleanup must not fail the operation that +// triggered it. +// +// In an on-demand Run, the end of ctx stops the run between files: a cleanup in synchronous +// mode still finishes its file, and a caller blocked on a full queue gives up, reporting the +// file as skipped. +// +// It checks no permissions, see Cleaner.Cleanup. +func (s *Service) HandleRevisionCreated(ctx context.Context, n *node.Node, keep ...string) { + if !s.Enabled() || n == nil { + return + } + s.handle(ctx, n.SpaceID, n.ID, n, keep) +} + +// HandleRevisionCreatedByID is HandleRevisionCreated for a node given by its ids, for callers +// whose node is still in use elsewhere, as a node must not be shared between goroutines. It +// reads a node of its own only to clean it up: not at all if the cleanup is disabled, and in +// asynchronous mode only in the worker, which reads the node anyway. +// +// It checks no permissions, see Cleaner.Cleanup. +func (s *Service) HandleRevisionCreatedByID(ctx context.Context, spaceID, nodeID string, keep ...string) { + if !s.Enabled() { + return + } + s.handle(ctx, spaceID, nodeID, nil, keep) +} + +// handle cleans up the node right away in synchronous mode, n if given, otherwise read by its +// ids, and queues it in asynchronous mode. +func (s *Service) handle(ctx context.Context, spaceID, nodeID string, n *node.Node, keep []string) { + if !s.async { + if s.onDemand { + // don't leave a file half cleaned up, the run stops before the next one + ctx = context.WithoutCancel(ctx) + } + if n == nil { + s.cleanupNode(ctx, spaceID, nodeID, keep) + return + } + s.clean(ctx, n, keep) + return + } + // the cleanup outlives the request that triggered it, see detach + detached, link := detach(ctx) + s.enqueue(ctx, &request{ctx: detached, link: link, spaceID: spaceID, nodeID: nodeID, keep: slices.Clone(keep)}) +} + +// detach returns the context for a cleanup in the background, which outlives the request that +// triggered it, and a link to the request's span. +// +// The context is never cancelled, and it carries none of the values of the request's context, +// which may only be valid during the request, and which a request waiting in the queue would +// otherwise keep alive, except for the logger, so that the log lines of the cleanup keep the +// fields of the request, e.g. its id. The cleanup gets a trace of its own (see startSpan) with +// the link, rather than a span of the request's trace, which would usually have ended long +// before. +func detach(ctx context.Context) (context.Context, trace.Link) { + detached := appctx.WithLogger(context.Background(), appctx.GetLogger(ctx)) + return detached, trace.LinkFromContext(ctx) +} + +// startSpan starts the span of the cleanup of r in a worker, linked to the span of the request +// that triggered it, if there was one, see detach. +func startSpan(r *request) (context.Context, trace.Span) { + opts := []trace.SpanStartOption{trace.WithAttributes( + attribute.String("spaceid", r.spaceID), + attribute.String("nodeid", r.nodeID), + )} + if r.link.SpanContext.IsValid() { + opts = append(opts, trace.WithLinks(r.link)) + } + return tracer.Start(r.ctx, "revision cleanup", opts...) +} + +// enqueue queues r; ctx is the caller's context, which only an on-demand Run waits on. +func (s *Service) enqueue(ctx context.Context, r *request) { + log := appctx.GetLogger(r.ctx).With().Str("spaceid", r.spaceID).Str("nodeid", r.nodeID).Logger() + key := r.key() + + s.mu.Lock() + if s.closed { + s.mu.Unlock() + log.Debug().Msg("revision cleanup: shutting down, not scheduling cleanup") + return + } + if waiting, ok := s.queued[key]; ok { + // already waiting in the queue, it will see the new revision as well + waiting.merge(r) + s.mu.Unlock() + return + } + if again, ok := s.running[key]; ok { + // being cleaned up right now: it is queued once more when that is done + if again == nil { + s.running[key] = r + s.addPending() + s.m().queueLength.Inc() + } else { + again.merge(r) + } + s.mu.Unlock() + return + } + s.queued[key] = r + s.addPending() + s.m().queueLength.Inc() + s.mu.Unlock() + s.send(ctx, r) +} + +// send puts r, which waits in s.queued, into the queue, applying the queue full policy. +// s.mu is not held while sending, so that a blocked sender does not block the workers. +// ctx is the caller's context, see enqueue. +func (s *Service) send(ctx context.Context, r *request) { + log := appctx.GetLogger(r.ctx).With().Str("spaceid", r.spaceID).Str("nodeid", r.nodeID).Logger() + select { + case s.queue <- r: + return + default: + } + if s.policy == QueueFullDrop { + s.m().queueFull.WithLabelValues(queueFullDropped).Inc() + if s.forget(r) { + s.reportResult(Result{SpaceID: r.spaceID, NodeID: r.nodeID, DryRun: s.dryRun, + Skipped: fmt.Sprintf("the queue was full (queue full policy %q)", QueueFullDrop)}) + } + log.Warn().Str("path", s.internalPath(r.spaceID, r.nodeID)). + Msg("revision cleanup: queue is full, skipping the cleanup of this node until its next revision") + return + } + s.m().queueFull.WithLabelValues(queueFullBlocked).Inc() + log.Debug().Msg("revision cleanup: queue is full, waiting for room") + // with a timeout, a caller doesn't wait for longer than that: under sustained load, it + // would otherwise slow down every write, e.g. the consumer of the postprocessing events + var timeout <-chan time.Time + if s.queueFullTimeout > 0 { + timer := time.NewTimer(s.queueFullTimeout) + defer timer.Stop() + timeout = timer.C + } + var stopped <-chan struct{} // nil, i.e. never, unless on demand + if s.onDemand { + stopped = ctx.Done() + } + select { + case s.queue <- r: + case <-s.done: + // Shutdown abandoned (and reported) it + s.forget(r) + case <-timeout: + s.m().queueFull.WithLabelValues(queueFullDropped).Inc() + if s.forget(r) { + s.reportResult(Result{SpaceID: r.spaceID, NodeID: r.nodeID, DryRun: s.dryRun, + Skipped: fmt.Sprintf("the queue was still full after %s (queue full policy %q)", s.queueFullTimeout, QueueFullBlock)}) + } + log.Warn().Str("path", s.internalPath(r.spaceID, r.nodeID)).Dur("timeout", s.queueFullTimeout). + Msg("revision cleanup: queue is still full after the timeout, skipping the cleanup of this node until its next revision") + case <-stopped: + if s.forget(r) { + s.reportResult(Result{SpaceID: r.spaceID, NodeID: r.nodeID, DryRun: s.dryRun, Skipped: skippedStopped}) + } + } +} + +// the reason for skipping files whose cleanup was queued when an on-demand run was stopped +const skippedStopped = "the cleanup was stopped before this file was reached" + +// internalPath returns the path of a node on disk, e.g. for logging what is skipped, so that +// admins can find it. It is empty for a Service without a lookup. +func (s *Service) internalPath(spaceID, nodeID string) string { + if s.lu == nil { + return "" + } + return s.lu.InternalPath(spaceID, nodeID) +} + +// forget removes a request that did not make it into the queue. It returns false if the +// request was not waiting anymore, e.g. as Shutdown abandoned it. +func (s *Service) forget(r *request) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.queued[r.key()] != r { + return false + } + delete(s.queued, r.key()) + s.m().queueLength.Dec() + s.donePending(1) + return true +} + +func (s *Service) work() { + defer s.wg.Done() + for { + // a select with several ready cases picks one at random, so check for the + // shutdown first, otherwise a worker could keep taking requests from a + // non-empty queue after Shutdown + select { + case <-s.done: + return + default: + } + select { + case <-s.done: + return + case r := <-s.queue: + select { + case <-s.done: + // shut down while waiting for this request: abandon it, Shutdown + // counts it as abandoned + return + default: + } + // from here on, new requests for the node are collected to queue it once more + // when this cleanup is done, instead of being merged into this one + s.mu.Lock() + if s.closed { + // shut down in the meantime: Shutdown already counted this request as abandoned + s.mu.Unlock() + return + } + if s.queued[r.key()] == r { + delete(s.queued, r.key()) + s.m().queueLength.Dec() + } + s.running[r.key()] = nil + keep := slices.Clone(r.keep) + s.mu.Unlock() + s.m().running.Inc() + ctx, span := startSpan(r) + s.process(ctx, r.spaceID, r.nodeID, keep) + span.End() + s.m().running.Dec() + s.mu.Lock() + again := s.running[r.key()] + delete(s.running, r.key()) + s.donePending(1) + if again == nil || s.closed { + // done, or shut down, which abandoned a request to queue again + s.mu.Unlock() + continue + } + // queue the node once more, at the end of the queue, so that the other nodes get + // their turn; sending in a goroutine of its own, as a worker must never wait for + // room in the queue it empties. QueueLength and pending already count the request. + s.queued[again.key()] = again + s.mu.Unlock() + // No caller waits for it: in an on-demand Run, Shutdown stops it. + go s.send(context.Background(), again) + } + } +} + +// cleanupNode reads the node again and cleans it up, it runs in a worker. +func (s *Service) cleanupNode(ctx context.Context, spaceID, nodeID string, keep []string) { + // read also in a disabled space, to tell it apart from an error + n, err := node.ReadNode(ctx, s.lu, spaceID, nodeID, "", true, nil, false) + if err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", spaceID).Str("nodeid", nodeID).Msg("revision cleanup: could not read node") + s.reportResult(Result{SpaceID: spaceID, NodeID: nodeID, DryRun: s.dryRun, Err: fmt.Errorf("could not read node: %w", err)}) + return + } + if !n.Exists { + // deleted in the meantime, its revisions went with it (or into the trash) + s.reportResult(Result{SpaceID: spaceID, NodeID: nodeID, DryRun: s.dryRun, Skipped: "the file no longer exists"}) + return + } + if n.SpaceRoot != nil && n.SpaceRoot.IsDisabled(ctx) { + // disabled in the meantime: the space is in the trash, leave its files alone + appctx.GetLogger(ctx).Debug().Str("spaceid", spaceID).Str("nodeid", nodeID).Msg("revision cleanup: the space is disabled, skipping") + s.reportResult(Result{SpaceID: spaceID, NodeID: nodeID, DryRun: s.dryRun, Skipped: "the space is disabled"}) + return + } + s.clean(ctx, n, keep) +} + +// clean runs the cleanup of n, or only decides about it in a dry run, and reports the Result. +func (s *Service) clean(ctx context.Context, n *node.Node, keep []string) { + var result Result + if s.dryRun { + result = s.cleaner.DryRun(ctx, s.lu, s.rs, n, keep...) + } else { + result = s.cleaner.Cleanup(ctx, s.lu, s.rs, n, keep...) + } + s.reportResult(result) +} + +// m returns the metrics this Service updates: the exported ones, unless it is the Service of an +// on-demand Run. +func (s *Service) m() *serviceMetrics { + if s.metrics != nil { + return s.metrics + } + return &driverMetrics +} + +func (s *Service) reportResult(result Result) { + if s.report != nil { + s.report(result) + } +} + +// addPending counts a new request, s.mu must be held. +func (s *Service) addPending() { + if s.pending == 0 { + s.idle = make(chan struct{}) + } + s.pending++ +} + +// donePending counts n requests as finished, s.mu must be held. +func (s *Service) donePending(n int) { + if n <= 0 { + return + } + s.pending -= n + if s.pending <= 0 { + s.pending = 0 + close(s.idle) + } +} + +// Wait waits until no cleanup is queued or running anymore, or until ctx ends, in which case +// it returns ctx's error. Requests that were dropped (queue full policy "drop") or abandoned +// by Shutdown don't keep it waiting. In synchronous mode, and for a nil Service, it returns +// right away, as the cleanups already ran in their callers. +// +// Unlike Shutdown, it doesn't stop anything: it is meant for callers that queue a batch of +// cleanups and need them to be done, e.g. a command cleaning up all files of a space. +// Cleanups queued while it waits are waited for as well, until the Service is idle once. +func (s *Service) Wait(ctx context.Context) error { + if s == nil || !s.async { + return nil + } + s.mu.Lock() + idle := s.idle + s.mu.Unlock() + select { + case <-idle: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Shutdown stops the background workers. A cleanup that is running is finished, the ones +// still waiting in the queue are abandoned (and logged, and reported as skipped in an +// on-demand Run): they are picked up again when their nodes get their next revision. +// Blocked callers are released. It returns when the +// workers have stopped, or with ctx's error if ctx ends first. +func (s *Service) Shutdown(ctx context.Context) error { + if s == nil || !s.async { + return nil + } + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return nil + } + s.closed = true + close(s.done) + // abandon the requests still waiting, in the queue or to be queued once more: they won't + // be processed anymore, and their nodes are cleaned up again when they get their next + // revision + var abandonedRequests []*request + for _, r := range s.queued { + abandonedRequests = append(abandonedRequests, r) + } + clear(s.queued) + for key, r := range s.running { + if r != nil { + abandonedRequests = append(abandonedRequests, r) + s.running[key] = nil + } + } + abandoned := len(abandonedRequests) + var skipped []Result + if s.report != nil { + for _, r := range abandonedRequests { + skipped = append(skipped, Result{SpaceID: r.spaceID, NodeID: r.nodeID, DryRun: s.dryRun, Skipped: skippedStopped}) + } + slices.SortFunc(skipped, func(a, b Result) int { return strings.Compare(a.SpaceID+"/"+a.NodeID, b.SpaceID+"/"+b.NodeID) }) + } + s.donePending(abandoned) + s.m().queueLength.Sub(float64(abandoned)) + s.m().workers.Sub(float64(s.workers)) + s.m().queueCapacity.Sub(float64(s.queueSize)) + s.mu.Unlock() + for _, result := range skipped { + s.reportResult(result) + } + + stopped := make(chan struct{}) + go func() { + s.wg.Wait() + close(stopped) + }() + select { + case <-stopped: + case <-ctx.Done(): + return ctx.Err() + } + + if abandoned > 0 { + s.log.Info().Int("abandoned", abandoned).Msg("revision cleanup: stopped, abandoned the cleanups still waiting in the queue") + } + return nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisions.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisions.go index b27e2fe4d2..c248f8dfd1 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisions.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisions.go @@ -105,10 +105,33 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer if err != nil { return err } - defer func() { + // released before the revision cleanup at the end, or on any return before that + unlocked := false + unlockNode := func() { + if unlocked { + return + } + unlocked = true _ = unlock() _ = os.Remove(fs.lu.MetadataBackend().LockfilePath(n)) - }() + } + defer unlockNode() + + // The revision must still exist on disk. Its metadata may come from a stale cache + // entry: another process, e.g. `opencloud storage-users revisions cleanup` with the + // default in-memory metadata cache, may have deleted it. Restoring it anyway would + // point the node to the revision's deleted blob, and it would leave a revision of the + // current content behind. Checked under the node's lock, which deleting a revision + // takes too (Node.DeleteRevisionUnchecked). + revisionNode := node.NewBaseNode(spaceID, revisionKey, fs.lu) + if _, err := os.Stat(revisionNode.InternalPath()); err != nil { + if !os.IsNotExist(err) { + return err + } + // drop the stale cache entry, if there is one + _ = fs.lu.MetadataBackend().Purge(ctx, revisionNode) + return errtypes.NotFound(revisionKey) + } // move current version to new revision mtime, err := n.GetMTime(ctx) @@ -118,13 +141,17 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer } // create a revision of the current node - if _, err := fs.tp.CreateRevision(ctx, n, mtime.UTC().Format(time.RFC3339Nano)); err != nil { + // (the version is kept around to protect this revision from the cleanup below) + // It is named after the mtime of the content it preserves, not after the time it is + // created; the revision cleanup takes the age of a revision from this name (see + // revisioncleanup.revisionTime). + newRevision := mtime.UTC().Format(time.RFC3339Nano) + if _, err := fs.tp.CreateRevision(ctx, n, newRevision); err != nil { return err } // revision 5, current 10 (restore a smaller blob) -> 5-10 = -5 // revision 10, current 5 (restore a bigger blob) -> 10-5 = +5 - revisionNode := node.NewBaseNode(spaceID, revisionKey, fs.lu) revisionSize, err := fs.lu.MetadataBackend().GetInt64(ctx, revisionNode, prefixes.BlobsizeAttr) if err != nil { return errtypes.InternalError("failed to read blob size xattr from old revision") @@ -151,7 +178,25 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer log.Warn().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("could not purge old revision from cache, continuing") } - return fs.tp.Propagate(ctx, n, sizeDiff) + if err := fs.tp.Propagate(ctx, n, sizeDiff); err != nil { + return err + } + + // The restore created a new revision from the previous content, so apply the + // revision cleanup strategy (see package revisioncleanup). It runs last, once the + // restore is complete, and never fails the restore. + // The revision holding the state from before the restore is always kept: a restore + // is a deliberate user action, and the strategy could otherwise drop the very + // content the user just replaced, e.g. the 'auto' strategy thins out revisions + // that are close in time. + // The node lock is released first: the cleanup takes it itself, and when it runs in the + // background, waiting for room in its queue while holding the lock could deadlock with + // a worker waiting for the lock. Releasing it also removes the lock file, which must not + // happen while the cleanup holds a lock on it, as the next writer would then lock a new + // lock file and run concurrently with the cleanup. + unlockNode() + fs.revisionsCleanup.HandleRevisionCreated(ctx, n, n.ID+node.RevisionIDDelimiter+newRevision) + return nil } // DeleteRevision deletes the specified revision of the resource @@ -177,7 +222,7 @@ func (fs *Decomposedfs) deleteRevisionFile(ctx context.Context, n *node.Node, re return errtypes.NotFound(revisionKey) } - blobID, err := n.DeleteRevision(ctx, kp[1]) + blobID, err := n.DeleteRevisionUnchecked(ctx, kp[1]) if err != nil { return err } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/spaceidindex/spaceidindex.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/spaceidindex/spaceidindex.go index 5ed1de11b1..f538f56e3e 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/spaceidindex/spaceidindex.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/spaceidindex/spaceidindex.go @@ -4,6 +4,8 @@ import ( "io" "os" "path/filepath" + "slices" + "strings" "time" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/mtimesyncedcache" @@ -49,6 +51,22 @@ func (i *Index) Load(index string) (map[string]string, error) { return i.readSpaceIndex(indexPath, i.name+":"+index, fi.ModTime()) } +// Indexes returns the names of the indexes, sorted, e.g. the space types of an index by type. +func (i *Index) Indexes() ([]string, error) { + entries, err := os.ReadDir(filepath.Join(i.root, i.name)) + if err != nil { + return nil, err + } + names := []string{} + for _, e := range entries { + if name, ok := strings.CutSuffix(e.Name(), ".mpk"); ok && e.Type().IsRegular() { + names = append(names, name) + } + } + slices.Sort(names) + return names, nil +} + // Add adds an entry to an index // Consider calling AddAll() when trying to add multiple entries as every Add call has to lock the index func (i *Index) Add(index, key string, value string) error { diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/revisions.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/revisions.go index 33f731a7a8..5261d44561 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/revisions.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/revisions.go @@ -154,6 +154,13 @@ func (tp *Tree) ListRevisions(ctx context.Context, ref *provider.Reference) (rev return nil, errtypes.NotFound(f) } + return tp.ListRevisionsUnchecked(ctx, n) +} + +// ListRevisionsUnchecked lists all revisions of the given node without checking permissions. +// ListRevisions checks the permissions of the requesting user before delegating here, +// the revision cleanup calls it directly as it is not acting on behalf of a user. +func (tp *Tree) ListRevisionsUnchecked(ctx context.Context, n *node.Node) (revisions []*provider.FileVersion, err error) { revisions = []*provider.FileVersion{} np := n.InternalPath() if items, err := filepath.Glob(np + node.RevisionIDDelimiter + "*"); err == nil { @@ -237,6 +244,14 @@ func (tp *Tree) DownloadRevision(ctx context.Context, ref *provider.Reference, r } baseNode := node.NewBaseNode(spaceID, revisionKey, tp.lookup) + // the revision's metadata may come from a stale cache entry, after another process + // deleted the revision (see Decomposedfs.RestoreRevision) + if _, err := os.Stat(baseNode.InternalPath()); err != nil { + if os.IsNotExist(err) { + return nil, nil, errtypes.NotFound(revisionKey) + } + return nil, nil, err + } blobid, blobsize, err := tp.lookup.ReadBlobIDAndSizeAttr(ctx, baseNode, nil) if err != nil { return nil, nil, errors.Wrapf(err, "Decomposedfs: could not read blob id and size for revision '%s' of node '%s'", kp[1], n.ID) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/tree.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/tree.go index 9bb49bfc25..ad48ba3441 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/tree.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/tree.go @@ -40,6 +40,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator" "github.com/opencloud-eu/reva/v2/pkg/utils" "github.com/pkg/errors" @@ -68,6 +69,8 @@ type Tree struct { idCache store.Store } +var _ node.Tree = &Tree{} + // PermissionCheckFunc defined a function used to check resource permissions type PermissionCheckFunc func(rp *provider.ResourcePermissions) bool @@ -321,6 +324,24 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) // ListFolder lists the content of a folder node func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, error) { + return t.listFolder(ctx, n, true) +} + +var _ revisioncleanup.Walker = (*Tree)(nil) + +// ListFolderKnownUnchecked lists the children of n that are known to the storage, for +// maintenance, see revisioncleanup.Walker. Unlike ListFolder, it checks no permissions, so it +// needs no user in the context. Known children are the files and folders the storage has +// registered with a node id; it never assimilates (registers) unknown ones. In this tree, every +// child is known, as all files are created through the storage, so it lists all children. The +// distinction matters for posix, where files can be created on disk directly. +func (t *Tree) ListFolderKnownUnchecked(ctx context.Context, n *node.Node) ([]*node.Node, error) { + return t.listFolder(ctx, n, false) +} + +// listFolder lists the children of n, leaving out the ones denied to the user in ctx if +// forUser. +func (t *Tree) listFolder(ctx context.Context, n *node.Node, forUser bool) ([]*node.Node, error) { ctx, span := tracer.Start(ctx, "ListFolder") defer span.End() dir := n.InternalPath() @@ -391,8 +412,10 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro continue } - // prevent listing denied resources - if !child.IsDenied(ctx) { + // prevent listing denied resources, for users only: a listing for maintenance + // (ListFolderKnownUnchecked) has no user in the context, IsDenied would panic + // without one, and it must see all children, also the ones denied to users + if !forUser || !child.IsDenied(ctx) { if child.SpaceRoot == nil { child.SpaceRoot = n.SpaceRoot } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/store.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/store.go index 6de0976f47..d66ac28f53 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/store.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/store.go @@ -42,6 +42,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -66,7 +67,9 @@ type DecomposedFsStore struct { async bool tknopts options.TokenOptions disableVersioning bool - log *zerolog.Logger + // deletes old revisions after an upload created a new one, disabled without a strategy + revisionsCleanup *revisioncleanup.Service + log *zerolog.Logger } // NewSessionStore returns a new DecomposedFsStore @@ -80,11 +83,20 @@ func NewSessionStore(fs storage.FS, aspects aspects.Aspects, root string, async async: async, tknopts: tknopts, disableVersioning: aspects.DisableVersioning, + revisionsCleanup: aspects.RevisionsCleanup, um: aspects.UserMapper, log: log, } } +// CleanupRevisions deletes old revisions of the node according to the configured strategy, +// right away or in the background (see revisioncleanup.Service). +// It is a noop if no strategy is configured or while an upload to the node is still processing. +// See package revisioncleanup for when and why it is called. +func (store DecomposedFsStore) CleanupRevisions(ctx context.Context, n *node.Node) { + store.revisionsCleanup.HandleRevisionCreated(ctx, n) +} + // New returns a new upload session func (store DecomposedFsStore) New(ctx context.Context) *DecomposedFsSession { return &DecomposedFsSession{ @@ -349,6 +361,9 @@ func (store DecomposedFsStore) updateExistingNode(ctx context.Context, session * if !store.disableVersioning { span.AddEvent("CreateVersion") + // the revision is named after the mtime of the content it preserves, not after the + // time it is created; the revision cleanup takes the age of a revision from this + // name (see revisioncleanup.revisionTime) timestamp := oldNodeMtime.UTC().Format(time.RFC3339Nano) versionID := n.ID + node.RevisionIDDelimiter + timestamp versionPath, err := session.store.tp.CreateRevision(ctx, n, timestamp) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go index 813c2e57f4..10a29f18fa 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go @@ -264,6 +264,17 @@ func (session *DecomposedFsSession) FinishUploadDecomposed(ctx context.Context) return err } metrics.UploadSessionsFinalized.Inc() + + // Apply the revision cleanup strategy (see package revisioncleanup), but only if + // this upload replaced an existing file and thus created a revision, recorded in + // versionID by DecomposedFsStore.updateExistingNode. It runs only now, after + // Finalize wrote the blob and session.Cleanup removed the processing mark: from + // that point on this upload no longer needs any revision to write into or to + // roll back to. Async uploads get the same treatment in + // Decomposedfs.Postprocessing once their postprocessing finished. + if session.info.MetaData["versionID"] != "" { + session.store.CleanupRevisions(ctx, n) + } } return session.store.tp.Propagate(ctx, n, session.SizeDiff()) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_auto.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_auto.go new file mode 100644 index 0000000000..be977cc42a --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_auto.go @@ -0,0 +1,195 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisions + +import ( + "context" + "fmt" + "time" + + "github.com/rs/zerolog" +) + +const ( + autoStrategy = "auto" +) + +type autoRevisionCleanupStrategy struct { +} + +var _ RevisionCleanupStrategy = &autoRevisionCleanupStrategy{} + +func newAutoRevisionCleanupStrategy() (RevisionCleanupStrategy, error) { + return &autoRevisionCleanupStrategy{}, nil +} + +func newAutoRevisionCleanupStrategyFromString(config string) (RevisionCleanupStrategy, error) { + if config != autoStrategy { + return nil, nil // we don't support this + } + return newAutoRevisionCleanupStrategy() +} + +// bucketPolicy defines a time window and the resolution slot within that window. +type bucketPolicy struct { + name string // value only matters for debugging, but must be unique + description string // the time window in words, for the reasons in the Plan + ageLimit time.Duration // Max age relative to 'now' to belong to this bucket + interval time.Duration // Slot duration (<=0 means keep all) + maxKeep int // Max revisions to keep per slot (<=0 means keep all, >0 caps per slot) +} + +// Standard GFS (Grandfather-Father-Son) style retention tiers. +var defaultPolicy = []bucketPolicy{ + { + name: "LastMinute", + description: "last minute", + ageLimit: 1 * time.Minute, // last minute + maxKeep: 1, // one version + interval: 10 * time.Second, // is kept every 10 seconds + }, + { + name: "LastHour", + description: "last hour", + ageLimit: 1 * time.Hour, // last hour + maxKeep: 1, // one version + interval: 1 * time.Minute, // is kept every minute + }, + { + name: "Last24H", + description: "last 24 hours", + ageLimit: 24 * time.Hour, // last 24 hours + maxKeep: 1, // one version + interval: 1 * time.Hour, // is kept every hour + }, + { + name: "Last30D", + description: "last 30 days", + ageLimit: 30 * 24 * time.Hour, // last 30 days + maxKeep: 1, // one version + interval: 24 * time.Hour, // is kept every day + }, + { + name: "Older30D", + description: "older than 30 days", + ageLimit: 0, // older than 30 days (0 AgeLimit denotes the catch-all "older" tier) + maxKeep: 1, // one version + interval: 7 * 24 * time.Hour, // s kept every week + }, +} + +// evaluateRetention thins revisions across dynamic time tiers based on interval slots. +// Every decision in the returned Plan comes with a reason that names the tier and, for tiers +// with slots, the slot, so that the outcome of this not quite trivial bucket system can be +// explained (e.g. in the debug log or in verbose output). +func evaluateRetention(revisions []Revision, policies []bucketPolicy, now time.Time) Plan { + // First, iterate over all revisions to group/sort them into discrete, + // mutually exclusive buckets (one revision can only ever be in one bucket). + // One bucket (of revisions) corresponds to one bucketPolicy. + bucket := make(map[string][]Revision) + for _, rev := range revisions { + t := rev.LastModificationTime() + age := now.Sub(t) + + matched := false + for _, policy := range policies { + if policy.ageLimit > 0 && age <= policy.ageLimit { + bucket[policy.name] = append(bucket[policy.name], rev) + matched = true + break + } + } + + // if we couldn't find a fitting bucket for this rev yet, + // then put it into the last bucket, which is assumed to be the + // "catch-all" one for "older than": + if !matched && len(policies) > 0 { + lastPolicy := policies[len(policies)-1] + bucket[lastPolicy.name] = append(bucket[lastPolicy.name], rev) + } + } + + plan := NewSimplePlan() + + // Thin each bucket according to its interval resolution and MaxKeep limit + for _, policy := range policies { + items := bucket[policy.name] + if len(items) == 0 { + continue // let's stop right here, nothing to do + } + kept := fmt.Sprintf("kept by %q (%s): ", autoStrategy, policy.description) + deleted := fmt.Sprintf("deleted by %q (%s): ", autoStrategy, policy.description) + + if policy.interval <= 0 { + // if interval is not set (<=0), then we just keep maxKeep items across the whole bucket: + // keep the first maxKeep, and delete everything else (keeping in mind that revisions + // are sorted in newest-first order) + switch { + case policy.maxKeep <= 0: + // if maxKeep is not set (<=0), then we retain *all* of the revisions in the bucket + plan.Retain(kept+"all revisions are kept", items...) + case policy.maxKeep < len(items): + // we keep less revisions than we have in total in this bucket + // => we need to do something: since they are sorted in newest-first order, + // we can simply split the revisions slice in two: + retain := items[:policy.maxKeep] // the first ones up to maxKeep are retained + delete := items[policy.maxKeep:] // everything after maxKeep is deleted + plan.Retain(kept+fmt.Sprintf("one of the %d newest", policy.maxKeep), retain...) + plan.Delete(deleted+fmt.Sprintf("beyond the %d newest", policy.maxKeep), delete...) + default: + // maxKeep is larger than the number of items => retain everything + plan.Retain(kept+fmt.Sprintf("one of the %d newest", policy.maxKeep), items...) + } + } else { + // interval is set for this bucket, in which case we need to apply the slightly + // more complex policy of putting revisions within this bucket into slots of + // duration of ${interval}, and then count the number of revisions in each slot + // to determine which to delete (or not) + + // we will use unix nano timestamps as the keys for the map of slots, + // just as a performance improvement to avoid using time.Duration objects + slotCounts := make(map[int64]int) + + for _, rev := range items { + // we take the revision's mtime, but truncate it to the interval for this + // bucket (e.g. 1m or 1h), in order to count how many revisions we have + // in each slot within the bucket; note that we also transform the + // time.Duration into an int64 to use that as the key in the slot map + slotKey := rev.LastModificationTime().Truncate(policy.interval).UnixNano() + slot := fmt.Sprintf("its slot of %s starting %s", humanDuration(policy.interval), formatTime(time.Unix(0, slotKey).UTC())) + + count := slotCounts[slotKey] + switch { + case policy.maxKeep <= 0: + // if maxKeep is not set (<=0), then we keep them all anyways + plan.Retain(kept+"all revisions are kept", rev) + case count < policy.maxKeep: + // if the current count of revisions in this slot is smaller than maxKeep, + // then we also keep this revision + slotCounts[slotKey] = count + 1 + if policy.maxKeep == 1 { + plan.Retain(kept+"the newest in "+slot, rev) + } else { + plan.Retain(kept+fmt.Sprintf("one of the %d newest in %s", policy.maxKeep, slot), rev) + } + default: + // if maxKeep is set (>0) and the count of revisions within its assigned + // slot is higher than maxKeep, then the revision must be deleted + if policy.maxKeep == 1 { + plan.Delete(deleted+slot+" already keeps a newer revision", rev) + } else { + plan.Delete(deleted+fmt.Sprintf("%s already keeps %d newer revisions", slot, policy.maxKeep), rev) + } + } + } + } + } + + return plan +} + +func (s *autoRevisionCleanupStrategy) Cleanup(now time.Time, revs []Revision, ctx context.Context, logger *zerolog.Logger) (Plan, error) { + // no need to sort revisions, they are guaranteed to be sorted from newest to oldest + return evaluateRetention(revs, defaultPolicy, now), nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_chain.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_chain.go new file mode 100644 index 0000000000..cece35b722 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_chain.go @@ -0,0 +1,78 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisions + +import ( + "context" + "slices" + "time" + + "github.com/rs/zerolog" +) + +// A RevisionCleanupStrategy that applies a sequence of strategies one after the other. +// +// Every strategy in the chain only gets to see the revisions that the previous ones +// retained: revisions that were deleted or safeguarded by an earlier strategy are final. +type chainRevisionCleanupStrategy struct { + chain []RevisionCleanupStrategy +} + +// chainAwareStrategy is implemented by strategies whose decision depends on how many +// revisions the earlier strategies of a chain have already safeguarded. +// +// This is needed for count based deletion: in "7d, 10", "10" means "keep at most 10 +// revisions in total", not "keep 10 more on top of the ones younger than 7 days". As +// the chain only passes on the remaining revisions, the strategy needs to be told how +// many revisions it has to count in addition. Safeguarded revisions are always newer +// than the remaining ones (both safeguarding strategies protect the newest revisions), +// so they are exactly the ones counting towards the limit first. +type chainAwareStrategy interface { + cleanupInChain(now time.Time, revs []Revision, safeguarded int, ctx context.Context, logger *zerolog.Logger) (Plan, error) +} + +var _ RevisionCleanupStrategy = &chainRevisionCleanupStrategy{} + +func newChainRevisionCleanupStrategy(chain []RevisionCleanupStrategy) (RevisionCleanupStrategy, error) { + return &chainRevisionCleanupStrategy{ + chain: chain, + }, nil +} + +func (s *chainRevisionCleanupStrategy) Cleanup(now time.Time, revs []Revision, ctx context.Context, logger *zerolog.Logger) (Plan, error) { + masterPlan := NewSimplePlan() + remaining := slices.Clone(revs) + var last Plan + for _, strategy := range s.chain { + var plan Plan + var err error + if aware, ok := strategy.(chainAwareStrategy); ok { + plan, err = aware.cleanupInChain(now, remaining, len(masterPlan.Safeguarded()), ctx, logger) + } else { + plan, err = strategy.Cleanup(now, remaining, ctx, logger) + } + if err != nil { + return nil, err + } + + // the decisions of this step are final, they keep the reasons this step gave + for _, rev := range plan.Deleted() { + masterPlan.Delete(plan.Reason(rev.Key()), rev) + } + for _, rev := range plan.Safeguarded() { + masterPlan.Safeguard(plan.Reason(rev.Key()), rev) + } + remaining = slices.Clone(plan.Retained()) + last = plan + } + // the revisions no step decided about are kept, with the reason the last step gave + for _, rev := range remaining { + reason := "kept: not decided by any rule" + if last != nil && last.Reason(rev.Key()) != "" { + reason = last.Reason(rev.Key()) + } + masterPlan.Retain(reason, rev) + } + return masterPlan, nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_count_deletion.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_count_deletion.go new file mode 100644 index 0000000000..c00b9e3584 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_count_deletion.go @@ -0,0 +1,67 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisions + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/rs/zerolog" +) + +// A RevisionCleanupStrategy that keeps at most a given number of revisions, the newest +// ones, and deletes all older revisions. +// +// Within a chain the limit is absolute: revisions safeguarded by earlier strategies of +// the chain count towards it (see chainAwareStrategy). +type countDeletionRevisionCleanupStrategy struct { + maxVersionsPerFile int +} + +var _ RevisionCleanupStrategy = &countDeletionRevisionCleanupStrategy{} + +var _ chainAwareStrategy = &countDeletionRevisionCleanupStrategy{} + +func newCountDeletionRevisionCleanupStrategy(maxVersionsPerFile int) (RevisionCleanupStrategy, error) { + return &countDeletionRevisionCleanupStrategy{ + maxVersionsPerFile: maxVersionsPerFile, + }, nil +} + +func newCountDeletionRevisionCleanupStrategyFromString(config string) (RevisionCleanupStrategy, error) { + if match, maxVersionsPerFile, err := countFromConfig(config); !match { + return nil, nil + } else if err != nil { + return nil, err + } else { + return newCountDeletionRevisionCleanupStrategy(maxVersionsPerFile) + } +} + +func (s *countDeletionRevisionCleanupStrategy) Cleanup(now time.Time, revs []Revision, ctx context.Context, logger *zerolog.Logger) (Plan, error) { + return s.cleanupInChain(now, revs, 0, ctx, logger) +} + +func (s *countDeletionRevisionCleanupStrategy) cleanupInChain(now time.Time, revs []Revision, safeguarded int, ctx context.Context, logger *zerolog.Logger) (Plan, error) { + plan := NewSimplePlan() + // the safeguarded revisions already use up part of the limit, and if there are more + // of them than the limit allows, all remaining revisions have to go + keep := max(s.maxVersionsPerFile-safeguarded, 0) + spec := strconv.Itoa(s.maxVersionsPerFile) + counted := "" + if safeguarded > 0 { + counted = fmt.Sprintf(", counting the %d protected ones", safeguarded) + } + retained := fmt.Sprintf("kept by %q: one of the %d newest%s", spec, s.maxVersionsPerFile, counted) + if len(revs) <= keep { + plan.Retain(retained, revs...) + return plan, nil + } + // no need to sort revisions, they are guaranteed to be sorted from newest to oldest + plan.Delete(fmt.Sprintf("deleted by %q: beyond the %d newest%s", spec, s.maxVersionsPerFile, counted), revs[keep:]...) + plan.Retain(retained, revs[:keep]...) + return plan, nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_count_retention.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_count_retention.go new file mode 100644 index 0000000000..ae606c6813 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_count_retention.go @@ -0,0 +1,44 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisions + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/rs/zerolog" +) + +type countRetentionRevisionCleanupStrategy struct { + versionsToSafeguard int +} + +var _ RevisionCleanupStrategy = &countRetentionRevisionCleanupStrategy{} + +func newCountRetentionRevisionCleanupStrategy(maxVersionsPerFile int) (RevisionCleanupStrategy, error) { + return &countRetentionRevisionCleanupStrategy{ + versionsToSafeguard: maxVersionsPerFile, + }, nil +} + +func newCountRetentionRevisionCleanupStrategyFromString(config string) (RevisionCleanupStrategy, error) { + if match, maxVersionsPerFile, err := countFromConfig(config); !match { + return nil, nil + } else if err != nil { + return nil, err + } else { + return newCountRetentionRevisionCleanupStrategy(maxVersionsPerFile) + } +} + +func (s *countRetentionRevisionCleanupStrategy) Cleanup(now time.Time, revs []Revision, ctx context.Context, logger *zerolog.Logger) (Plan, error) { + plan := NewSimplePlan() + n := min(s.versionsToSafeguard, len(revs)) + spec := strconv.Itoa(s.versionsToSafeguard) + plan.Safeguard(fmt.Sprintf("protected by %q: one of the %d newest", spec, s.versionsToSafeguard), revs[:n]...) + plan.Retain(fmt.Sprintf("not protected by %q: not one of the %d newest", spec, s.versionsToSafeguard), revs[n:]...) + return plan, nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_duration_deletion.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_duration_deletion.go new file mode 100644 index 0000000000..5cd69ab6c8 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_duration_deletion.go @@ -0,0 +1,68 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisions + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/rs/zerolog" +) + +// A RevisionCleanupStrategy that deletes files that are older than a given duration. +// +// With truncateToMidnight ("ND" or "NW" instead of "Nd" or "Nw"), the cutoff is aligned to +// local midnight instead of being exactly the duration before now, see computeCutoffFromRetention. +type durationDeletionRevisionCleanupStrategy struct { + retention time.Duration + truncateToMidnight bool + // spec is the duration as configured, e.g. "7d", for the reasons in the Plan; when empty, + // it is derived from retention, which would turn "7d" into "1w" + spec string +} + +var _ RevisionCleanupStrategy = &durationDeletionRevisionCleanupStrategy{} + +func newDurationDeletionRevisionCleanupStrategy(retention time.Duration, truncateToMidnight bool) (RevisionCleanupStrategy, error) { + return &durationDeletionRevisionCleanupStrategy{ + retention: retention, + truncateToMidnight: truncateToMidnight, + }, nil +} + +func newDurationDeletionRevisionCleanupStrategyFromString(config string) (RevisionCleanupStrategy, error) { + if matches, duration, truncateToMidnight, err := durationFromConfig(config); !matches { + return nil, nil + } else if err != nil { + return nil, err + } else { + strategy, err := newDurationDeletionRevisionCleanupStrategy(duration, truncateToMidnight) + if err != nil { + return nil, err + } + strategy.(*durationDeletionRevisionCleanupStrategy).spec = strings.TrimSpace(config) + return strategy, nil + } +} + +func (s *durationDeletionRevisionCleanupStrategy) Cleanup(now time.Time, revs []Revision, ctx context.Context, logger *zerolog.Logger) (Plan, error) { + cutoff := computeCutoffFromRetention(now, -s.retention, s.truncateToMidnight) + spec := s.spec + if spec == "" { + spec = formatDurationSpec(s.retention, s.truncateToMidnight) + } + deleted := fmt.Sprintf("deleted by %q: older than %s", spec, formatTime(cutoff)) + retained := fmt.Sprintf("kept by %q: not older than %s", spec, formatTime(cutoff)) + plan := NewSimplePlan() + for _, rev := range revs { + if rev.LastModificationTime().Before(cutoff) { + plan.Delete(deleted, rev) + } else { + plan.Retain(retained, rev) + } + } + return plan, nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_duration_retention.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_duration_retention.go new file mode 100644 index 0000000000..37c6cfeb87 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/cleanup_strategy_duration_retention.go @@ -0,0 +1,74 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisions + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/rs/zerolog" +) + +// A RevisionCleanupStrategy that retains files that are older than a given duration. +// +// With truncateToMidnight ("ND" or "NW" instead of "Nd" or "Nw"), the cutoff is aligned to +// local midnight instead of being exactly the duration before now, see computeCutoffFromRetention. +type durationRetentionRevisionCleanupStrategy struct { + retention time.Duration + truncateToMidnight bool + // spec is the duration as configured, e.g. "7d", for the reasons in the Plan; when empty, + // it is derived from retention, which would turn "7d" into "1w" + spec string +} + +var _ RevisionCleanupStrategy = &durationRetentionRevisionCleanupStrategy{} + +func newDurationRetentionRevisionCleanupStrategy(retention time.Duration, truncateToMidnight bool) (RevisionCleanupStrategy, error) { + return &durationRetentionRevisionCleanupStrategy{ + retention: retention, + truncateToMidnight: truncateToMidnight, + }, nil +} + +func newDurationRetentionRevisionCleanupStrategyFromString(config string) (RevisionCleanupStrategy, error) { + if matches, duration, truncateToMidnight, err := durationFromConfig(config); !matches { + return nil, nil + } else if err != nil { + return nil, err + } else { + strategy, err := newDurationRetentionRevisionCleanupStrategy(duration, truncateToMidnight) + if err != nil { + return nil, err + } + strategy.(*durationRetentionRevisionCleanupStrategy).spec = strings.TrimSpace(config) + return strategy, nil + } +} + +func (s *durationRetentionRevisionCleanupStrategy) Cleanup(now time.Time, revs []Revision, ctx context.Context, logger *zerolog.Logger) (Plan, error) { + cutoff := computeCutoffFromRetention(now, -s.retention, s.truncateToMidnight) + spec := s.spec + if spec == "" { + spec = formatDurationSpec(s.retention, s.truncateToMidnight) + } + safeguarded := fmt.Sprintf("protected by %q: newer than %s", spec, formatTime(cutoff)) + retained := fmt.Sprintf("not protected by %q: not newer than %s", spec, formatTime(cutoff)) + plan := NewSimplePlan() + for _, rev := range revs { + if rev.LastModificationTime().After(cutoff) { + // revision is younger than the cutoff time: + // do nothing in order to drop the revision from the revisions that + // need to be processed by following-up strategies + plan.Safeguard(safeguarded, rev) + } else { + // revision is older than the cutoff time: + // mark it for retention, in order to be processed by any + // strategy that comes next + plan.Retain(retained, rev) + } + } + return plan, nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/model.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/model.go new file mode 100644 index 0000000000..9d13fe9ea0 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/model.go @@ -0,0 +1,41 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisions + +import ( + "context" + "time" + + "github.com/rs/zerolog" +) + +type Revision interface { + Key() string + LastModificationTime() time.Time + Size() uint64 +} + +// Plan is the outcome of a RevisionCleanupStrategy: which revisions to delete, which to +// retain (keep, or hand on to the next strategy of a chain) and which to safeguard (keep +// for sure). +// +// Every decision comes with a human readable reason, e.g. `deleted by "30d": older than +// 2026-09-14T10:00:00Z`, so that it can be explained why a revision was deleted or kept, +// in the debug log of the cleanup or in verbose output. The reasons name the rule in the +// syntax of the configuration and the concrete boundary it was evaluated with. +type Plan interface { + Delete(reason string, revs ...Revision) + Retain(reason string, revs ...Revision) + Safeguard(reason string, revs ...Revision) + Deleted() []Revision + Retained() []Revision + Safeguarded() []Revision + // Reason returns the reason for the decision about the revision with the given key, + // "" if the plan has no decision about it. + Reason(key string) string +} + +type RevisionCleanupStrategy interface { + Cleanup(now time.Time, revs []Revision, ctx context.Context, logger *zerolog.Logger) (Plan, error) +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/model_simple.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/model_simple.go new file mode 100644 index 0000000000..b6a97cc787 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/model_simple.go @@ -0,0 +1,59 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisions + +type SimplePlan struct { + retained []Revision + deleted []Revision + safeguarded []Revision + reasons map[string]string // by revision key +} + +var _ Plan = &SimplePlan{} + +func NewSimplePlan() Plan { + return &SimplePlan{ + retained: []Revision{}, + deleted: []Revision{}, + safeguarded: []Revision{}, + reasons: map[string]string{}, + } +} + +func (p *SimplePlan) Delete(reason string, revs ...Revision) { + p.deleted = append(p.deleted, revs...) + p.setReason(reason, revs) +} + +func (p *SimplePlan) Retain(reason string, revs ...Revision) { + p.retained = append(p.retained, revs...) + p.setReason(reason, revs) +} + +func (p *SimplePlan) Safeguard(reason string, revs ...Revision) { + p.safeguarded = append(p.safeguarded, revs...) + p.setReason(reason, revs) +} + +func (p *SimplePlan) setReason(reason string, revs []Revision) { + for _, rev := range revs { + p.reasons[rev.Key()] = reason + } +} + +func (p *SimplePlan) Deleted() []Revision { + return p.deleted +} + +func (p *SimplePlan) Retained() []Revision { + return p.retained +} + +func (p *SimplePlan) Safeguarded() []Revision { + return p.safeguarded +} + +func (p *SimplePlan) Reason(key string) string { + return p.reasons[key] +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/registry.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/registry.go new file mode 100644 index 0000000000..a23ac61c61 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/registry.go @@ -0,0 +1,296 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisions + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +// valuePattern matches a single value X: a number of versions, or a duration with a unit +// suffix, where durations in days or weeks with an uppercase unit (D, W) are aligned to midnight. +const valuePattern = `\d+[wdhmsDW]?` + +var ( + configPatternAutoOnly = regexp.MustCompile(`^(auto)$`) + configPatternDeleteOnly = regexp.MustCompile(`^(` + valuePattern + `)$`) + configPatternSafeguardThenAuto = regexp.MustCompile(`^(` + valuePattern + `)\s*,\s*(auto)$`) + configPatternAutoThenDelete = regexp.MustCompile(`^(auto)\s*,\s*(` + valuePattern + `)$`) + configPatternSafeguardThenDelete = regexp.MustCompile(`^(` + valuePattern + `)\s*,\s*(` + valuePattern + `)$`) + configPatternSafeguardAutoAndDelete = regexp.MustCompile(`^(` + valuePattern + `)\s*,\s*(auto)\s*,\s*(` + valuePattern + `)$`) + + configPatternCount = regexp.MustCompile(`^\d+$`) +) + +// This is the entry point for instantiating a strategy that may be composed of multiple ones. +// +// The config parameter is a string that denotes which strategy/ies to use, as a comma separated +// list of specifications. +// +// Each X is either a plain number, which is a number of revisions, or a number with a unit +// suffix (w for weeks, d for days, h for hours, m for minutes, s for seconds), which is a +// duration. A number of revisions always refers to the newest revisions. +// +// A duration in days or weeks with an uppercase unit (D, W) is aligned to midnight in the server's +// local time zone: "1d" means the last 24 hours, while "1D" means since yesterday 00:00, so +// today and yesterday; "0D" means since today 00:00, and "1W" since 00:00 seven days ago. +// Depending on the time of day, "ND" thus spans between N and N+1 days. +// +// Supported forms: +// - "auto": Default, default clean-up pattern applies +// - "X": Delete all versions beyond X: keep the X newest versions, or delete versions +// that are older than X, without any further thinning +// "0" deletes all versions: unlike disabling versioning, which only stops new versions +// from being created, this also removes the versions that have accumulated so far. +// - "X, auto": Keeps versions within X, apply default clean-up pattern to all other versions +// - "auto, X": Delete all versions beyond X, apply the default clean-up pattern to the others +// - "X1, X2": Keep versions within X1 and delete versions beyond X2. X1 and X2 must be of +// different kinds, i.e. one number of versions and one duration, e.g. "5, 30d" (delete +// versions older than 30 days, but always keep the 5 newest) or "7d, 10" (keep at most +// 10 versions, but never delete versions younger than 7 days). +// Two values of the same kind are rejected, as one of them would have no effect: +// "5, 10" is the same as "10", and "7d, 30d" is the same as "30d". +// - "X1, auto, X2": Keep versions within X1, delete versions beyond X2, and apply the +// default clean-up pattern to the versions in between, e.g. "7d, auto, 90d". +// If X1 and X2 are of the same kind, X1 must be smaller than X2, otherwise the default +// clean-up pattern would never apply. +// +// "disabled" and an empty specification are not strategies: the callers handle them +// without parsing a strategy (see revisioncleanup.New), as there is nothing to plan. +// +// A deletion by number of versions is absolute: in "7d, 10" the versions younger than 7 +// days count towards the 10 versions to keep. +// +// In "auto, N" and "X1, auto, N", the number N applies before "auto", not after: "auto, 10" +// first deletes all but the 10 newest versions, and then thins out those 10, so it usually +// keeps fewer than 10, and possibly fewer than "auto" alone would. If the 10 newest versions +// were all saved within the same minute yesterday, "auto, 10" keeps a single version, while +// "auto" would keep one of them plus older ones. This is easily misread as "thin out first, +// then keep at most 10", which is not what it means. +// +// The age of a version, which durations are compared with, is Revision.LastModificationTime: +// the time its content was written, i.e. the modification time the file had before the change +// that turned it into a version, not the time of that change (see revisioncleanup.revisionTime). A version created today from content that was unchanged for +// two months is two months old. +func NewRevisionCleanupStrategy(config string) (RevisionCleanupStrategy, error) { + config = strings.TrimSpace(config) + + // "auto": Default, default clean-up pattern applies + if m := configPatternAutoOnly.FindStringSubmatch(config); len(m) > 0 { + auto := m[1] + return newAutoRevisionCleanupStrategyFromString(auto) + } + + // "X": Delete all versions beyond X, no thinning + if m := configPatternDeleteOnly.FindStringSubmatch(config); len(m) > 0 { + return parseDelete(m[1]) + } + + // "X, auto": Keeps versions within X, apply default clean-up pattern to all other versions + if m := configPatternSafeguardThenAuto.FindStringSubmatch(config); len(m) > 0 { + return parseChain( + func() (RevisionCleanupStrategy, error) { return parseKeep(m[1]) }, + func() (RevisionCleanupStrategy, error) { return parseAuto(m[2]) }, + ) + } + + // "auto, X": Delete all versions beyond X, apply the default clean-up pattern to the others. + // The deletion comes first in the chain, so a number of versions is applied before the + // thinning, see the doc comment above. + if m := configPatternAutoThenDelete.FindStringSubmatch(config); len(m) > 0 { + return parseChain( + func() (RevisionCleanupStrategy, error) { return parseDelete(m[2]) }, + func() (RevisionCleanupStrategy, error) { return parseAuto(m[1]) }, + ) + } + + // "X1, X2": Keep versions within X1 and delete versions beyond X2 + if m := configPatternSafeguardThenDelete.FindStringSubmatch(config); len(m) > 0 { + keep, delete := m[1], m[2] + if sameKind(keep, delete) { + equivalent, err := sameKindEquivalent(keep, delete) + if err != nil { + return nil, err + } + return nil, fmt.Errorf("unsupported revision cleanup strategy specification %q: "+ + "%q and %q are both %s, so one of them has no effect and it is the same as %s; "+ + "combine a number of versions with a duration instead, e.g. \"5, 30d\" or \"7d, 10\"", + config, keep, delete, kindName(keep), equivalent) + } + return parseChain( + func() (RevisionCleanupStrategy, error) { return parseKeep(keep) }, + func() (RevisionCleanupStrategy, error) { return parseDelete(delete) }, + ) + } + + // "X1, auto, X2": Keep versions within X1, delete versions beyond X2, thin out the ones in between. + // As in "auto, X", a number of versions X2 is applied before the thinning. + if m := configPatternSafeguardAutoAndDelete.FindStringSubmatch(config); len(m) > 0 { + keep, auto, delete := m[1], m[2], m[3] + if sameKind(keep, delete) { + smaller, err := alwaysSmaller(keep, delete) + if err != nil { + return nil, err + } + if !smaller { + return nil, fmt.Errorf("unsupported revision cleanup strategy specification %q: "+ + "%q must be smaller than %q at any time of day, otherwise the default clean-up "+ + "pattern does not apply in between", config, keep, delete) + } + } + return parseChain( + func() (RevisionCleanupStrategy, error) { return parseKeep(keep) }, + func() (RevisionCleanupStrategy, error) { return parseDelete(delete) }, + func() (RevisionCleanupStrategy, error) { return parseAuto(auto) }, + ) + } + + return nil, fmt.Errorf("unsupported revision cleanup strategy specification %q", config) +} + +// parseChain parses the given strategies and chains them in that order. +func parseChain(parsers ...func() (RevisionCleanupStrategy, error)) (RevisionCleanupStrategy, error) { + chain := make([]RevisionCleanupStrategy, len(parsers)) + for i, parse := range parsers { + strategy, err := parse() + if err != nil { + return nil, err + } + chain[i] = strategy + } + return newChainRevisionCleanupStrategy(chain) +} + +// sameKind tells whether both values are numbers of versions, or both are durations. +func sameKind(a, b string) bool { + return configPatternCount.MatchString(a) == configPatternCount.MatchString(b) +} + +func kindName(value string) string { + if configPatternCount.MatchString(value) { + return "numbers of versions" + } + return "durations" +} + +// valueRange returns the smallest and the largest extent a value can have, as a number of +// versions for counts and in nanoseconds for durations. They are the same, except for durations +// aligned to midnight: "ND" covers between N days and just under N+1 days, depending on the +// time of day. +// +// A day counts as 24 hours here, although the days on which daylight saving time begins or ends +// have 23 or 25 hours. The comparisons based on it can therefore be off by up to an hour on those +// days, e.g. "1D, auto, 2d" is accepted, although "1D" can reach slightly beyond "2d" in the +// night after the clocks were set back. That is deliberate: rejecting such strategies for the +// sake of one hour a year would be worse, and during that hour, the versions in question are +// only kept, never deleted, as protecting them comes first in the chain. +func valueRange(value string) (int64, int64, error) { + if configPatternCount.MatchString(value) { + n, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("invalid number of versions %q: %w", value, err) + } + return n, n, nil + } + truncated := strings.HasSuffix(value, "D") || strings.HasSuffix(value, "W") + d, err := parseDuration(strings.ToLower(value)) + if err != nil { + return 0, 0, err + } + if truncated { + return int64(d), int64(d + 24*time.Hour - 1), nil + } + return int64(d), int64(d), nil +} + +// alwaysSmaller tells whether a is smaller than b at any time of day (but see valueRange about +// daylight saving time). Both must be of the same kind. Values that only reach each other at a +// single instant count as smaller, e.g. "1d" and "1D", which are equal only at midnight; values +// that are always equal, e.g. "1w" and "7d", don't. +func alwaysSmaller(a, b string) (bool, error) { + aMin, aMax, err := valueRange(a) + if err != nil { + return false, err + } + bMin, bMax, err := valueRange(b) + if err != nil { + return false, err + } + return aMax < bMin || (aMax == bMin && (aMin < aMax || bMin < bMax)), nil +} + +// sameKindEquivalent describes what an "X1, X2" pair of values of the same kind is equivalent to: +// the larger of the two, as the smaller one has no effect. If the order depends on the time of +// day, as with "1D, 30h", it is the larger of the two at any given moment. +func sameKindEquivalent(a, b string) (string, error) { + aMin, aMax, err := valueRange(a) + if err != nil { + return "", err + } + bMin, bMax, err := valueRange(b) + if err != nil { + return "", err + } + // a value that is never larger than the other one has no effect: "1d" is never larger than + // "1D", so "1d, 1D" and "1D, 1d" are both the same as "1D" + switch { + case aMax <= bMin: + return fmt.Sprintf("%q", b), nil + case bMax <= aMin, aMin == bMin && aMax == bMax: + return fmt.Sprintf("%q", a), nil + default: + return fmt.Sprintf("%q or %q, depending on the time of day", a, b), nil + } +} + +func parseKeep(retain string) (RevisionCleanupStrategy, error) { + strategy, err := newDurationRetentionRevisionCleanupStrategyFromString(retain) + if err != nil { + return nil, err + } + if strategy != nil { + return strategy, nil + } + strategy, err = newCountRetentionRevisionCleanupStrategyFromString(retain) + if err != nil { + return nil, err + } + if strategy != nil { + return strategy, nil + } + // doesn't match either + return nil, fmt.Errorf("unsupported revision safeguarding specification %q", retain) +} + +func parseDelete(delete string) (RevisionCleanupStrategy, error) { + strategy, err := newDurationDeletionRevisionCleanupStrategyFromString(delete) + if err != nil { + return nil, err + } + if strategy != nil { + return strategy, nil + } + strategy, err = newCountDeletionRevisionCleanupStrategyFromString(delete) + if err != nil { + return nil, err + } + if strategy != nil { + return strategy, nil + } + // doesn't match either + return nil, fmt.Errorf("unsupported revision deletion specification %q", delete) +} + +func parseAuto(auto string) (RevisionCleanupStrategy, error) { + if strategy, err := newAutoRevisionCleanupStrategyFromString(auto); err != nil { + return nil, err + } else if strategy != nil { + return strategy, nil + } else { + return nil, fmt.Errorf("unsupported revision 'auto' specification %q", auto) + } +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/utils.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/utils.go new file mode 100644 index 0000000000..7487461d66 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/revisions/utils.go @@ -0,0 +1,307 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package revisions + +import ( + "cmp" + "context" + "errors" + "fmt" + "math" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "github.com/rs/zerolog" +) + +// Compares two Revisions, newest first. +// +// Revisions with the same time are told apart by their keys, so that the order, and with it +// which of them a strategy keeps, never depends on the order they were listed in. When the +// posix driver creates a revision whose key is already taken, it renames the existing +// revision to ".", with N one higher than any existing suffix, and creates the new +// one under the plain key. The plain key is therefore the newest, followed by the suffixed +// ones from the highest N down. +func CompareNewestFirst[T Revision](a, b T) int { + if c := b.LastModificationTime().Compare(a.LastModificationTime()); c != 0 { + return c + } + if c := cmp.Compare(collisionRank(b.Key()), collisionRank(a.Key())); c != 0 { + return c + } + return strings.Compare(a.Key(), b.Key()) +} + +// collisionRank ranks revisions with the same time, see CompareNewestFirst: the higher, the +// newer. A key ending in "." ranks N, any other key ranks above all of those. A timestamp +// ending in fractions of a second ("...05.123Z") is not mistaken for a suffix, as it ends in +// "Z" or a time zone offset. +func collisionRank(key string) int { + i := strings.LastIndexByte(key, '.') + if i < 0 { + return math.MaxInt + } + n, err := strconv.Atoi(key[i+1:]) + if err != nil || n < 0 { + return math.MaxInt + } + return n +} + +// Sorts a slice of Revisions canonically: newest first. +func SortRevisionsCanonically[T Revision](revisions []T) { + slices.SortFunc(revisions, CompareNewestFirst) +} + +// CleanupStrategyFunc is an adapter to use an ordinary function as a RevisionCleanupStrategy, +// like http.HandlerFunc: CleanupStrategyFunc(f) is a RevisionCleanupStrategy that calls f. +type CleanupStrategyFunc func(now time.Time, revs []Revision, ctx context.Context, logger *zerolog.Logger) (Plan, error) + +var _ RevisionCleanupStrategy = CleanupStrategyFunc(nil) + +// Cleanup calls f(now, revs, ctx, logger). +func (f CleanupStrategyFunc) Cleanup(now time.Time, revs []Revision, ctx context.Context, logger *zerolog.Logger) (Plan, error) { + return f(now, revs, ctx, logger) +} + +// Takes a RevisionCleanupStrategy and acts as a glue to piece everything together, +// except the deletion of Revisions itself, that it delegates to the cleanup function +// that is passed as a parameter, together with the reason the strategy gave for deleting +// the revision (see Plan). +// +// It returns the Plan of the strategy, so that callers can report on all revisions, also +// the ones that were kept or protected, together with the errors of the cleanup function. +// The Plan is nil if the strategy failed. +// +// Note that this implementation is assuming that it is used in a single-threaded +// fashion as it does not perform any locking of anything. +func ExecuteCleanupStrategy(now time.Time, strategy RevisionCleanupStrategy, revs []Revision, ctx context.Context, logger *zerolog.Logger, + cleanup func(rev Revision, reason string) error, +) (Plan, error) { + revs = slices.Clone(revs) + SortRevisionsCanonically(revs) + + plan, err := strategy.Cleanup(now, revs, ctx, logger) + if err != nil { + return nil, err + } + plan = withFutureDates(plan, revs, now, logger) + + deleted := plan.Deleted() + + errs := make([]error, len(deleted)) + for i, rev := range deleted { + err := cleanup(rev, plan.Reason(rev.Key())) + errs[i] = err + } + return plan, errors.Join(errs...) +} + +var ( + durationParserRegex = regexp.MustCompile(`^(\d+)([wdhms])$`) +) + +func parseDuration(s string) (time.Duration, error) { + s = strings.TrimSpace(s) + m := durationParserRegex.FindStringSubmatch(s) + if len(m) < 1 { + return 0, fmt.Errorf("invalid duration number: %q", s) + } + value := m[1] + unit := m[2] + + n, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("invalid duration number: %w", err) + } + + var u time.Duration + switch unit { + case "w": + u = 7 * 24 * time.Hour + case "d": + u = 24 * time.Hour + case "h": + u = time.Hour + case "m": + u = time.Minute + case "s": + u = time.Second + default: + return 0, fmt.Errorf("unknown duration unit: %q", unit) + } + // A time.Duration holds at most about 292 years. Beyond that the multiplication would + // silently overflow, e.g. into a negative duration, which would put the cutoff into the + // future and delete all revisions instead of keeping them. + if limit := int64(math.MaxInt64 / u); int64(n) > limit { + return 0, fmt.Errorf("duration %q is too large, at most %d%s is supported", s, limit, unit) + } + return time.Duration(n) * u, nil +} + +var ( + // A duration: a number with a unit suffix (w, d, h, m, s). Durations in days or weeks + // with an uppercase unit (D, W) align the cutoff to midnight (see computeCutoffFromRetention). + durationSpec = regexp.MustCompile(`^\d+[wdhmsDW]$`) +) + +func durationFromConfig(config string) (bool, time.Duration, bool, error) { + config = strings.TrimSpace(config) + if !durationSpec.MatchString(config) { + return false, 0, false, nil + } + + truncateToMidnight := false + if unit := config[len(config)-1]; unit == 'D' || unit == 'W' { + config = strings.ToLower(config) + truncateToMidnight = true + } + + duration, err := parseDuration(config) + if err != nil { + return true, 0, false, err + } + + return true, duration, truncateToMidnight, nil +} + +// computeCutoffFromRetention computes the point in time that separates the revisions within a +// duration from the ones beyond it. d is the negative duration, e.g. -48h for "2d". +// +// With truncateToMidnight, the cutoff is aligned to midnight in the time zone of now (the server's +// local time zone in practice): it is today's midnight minus the given number of days, so "0D" +// means "since today 00:00", "1D" "since yesterday 00:00" and "1W" "since 00:00 seven days ago" +// (not since the start of the calendar week). Calendar arithmetic is used rather than subtracting +// d, so that the cutoff is exactly at midnight also across daylight saving time changes. +// Only durations in days or weeks can be truncated to midnight (see durationSpec), so d is always +// a multiple of 24h in that case. +func computeCutoffFromRetention(now time.Time, d time.Duration, truncateToMidnight bool) time.Time { + if truncateToMidnight { + year, month, day := now.Date() + midnight := time.Date(year, month, day, 0, 0, 0, 0, now.Location()) + return midnight.AddDate(0, 0, int(d/(24*time.Hour))) + } + return now.Add(d) +} + +var ( + countConfigFormat = regexp.MustCompile(`^(\d+)$`) +) + +func countFromConfig(config string) (bool, int, error) { + if m := countConfigFormat.FindStringSubmatch(config); len(m) > 1 { + if maxVersionsPerFile, err := strconv.Atoi(m[1]); err != nil { + return true, 0, err + } else if maxVersionsPerFile < 0 { + return true, 0, fmt.Errorf("invalid value for maxVersionsPerFile: %q", m[1]) + } else { + return true, maxVersionsPerFile, nil + } + } else { + return false, 0, nil + } +} + +const ( + day = 24 * time.Hour + week = 7 * day +) + +// formatDurationSpec renders a duration in the syntax of the configuration, e.g. "7d", "2w" or +// "1D" when aligned to midnight, for the reasons in a Plan. +func formatDurationSpec(d time.Duration, truncateToMidnight bool) string { + var spec string + switch { + case d == 0 && truncateToMidnight: + spec = "0d" + case d == 0: + spec = "0s" + case d%week == 0: + spec = fmt.Sprintf("%dw", d/week) + case d%day == 0: + spec = fmt.Sprintf("%dd", d/day) + case d%time.Hour == 0: + spec = fmt.Sprintf("%dh", d/time.Hour) + case d%time.Minute == 0: + spec = fmt.Sprintf("%dm", d/time.Minute) + default: + spec = fmt.Sprintf("%ds", d/time.Second) + } + if truncateToMidnight { + spec = strings.ToUpper(spec) + } + return spec +} + +// humanDuration renders a duration in words, e.g. "1 minute" or "10 seconds", for the +// reasons in a Plan. +func humanDuration(d time.Duration) string { + unit := func(n time.Duration, name string) string { + if n == 1 { + return "1 " + name + } + return fmt.Sprintf("%d %ss", n, name) + } + switch { + case d >= week && d%week == 0: + return unit(d/week, "week") + case d >= day && d%day == 0: + return unit(d/day, "day") + case d >= time.Hour && d%time.Hour == 0: + return unit(d/time.Hour, "hour") + case d >= time.Minute && d%time.Minute == 0: + return unit(d/time.Minute, "minute") + case d >= time.Second && d%time.Second == 0: + return unit(d/time.Second, "second") + default: + return d.String() + } +} + +// formatTime renders a point in time for the reasons in a Plan. +func formatTime(t time.Time) string { + return t.Format(time.RFC3339) +} + +// withFutureDates points out the revisions dated in the future in the reasons of plan, and +// logs a warning about each of them. +// +// A revision's time is the modification time of its content, which clients and changes on disk +// can set to any time, e.g. a client with a wrong clock. A revision dated in the future counts +// as the newest one: it takes the first places of a number of versions to keep, and no +// duration deletes it before its date has passed. That is kept as it is, but made visible, so +// that the admin can look into the client or the clock. +func withFutureDates(plan Plan, revs []Revision, now time.Time, logger *zerolog.Logger) Plan { + future := map[string]time.Time{} + for _, rev := range revs { + if t := rev.LastModificationTime(); t.After(now) { + future[rev.Key()] = t + if logger != nil { + logger.Warn().Str("revision", rev.Key()).Time("time", t). + Msg("revision cleanup: revision dated in the future, it counts as the newest one") + } + } + } + if len(future) == 0 { + return plan + } + return &futureDatesPlan{Plan: plan, future: future} +} + +// futureDatesPlan adds a note to the reasons of the revisions dated in the future. +type futureDatesPlan struct { + Plan + future map[string]time.Time +} + +func (p *futureDatesPlan) Reason(key string) string { + reason := p.Plan.Reason(key) + if t, ok := p.future[key]; ok && reason != "" { + reason += fmt.Sprintf("; dated in the future (%s), so it counts as the newest", formatTime(t)) + } + return reason +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/prefixes.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/prefixes.go index 855b3998ef..1a6b46186d 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/prefixes.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/prefixes.go @@ -53,6 +53,7 @@ const ( GrantPrefix string = OcisPrefix + "grant." GrantUserAcePrefix string = OcisPrefix + "grant." + ace.UserAcePrefix GrantGroupAcePrefix string = OcisPrefix + "grant." + ace.GroupAcePrefix + GrantMailAcePrefix string = OcisPrefix + "grant." + ace.MailAcePrefix MetadataPrefix string = OcisPrefix + "md." // favorite flag, per user diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node.go index 27b27d78d2..b49c93c97c 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node.go @@ -40,6 +40,7 @@ import ( "github.com/google/uuid" "github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider" "github.com/opencloud-eu/reva/v2/pkg/appctx" + "github.com/opencloud-eu/reva/v2/pkg/conversions" ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/mime" @@ -798,8 +799,8 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi // share indicator if _, ok := fieldMaskKeysMap["share-types"]; returnAllFields || ok { - granteeTypes := n.getGranteeTypes(ctx) - if len(granteeTypes) > 0 { + granteeTypes, hasGuestShares := n.getGranteeTypes(ctx) + if len(granteeTypes) > 0 || hasGuestShares { // TODO add optional property to CS3 ResourceInfo to transport grants? var s strings.Builder first := true @@ -811,6 +812,12 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi } s.WriteString(strconv.Itoa(int(t))) } + if hasGuestShares { + if !first { + s.WriteString(",") + } + s.WriteString(strconv.Itoa(int(conversions.ShareTypeGuest))) + } ri.Opaque = utils.AppendPlainToOpaque(ri.Opaque, "share-types", s.String()) } } @@ -1200,17 +1207,25 @@ func (n *Node) ListGrants(ctx context.Context) ([]*provider.Grant, error) { return grants, nil } -func (n *Node) getGranteeTypes(ctx context.Context) []provider.GranteeType { +// getGranteeTypes returns the grantee types of the node's grants. The second +// return value reports whether the node has guest shares, which are stored as +// user grants with a mail ACE prefix. +func (n *Node) getGranteeTypes(ctx context.Context) ([]provider.GranteeType, bool) { types := []provider.GranteeType{} + hasGuestShares := false if g, err := n.ListGrantees(ctx); err == nil { hasUserShares, hasGroupShares := false, false for i := range g { switch { + case !hasGuestShares && strings.HasPrefix(g[i], prefixes.GrantMailAcePrefix): + hasGuestShares = true case !hasUserShares && strings.HasPrefix(g[i], prefixes.GrantUserAcePrefix): hasUserShares = true case !hasGroupShares && strings.HasPrefix(g[i], prefixes.GrantGroupAcePrefix): hasGroupShares = true - case hasUserShares && hasGroupShares: + } + + if hasUserShares && hasGroupShares && hasGuestShares { break } } @@ -1221,7 +1236,7 @@ func (n *Node) getGranteeTypes(ctx context.Context) []provider.GranteeType { types = append(types, provider.GranteeType_GRANTEE_TYPE_GROUP) } } - return types + return types, hasGuestShares } // FindStorageSpaceRoot calls n.Parent() and climbs the tree diff --git a/vendor/modules.txt b/vendor/modules.txt index efe1d473b5..4809ecf38c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1360,7 +1360,7 @@ github.com/opencloud-eu/icap-client # github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346 ## explicit; go 1.23 github.com/opencloud-eu/libre-graph-api-go -# github.com/opencloud-eu/reva/v2 v2.52.1 +# github.com/opencloud-eu/reva/v2 v2.52.1 => github.com/pbleser-oc/reva/v2 v2.0.0-20261028165100-414fb6332286 ## explicit; go 1.26.0 github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace github.com/opencloud-eu/reva/v2/cmd/revad/runtime @@ -1650,6 +1650,7 @@ github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/mtimesyncedcache github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions +github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisioncleanup github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/spaceidindex github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/timemanager github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/trashbin @@ -1661,6 +1662,7 @@ github.com/opencloud-eu/reva/v2/pkg/storage/registry/loader github.com/opencloud-eu/reva/v2/pkg/storage/registry/registry github.com/opencloud-eu/reva/v2/pkg/storage/registry/spaces github.com/opencloud-eu/reva/v2/pkg/storage/registry/static +github.com/opencloud-eu/reva/v2/pkg/storage/revisions github.com/opencloud-eu/reva/v2/pkg/storage/utils/ace github.com/opencloud-eu/reva/v2/pkg/storage/utils/acl github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking @@ -2708,3 +2710,4 @@ stash.kopano.io/kgol/rndm # github.com/unrolled/secure => github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 # go-micro.dev/v4 => github.com/butonic/go-micro/v4 v4.11.4-0.20260929213538-cd412c159a26 # github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-20250512152754-23325793059a +# github.com/opencloud-eu/reva/v2 => github.com/pbleser-oc/reva/v2 v2.0.0-20261028165100-414fb6332286