AUT-11295 Manage workspace secrets via dedicated secrets pipeline

README.md

@@ -239,6 +239,41 @@ map[string]any{
 - 	"ciba_authentication_service":               map[string]any{"type": string("mock")},
 ```
 
+### Managing secrets
+
+Workspace secrets live behind a dedicated system API and are managed with the
+secrets-exclusive `--workspace-secrets <workspace>` flag (mutually exclusive with
+`--workspace`, `--tenant`, and `--filter`). Secret *values* are never stored in
+the repository and never returned by the server — each secret file references an
+environment variable that is resolved at push time:
+
+```yaml
+# workspaces/demo/secrets/smtp_password.yaml
+id: smtp_password
+value: '{{ env "CAC_SECRET_SMTP_PASSWORD" }}'
+```
+
+```bash
+# create stub files for remote secrets that have no local definition
+cac --config ./cac.yaml --profile dev pull --workspace-secrets demo
+
+# preview what a push would change (secret ids only, never values)
+cac --config ./cac.yaml --profile dev push --workspace-secrets demo --dry-run
+
+# create + update remote secrets from local definitions
+CAC_SECRET_SMTP_PASSWORD=... cac --config ./cac.yaml --profile dev push --workspace-secrets demo
+
+# also delete remote secrets that have no local definition
+cac --config ./cac.yaml --profile dev push --workspace-secrets demo --prune
+
+# compare local definitions against the remote workspace
+cac --config ./cac.yaml --profile dev diff --workspace-secrets demo
+```
+
+> **Note:** `pull --workspace-secrets` never overwrites existing secret files, and
+> a push fails before any API call if a referenced environment variable is unset
+> or a secret resolves to an empty value.
+
 ## Templates
 
 Templates are used to generate configuration files. They are using [Go template language](https://golang.org/pkg/text/template/).

cmd/diff.go

@@ -1,13 +1,16 @@
 package cmd
 
 import (
+	"fmt"
+	"os"
+	"strings"
+
 	"github.com/cloudentity/cac/internal/cac"
 	"github.com/cloudentity/cac/internal/cac/api"
 	"github.com/cloudentity/cac/internal/cac/diff"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"golang.org/x/exp/slog"
-	"os"
 )
 
 var (
@@ -47,6 +50,14 @@ Examples:
 				err    error
 			)
 
+			if rootConfig.WorkspaceSecrets != "" {
+				return diffSecrets(cmd)
+			}
+
+			if diffConfig.Source == "" || diffConfig.Target == "" {
+				return errors.New(`required flag(s) "source", "target" not set`)
+			}
+
 			slog.
 				With("workspace", rootConfig.Workspace).
 				With("config", rootConfig.ConfigPath).
@@ -110,6 +121,95 @@ Examples:
 	}
 )
 
+func diffSecrets(cmd *cobra.Command) error {
+	var (
+		app *cac.Application
+		err error
+	)
+
+	if len(diffConfig.Filters) > 0 {
+		return errors.New("--filter cannot be combined with --workspace-secrets")
+	}
+
+	if diffConfig.Source != "" || diffConfig.Target != "" {
+		return errors.New("--source/--target do not apply to --workspace-secrets; local files are always compared against the remote workspace")
+	}
+
+	if app, err = cac.InitApp(rootConfig.ConfigPath, rootConfig.Profile, false); err != nil {
+		return err
+	}
+
+	dirStore, err := secretsDirStore(app)
+	if err != nil {
+		return err
+	}
+
+	wid := rootConfig.WorkspaceSecrets
+
+	localIDs, err := dirStore.ListIDs(wid)
+	if err != nil {
+		return errors.Wrap(err, "failed to read local secrets")
+	}
+
+	remoteIDs, err := app.Secrets.ListIDs(cmd.Context(), wid)
+	if err != nil {
+		return err
+	}
+
+	result := secretsDiffReport(wid, localIDs, remoteIDs)
+
+	if diffConfig.Out != "-" {
+		return os.WriteFile(diffConfig.Out, []byte(result), 0644)
+	}
+
+	_, err = os.Stdout.WriteString(result)
+
+	return err
+}
+
+func secretsDiffReport(wid string, localIDs []string, remoteIDs []string) string {
+	var (
+		b      strings.Builder
+		remote = map[string]bool{}
+		local  = map[string]bool{}
+
+		onlyLocal, onlyRemote, both []string
+	)
+
+	for _, id := range remoteIDs {
+		remote[id] = true
+	}
+	for _, id := range localIDs {
+		local[id] = true
+
+		if remote[id] {
+			both = append(both, id)
+		} else {
+			onlyLocal = append(onlyLocal, id)
+		}
+	}
+	for _, id := range remoteIDs {
+		if !local[id] {
+			onlyRemote = append(onlyRemote, id)
+		}
+	}
+
+	fmt.Fprintf(&b, "secrets diff for workspace %s\n", wid)
+
+	section := func(title string, ids []string) {
+		fmt.Fprintf(&b, "%s:\n", title)
+		for _, id := range ids {
+			fmt.Fprintf(&b, "  - %s\n", id)
+		}
+	}
+
+	section("only local (would create on push)", onlyLocal)
+	section("only remote (deleted on push --prune)", onlyRemote)
+	section("in both (values not comparable)", both)
+
+	return b.String()
+}
+
 func init() {
 	diffCmd.PersistentFlags().StringVar(&diffConfig.Source, "source", "", `Source of the comparison (required). Format: [profile@]source-type
 Source types: local, remote, merged
@@ -148,6 +248,4 @@ Examples:
 Example: --with-secrets`)
 	diffCmd.PersistentFlags().BoolVar(&diffConfig.FilterVolatile, "no-volatile", false, `Ignore volatile fields (e.g. timestamps, generated IDs) when comparing.
 Example: --no-volatile`)
-
-	mustMarkRequired(diffCmd, "source", "target")
 }

cmd/diff_secrets_test.go

@@ -0,0 +1,20 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSecretsDiffReport(t *testing.T) {
+	out := secretsDiffReport("demo", []string{"a", "c"}, []string{"b", "c"})
+
+	require.Equal(t, `secrets diff for workspace demo
+only local (would create on push):
+  - a
+only remote (deleted on push --prune):
+  - b
+in both (values not comparable):
+  - c
+`, out)
+}

cmd/pull.go

@@ -4,6 +4,8 @@ import (
 	"github.com/cloudentity/acp-client-go/clients/hub/models"
 	"github.com/cloudentity/cac/internal/cac"
 	"github.com/cloudentity/cac/internal/cac/api"
+	"github.com/cloudentity/cac/internal/cac/secrets"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"golang.org/x/exp/slog"
 )
@@ -34,6 +36,10 @@ Examples:
 				err  error
 			)
 
+			if rootConfig.WorkspaceSecrets != "" {
+				return pullSecrets(cmd)
+			}
+
 			if app, err = cac.InitApp(rootConfig.ConfigPath, rootConfig.Profile, rootConfig.Tenant); err != nil {
 				return err
 			}
@@ -67,6 +73,52 @@ Examples:
 	}
 )
 
+func pullSecrets(cmd *cobra.Command) error {
+	var (
+		app *cac.Application
+		err error
+	)
+
+	if len(pullConfig.Filters) > 0 {
+		return errors.New("--filter cannot be combined with --workspace-secrets")
+	}
+
+	if app, err = cac.InitApp(rootConfig.ConfigPath, rootConfig.Profile, false); err != nil {
+		return err
+	}
+
+	dirStore, err := secretsDirStore(app)
+	if err != nil {
+		return err
+	}
+
+	wid := rootConfig.WorkspaceSecrets
+
+	slog.With("workspace", wid).Info("Pulling secrets")
+
+	ids, err := app.Secrets.ListIDs(cmd.Context(), wid)
+	if err != nil {
+		return err
+	}
+
+	created, skipped, err := dirStore.WriteStubs(wid, ids)
+	if err != nil {
+		return err
+	}
+
+	slog.Info("Pulled secrets", "workspace", wid, "created", len(created), "skipped_existing", len(skipped))
+
+	return nil
+}
+
+func secretsDirStore(app *cac.Application) (*secrets.DirStore, error) {
+	if app.Config.Storage == nil || len(app.Config.Storage.DirPath) == 0 {
+		return nil, errors.New("no storage directories configured for the selected profile")
+	}
+
+	return secrets.NewDirStore(app.Config.Storage.DirPath), nil
+}
+
 func init() {
 	pullCmd.PersistentFlags().BoolVar(&pullConfig.WithSecrets, "with-secrets", false, `Include secret fields (client secrets, signing keys, etc.) in the pulled configuration.
 Example: --with-secrets`)