From ce622af01c0828ce79ece691d574ceedd13043e9 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Tue, 28 Jan 2025 13:30:39 +0800 Subject: [PATCH 1/3] feat: update vault secrets from config --- internal/db/start/start.go | 15 +++++++++++++++ pkg/config/db.go | 3 +++ pkg/config/templates/config.toml | 3 +++ pkg/config/testdata/config.toml | 3 +++ 4 files changed, 24 insertions(+) diff --git a/internal/db/start/start.go b/internal/db/start/start.go index 4180fe05c9..4d178f384c 100644 --- a/internal/db/start/start.go +++ b/internal/db/start/start.go @@ -368,10 +368,25 @@ func SetupLocalDatabase(ctx context.Context, version string, fsys afero.Fs, w io return apply.MigrateAndSeed(ctx, version, conn, fsys) } +const CREATE_VAULT_SECRET = "SELECT vault.create_secret($1, $2)" + func SetupDatabase(ctx context.Context, conn *pgx.Conn, host string, w io.Writer, fsys afero.Fs) error { if err := initSchema(ctx, conn, host, w); err != nil { return err } + // Create vault secrets first so roles.sql can reference them + batch := pgx.Batch{} + for k, v := range utils.Config.Db.Vault { + if len(v.SHA256) > 0 { + batch.Queue(CREATE_VAULT_SECRET, v.Value, k) + } + } + if batch.Len() > 0 { + fmt.Fprintln(os.Stderr, "Updating vault secrets...") + if err := conn.SendBatch(ctx, &batch).Close(); err != nil { + return errors.Errorf("failed to update migration history: %w", err) + } + } err := migration.SeedGlobals(ctx, []string{utils.CustomRolesPath}, conn, afero.NewIOFS(fsys)) if errors.Is(err, os.ErrNotExist) { return nil diff --git a/pkg/config/db.go b/pkg/config/db.go index d18cf8ebd6..3688ed50bc 100644 --- a/pkg/config/db.go +++ b/pkg/config/db.go @@ -60,8 +60,11 @@ type ( Pooler pooler `toml:"pooler"` Seed seed `toml:"seed"` Settings settings `toml:"settings"` + Vault vault `toml:"vault"` } + vault map[string]Secret + seed struct { Enabled bool `toml:"enabled"` GlobPatterns []string `toml:"sql_paths"` diff --git a/pkg/config/templates/config.toml b/pkg/config/templates/config.toml index c8528d8f52..c91ec4b31f 100644 --- a/pkg/config/templates/config.toml +++ b/pkg/config/templates/config.toml @@ -42,6 +42,9 @@ default_pool_size = 20 # Maximum number of client connections allowed. max_client_conn = 100 +# [db.vault] +# secret_key = "env(SECRET_VALUE)" + [db.seed] # If enabled, seeds the database after migrations during a db reset. enabled = true diff --git a/pkg/config/testdata/config.toml b/pkg/config/testdata/config.toml index 65705646c3..8fcffa41d8 100644 --- a/pkg/config/testdata/config.toml +++ b/pkg/config/testdata/config.toml @@ -42,6 +42,9 @@ default_pool_size = 20 # Maximum number of client connections allowed. max_client_conn = 100 +[db.vault] +test_key = "test_value" + [db.seed] # If enabled, seeds the database after migrations during a db reset. enabled = true From e8772e1d2063465bfb75fe0ea7103ad33ca8782d Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Tue, 28 Jan 2025 17:11:48 +0800 Subject: [PATCH 2/3] chore: create or update vault secrets --- internal/db/start/start.go | 64 ++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/internal/db/start/start.go b/internal/db/start/start.go index 4d178f384c..d6bc779ef2 100644 --- a/internal/db/start/start.go +++ b/internal/db/start/start.go @@ -25,6 +25,7 @@ import ( "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/internal/utils/flags" "github.com/supabase/cli/pkg/migration" + "github.com/supabase/cli/pkg/pgxv5" ) var ( @@ -368,24 +369,13 @@ func SetupLocalDatabase(ctx context.Context, version string, fsys afero.Fs, w io return apply.MigrateAndSeed(ctx, version, conn, fsys) } -const CREATE_VAULT_SECRET = "SELECT vault.create_secret($1, $2)" - func SetupDatabase(ctx context.Context, conn *pgx.Conn, host string, w io.Writer, fsys afero.Fs) error { if err := initSchema(ctx, conn, host, w); err != nil { return err } // Create vault secrets first so roles.sql can reference them - batch := pgx.Batch{} - for k, v := range utils.Config.Db.Vault { - if len(v.SHA256) > 0 { - batch.Queue(CREATE_VAULT_SECRET, v.Value, k) - } - } - if batch.Len() > 0 { - fmt.Fprintln(os.Stderr, "Updating vault secrets...") - if err := conn.SendBatch(ctx, &batch).Close(); err != nil { - return errors.Errorf("failed to update migration history: %w", err) - } + if err := upsertVaultSecrets(ctx, conn); err != nil { + return err } err := migration.SeedGlobals(ctx, []string{utils.CustomRolesPath}, conn, afero.NewIOFS(fsys)) if errors.Is(err, os.ErrNotExist) { @@ -393,3 +383,51 @@ func SetupDatabase(ctx context.Context, conn *pgx.Conn, host string, w io.Writer } return err } + +const ( + CREATE_VAULT_KV = "SELECT vault.create_secret($1, $2)" + READ_VAULT_KV = "SELECT id, name FROM vault.secrets WHERE name = ANY($1)" + UPDATE_VAULT_KV = "SELECT vault.update_secret($1, $2)" +) + +type VaultTable struct { + Id string + Name string +} + +func upsertVaultSecrets(ctx context.Context, conn *pgx.Conn) error { + var keys []string + toInsert := map[string]string{} + for k, v := range utils.Config.Db.Vault { + if len(v.SHA256) > 0 { + keys = append(keys, k) + toInsert[k] = v.Value + } + } + if len(keys) == 0 { + return nil + } + fmt.Fprintln(os.Stderr, "Updating vault secrets...") + rows, err := conn.Query(ctx, READ_VAULT_KV, keys) + if err != nil { + return errors.Errorf("failed to query rows: %w", err) + } + toUpdate, err := pgxv5.CollectRows[VaultTable](rows) + if err != nil { + return err + } + batch := pgx.Batch{} + for _, r := range toUpdate { + secret := utils.Config.Db.Vault[r.Name] + batch.Queue(UPDATE_VAULT_KV, r.Id, secret.Value) + delete(toInsert, r.Name) + } + // Remaining secrets should be created + for k, v := range toInsert { + batch.Queue(CREATE_VAULT_KV, v, k) + } + if err := conn.SendBatch(ctx, &batch).Close(); err != nil { + return errors.Errorf("failed to update vault secrets: %w", err) + } + return nil +} From 80f1be8a29848fc23509af26ee9ea88d9a8ee960 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Fri, 31 Jan 2025 14:32:09 +0800 Subject: [PATCH 3/3] chore: move vault to pkg --- internal/db/push/push.go | 4 +++ internal/db/reset/reset.go | 18 ++--------- internal/db/start/start.go | 52 ++------------------------------ internal/migration/up/up.go | 4 +++ pkg/config/db.go | 22 +++++++------- pkg/vault/batch.go | 60 +++++++++++++++++++++++++++++++++++++ 6 files changed, 83 insertions(+), 77 deletions(-) create mode 100644 pkg/vault/batch.go diff --git a/internal/db/push/push.go b/internal/db/push/push.go index 7e5241558c..4a1fdf6df5 100644 --- a/internal/db/push/push.go +++ b/internal/db/push/push.go @@ -14,6 +14,7 @@ import ( "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/internal/utils/flags" "github.com/supabase/cli/pkg/migration" + "github.com/supabase/cli/pkg/vault" ) func Run(ctx context.Context, dryRun, ignoreVersionMismatch bool, includeRoles, includeSeed bool, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -82,6 +83,9 @@ func Run(ctx context.Context, dryRun, ignoreVersionMismatch bool, includeRoles, } else if !shouldPush { return errors.New(context.Canceled) } + if err := vault.UpsertVaultSecrets(ctx, utils.Config.Db.Vault, conn); err != nil { + return err + } if err := migration.ApplyMigrations(ctx, pending, conn, afero.NewIOFS(fsys)); err != nil { return err } diff --git a/internal/db/reset/reset.go b/internal/db/reset/reset.go index 307f14f55c..d3e33e5829 100644 --- a/internal/db/reset/reset.go +++ b/internal/db/reset/reset.go @@ -23,11 +23,11 @@ import ( "github.com/supabase/cli/internal/db/start" "github.com/supabase/cli/internal/gen/keys" "github.com/supabase/cli/internal/migration/apply" - "github.com/supabase/cli/internal/migration/list" "github.com/supabase/cli/internal/migration/repair" "github.com/supabase/cli/internal/seed/buckets" "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/pkg/migration" + "github.com/supabase/cli/pkg/vault" ) func Run(ctx context.Context, version string, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -242,22 +242,10 @@ func resetRemote(ctx context.Context, version string, config pgconn.Config, fsys if err := migration.DropUserSchemas(ctx, conn); err != nil { return err } - migrations, err := list.LoadPartialMigrations(version, fsys) - if err != nil { - return err - } - if err := migration.ApplyMigrations(ctx, migrations, conn, afero.NewIOFS(fsys)); err != nil { + if err := vault.UpsertVaultSecrets(ctx, utils.Config.Db.Vault, conn); err != nil { return err } - if !utils.Config.Db.Seed.Enabled { - // Skip because --no-seed flag is set - return nil - } - seeds, err := migration.GetPendingSeeds(ctx, utils.Config.Db.Seed.SqlPaths, conn, afero.NewIOFS(fsys)) - if err != nil { - return err - } - return migration.SeedData(ctx, seeds, conn, afero.NewIOFS(fsys)) + return apply.MigrateAndSeed(ctx, version, conn, fsys) } func LikeEscapeSchema(schemas []string) (result []string) { diff --git a/internal/db/start/start.go b/internal/db/start/start.go index d6bc779ef2..85b5c6e35e 100644 --- a/internal/db/start/start.go +++ b/internal/db/start/start.go @@ -25,7 +25,7 @@ import ( "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/internal/utils/flags" "github.com/supabase/cli/pkg/migration" - "github.com/supabase/cli/pkg/pgxv5" + "github.com/supabase/cli/pkg/vault" ) var ( @@ -374,7 +374,7 @@ func SetupDatabase(ctx context.Context, conn *pgx.Conn, host string, w io.Writer return err } // Create vault secrets first so roles.sql can reference them - if err := upsertVaultSecrets(ctx, conn); err != nil { + if err := vault.UpsertVaultSecrets(ctx, utils.Config.Db.Vault, conn); err != nil { return err } err := migration.SeedGlobals(ctx, []string{utils.CustomRolesPath}, conn, afero.NewIOFS(fsys)) @@ -383,51 +383,3 @@ func SetupDatabase(ctx context.Context, conn *pgx.Conn, host string, w io.Writer } return err } - -const ( - CREATE_VAULT_KV = "SELECT vault.create_secret($1, $2)" - READ_VAULT_KV = "SELECT id, name FROM vault.secrets WHERE name = ANY($1)" - UPDATE_VAULT_KV = "SELECT vault.update_secret($1, $2)" -) - -type VaultTable struct { - Id string - Name string -} - -func upsertVaultSecrets(ctx context.Context, conn *pgx.Conn) error { - var keys []string - toInsert := map[string]string{} - for k, v := range utils.Config.Db.Vault { - if len(v.SHA256) > 0 { - keys = append(keys, k) - toInsert[k] = v.Value - } - } - if len(keys) == 0 { - return nil - } - fmt.Fprintln(os.Stderr, "Updating vault secrets...") - rows, err := conn.Query(ctx, READ_VAULT_KV, keys) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - toUpdate, err := pgxv5.CollectRows[VaultTable](rows) - if err != nil { - return err - } - batch := pgx.Batch{} - for _, r := range toUpdate { - secret := utils.Config.Db.Vault[r.Name] - batch.Queue(UPDATE_VAULT_KV, r.Id, secret.Value) - delete(toInsert, r.Name) - } - // Remaining secrets should be created - for k, v := range toInsert { - batch.Queue(CREATE_VAULT_KV, v, k) - } - if err := conn.SendBatch(ctx, &batch).Close(); err != nil { - return errors.Errorf("failed to update vault secrets: %w", err) - } - return nil -} diff --git a/internal/migration/up/up.go b/internal/migration/up/up.go index d33117f33b..331abce66e 100644 --- a/internal/migration/up/up.go +++ b/internal/migration/up/up.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/afero" "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/pkg/migration" + "github.com/supabase/cli/pkg/vault" ) func Run(ctx context.Context, includeAll bool, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -23,6 +24,9 @@ func Run(ctx context.Context, includeAll bool, config pgconn.Config, fsys afero. if err != nil { return err } + if err := vault.UpsertVaultSecrets(ctx, utils.Config.Db.Vault, conn); err != nil { + return err + } return migration.ApplyMigrations(ctx, pending, conn, afero.NewIOFS(fsys)) } diff --git a/pkg/config/db.go b/pkg/config/db.go index 3688ed50bc..eaa02e0368 100644 --- a/pkg/config/db.go +++ b/pkg/config/db.go @@ -51,20 +51,18 @@ type ( } db struct { - Image string `toml:"-"` - Port uint16 `toml:"port"` - ShadowPort uint16 `toml:"shadow_port"` - MajorVersion uint `toml:"major_version"` - Password string `toml:"-"` - RootKey string `toml:"-" mapstructure:"root_key"` - Pooler pooler `toml:"pooler"` - Seed seed `toml:"seed"` - Settings settings `toml:"settings"` - Vault vault `toml:"vault"` + Image string `toml:"-"` + Port uint16 `toml:"port"` + ShadowPort uint16 `toml:"shadow_port"` + MajorVersion uint `toml:"major_version"` + Password string `toml:"-"` + RootKey string `toml:"-" mapstructure:"root_key"` + Pooler pooler `toml:"pooler"` + Seed seed `toml:"seed"` + Settings settings `toml:"settings"` + Vault map[string]Secret `toml:"vault"` } - vault map[string]Secret - seed struct { Enabled bool `toml:"enabled"` GlobPatterns []string `toml:"sql_paths"` diff --git a/pkg/vault/batch.go b/pkg/vault/batch.go new file mode 100644 index 0000000000..8f44bb797e --- /dev/null +++ b/pkg/vault/batch.go @@ -0,0 +1,60 @@ +package vault + +import ( + "context" + "fmt" + "os" + + "github.com/go-errors/errors" + "github.com/jackc/pgx/v4" + "github.com/supabase/cli/pkg/config" + "github.com/supabase/cli/pkg/pgxv5" +) + +const ( + CREATE_VAULT_KV = "SELECT vault.create_secret($1, $2)" + READ_VAULT_KV = "SELECT id, name FROM vault.secrets WHERE name = ANY($1)" + UPDATE_VAULT_KV = "SELECT vault.update_secret($1, $2)" +) + +type VaultTable struct { + Id string + Name string +} + +func UpsertVaultSecrets(ctx context.Context, secrets map[string]config.Secret, conn *pgx.Conn) error { + var keys []string + toInsert := map[string]string{} + for k, v := range secrets { + if len(v.SHA256) > 0 { + keys = append(keys, k) + toInsert[k] = v.Value + } + } + if len(keys) == 0 { + return nil + } + fmt.Fprintln(os.Stderr, "Updating vault secrets...") + rows, err := conn.Query(ctx, READ_VAULT_KV, keys) + if err != nil { + return errors.Errorf("failed to read vault: %w", err) + } + toUpdate, err := pgxv5.CollectRows[VaultTable](rows) + if err != nil { + return err + } + batch := pgx.Batch{} + for _, r := range toUpdate { + secret := secrets[r.Name] + batch.Queue(UPDATE_VAULT_KV, r.Id, secret.Value) + delete(toInsert, r.Name) + } + // Remaining secrets should be created + for k, v := range toInsert { + batch.Queue(CREATE_VAULT_KV, v, k) + } + if err := conn.SendBatch(ctx, &batch).Close(); err != nil { + return errors.Errorf("failed to update vault: %w", err) + } + return nil +}