feat: add --no-expand-env-vars flag to deck file convert

cmd/file_convert.go

@@ -20,6 +20,7 @@ var (
 	convertCmdOutputFileDeprecated string
 	convertCmdAssumeYes            bool
 	convertCmdStateFormat          string // yaml/json output
+	convertCmdNoExpandEnvVars      bool
 )
 
 func executeConvert(_ *cobra.Command, _ []string) error {
@@ -35,6 +36,11 @@ func executeConvert(_ *cobra.Command, _ []string) error {
 	}
 
 	if convertCmdInputFile != "" {
+		envVarsMode := file.EnvVarsExpand
+		if convertCmdNoExpandEnvVars {
+			envVarsMode = file.EnvVarsSkip
+		}
+
 		if yes, err := utils.ConfirmFileOverwrite(
 			convertCmdOutputFile, "", convertCmdAssumeYes,
 		); err != nil {
@@ -49,11 +55,16 @@ func executeConvert(_ *cobra.Command, _ []string) error {
 			file.Format(strings.ToUpper(convertCmdStateFormat)),
 			sourceFormat,
 			destinationFormat,
-			false)
+			envVarsMode)
 		if err != nil {
 			return fmt.Errorf("converting file: %w", err)
 		}
 	} else if is2xTo3xConversion() {
+		envVarsMode := file.EnvVarsExpand
+		if convertCmdNoExpandEnvVars {
+			envVarsMode = file.EnvVarsSkip
+		}
+
 		path, err := os.Getwd()
 		if err != nil {
 			return fmt.Errorf("getting current working directory: %w", err)
@@ -69,7 +80,7 @@ func executeConvert(_ *cobra.Command, _ []string) error {
 				file.Format(strings.ToUpper(convertCmdStateFormat)),
 				sourceFormat,
 				destinationFormat,
-				false)
+				envVarsMode)
 			if err != nil {
 				return fmt.Errorf("converting '%s' file: %w", filename, err)
 			}
@@ -168,6 +179,8 @@ can be converted into a 'kong-gateway-3.x' configuration file.`,
 		false, "assume `yes` to prompts and run non-interactively.")
 	convertCmd.Flags().StringVar(&convertCmdStateFormat, "format",
 		"yaml", "output file format: json or yaml.")
+	convertCmd.Flags().BoolVar(&convertCmdNoExpandEnvVars, "no-expand-env-vars",
+		false, `do not expand ${{ env "DECK_VAR_NAME" }} placeholders in the output.`)
 
 	return convertCmd
 }

cmd/file_render.go

@@ -17,13 +17,18 @@ var (
 
 func executeFileRenderCmd(_ *cobra.Command, _ []string) error {
 	_ = sendAnalytics("file-render", "", modeLocal)
+	envVarsMode := file.EnvVarsMock
+	if fileRenderCmdPopulateEnvVars {
+		envVarsMode = file.EnvVarsExpand
+	}
+
 	return convert.Convert(
 		fileRenderCmdKongStateFile,
 		fileRenderCmdKongFileOutput,
 		file.Format(strings.ToUpper(fileRenderCmdStateFormat)),
 		convert.FormatDistributed,
 		convert.FormatKongGateway3x,
-		!fileRenderCmdPopulateEnvVars)
+		envVarsMode)
 }
 
 func newFileRenderCmd() *cobra.Command {

convert/convert.go

@@ -82,11 +82,11 @@ func Convert(
 	outputFormat file.Format,
 	from Format,
 	to Format,
-	mockEnvVars bool,
+	envVarsMode file.RenderEnvVarsMode,
 ) error {
 	var outputContent *file.Content
 
-	inputContent, err := file.GetContentFromFiles(inputFilenames, mockEnvVars)
+	inputContent, err := file.GetContentFromFilesWithEnvVars(inputFilenames, envVarsMode)
 	if err != nil {
 		return err
 	}

convert/convert_test.go

@@ -365,8 +365,12 @@ func Test_Convert(t *testing.T) {
 			for k, v := range tt.args.envVars {
 				t.Setenv(k, v)
 			}
+			envVarsMode := file.EnvVarsMock
+			if tt.args.disableMocks {
+				envVarsMode = file.EnvVarsExpand
+			}
 			err := Convert(inputFiles, tt.args.outputFilename, file.YAML, tt.args.fromFormat,
-				tt.args.toFormat, !tt.args.disableMocks)
+				tt.args.toFormat, envVarsMode)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Convert() error = %v, wantErr %v", err, tt.wantErr)
 			}

convert/rulesets/310-to-314/entrypoint.yaml

@@ -59,11 +59,11 @@ rules:
       In Kong Gateway 3.14, the global tls_certificate_verify changed from off to on,
       which affects the ssl_verify behavior for many plugins.
     given:
-      - $.plugins[?(@.name == 'acme' || @.name == 'aws-lambda' || @.name == 'azure-functions' || @.name == 'http-log' || @.name == 'openid-connect' || @.name == 'forward-proxy' || @.name == 'ldap-auth' || @.name == 'ldap-auth-advanced' || @.name == 'tcp-log' || @.name == 'upstream-oauth')].config
-      - $.services[*].plugins[?(@.name == 'acme' || @.name == 'aws-lambda' || @.name == 'azure-functions' || @.name == 'http-log' || @.name == 'openid-connect' || @.name == 'forward-proxy' || @.name == 'ldap-auth' || @.name == 'ldap-auth-advanced' || @.name == 'tcp-log' || @.name == 'upstream-oauth')].config
-      - $.routes[*].plugins[?(@.name == 'acme' || @.name == 'aws-lambda' || @.name == 'azure-functions' || @.name == 'http-log' || @.name == 'openid-connect' || @.name == 'forward-proxy' || @.name == 'ldap-auth' || @.name == 'ldap-auth-advanced' || @.name == 'tcp-log' || @.name == 'upstream-oauth')].config
-      - $.services[*].routes[*].plugins[?(@.name == 'acme' || @.name == 'aws-lambda' || @.name == 'azure-functions' || @.name == 'http-log' || @.name == 'openid-connect' || @.name == 'forward-proxy' || @.name == 'ldap-auth' || @.name == 'ldap-auth-advanced' || @.name == 'tcp-log' || @.name == 'upstream-oauth')].config
-      - $.consumers[*].plugins[?(@.name == 'acme' || @.name == 'aws-lambda' || @.name == 'azure-functions' || @.name == 'http-log' || @.name == 'openid-connect' || @.name == 'forward-proxy' || @.name == 'ldap-auth' || @.name == 'ldap-auth-advanced' || @.name == 'tcp-log' || @.name == 'upstream-oauth')].config
+      - $.plugins[?(@.name == 'ace' || @.name == 'acme' || @.name == 'ai-aws-guardrail' || @.name == 'ai-azure-content-safety' || @.name == 'ai-llm-as-judge' || @.name == 'ai-proxy-advanced' || @.name == 'ai-rag-injector' || @.name == 'ai-rate-limiting-advanced' || @.name == 'ai-request-transformer' || @.name == 'ai-response-transformer' || @.name == 'ai-semantic-cache' || @.name == 'ai-semantic-prompt-guard' || @.name == 'ai-semantic-response-guard' || @.name == 'aws-lambda' || @.name == 'azure-functions' || @.name == 'basic-auth' || @.name == 'confluent' || @.name == 'confluent-consume' || @.name == 'datakit' || @.name == 'forward-proxy' || @.name == 'graphql-proxy-cache-advanced' || @.name == 'graphql-rate-limiting-advanced' || @.name == 'header-cert-auth' || @.name == 'http-log' || @.name == 'jwt-signer' || @.name == 'kafka-consume' || @.name == 'kafka-log' || @.name == 'kafka-upstream' || @.name == 'ldap-auth' || @.name == 'ldap-auth-advanced' || @.name == 'mtls-auth' || @.name == 'opa' || @.name == 'openid-connect' || @.name == 'proxy-cache-advanced' || @.name == 'rate-limiting' || @.name == 'rate-limiting-advanced' || @.name == 'request-callout' || @.name == 'response-ratelimiting' || @.name == 'saml' || @.name == 'service-protection' || @.name == 'tcp-log' || @.name == 'upstream-oauth')].config
+      - $.services[*].plugins[?(@.name == 'ace' || @.name == 'acme' || @.name == 'ai-aws-guardrail' || @.name == 'ai-azure-content-safety' || @.name == 'ai-llm-as-judge' || @.name == 'ai-proxy-advanced' || @.name == 'ai-rag-injector' || @.name == 'ai-rate-limiting-advanced' || @.name == 'ai-request-transformer' || @.name == 'ai-response-transformer' || @.name == 'ai-semantic-cache' || @.name == 'ai-semantic-prompt-guard' || @.name == 'ai-semantic-response-guard' || @.name == 'aws-lambda' || @.name == 'azure-functions' || @.name == 'basic-auth' || @.name == 'confluent' || @.name == 'confluent-consume' || @.name == 'datakit' || @.name == 'forward-proxy' || @.name == 'graphql-proxy-cache-advanced' || @.name == 'graphql-rate-limiting-advanced' || @.name == 'header-cert-auth' || @.name == 'http-log' || @.name == 'jwt-signer' || @.name == 'kafka-consume' || @.name == 'kafka-log' || @.name == 'kafka-upstream' || @.name == 'ldap-auth' || @.name == 'ldap-auth-advanced' || @.name == 'mtls-auth' || @.name == 'opa' || @.name == 'openid-connect' || @.name == 'proxy-cache-advanced' || @.name == 'rate-limiting' || @.name == 'rate-limiting-advanced' || @.name == 'request-callout' || @.name == 'response-ratelimiting' || @.name == 'saml' || @.name == 'service-protection' || @.name == 'tcp-log' || @.name == 'upstream-oauth')].config
+      - $.routes[*].plugins[?(@.name == 'ace' || @.name == 'acme' || @.name == 'ai-aws-guardrail' || @.name == 'ai-azure-content-safety' || @.name == 'ai-llm-as-judge' || @.name == 'ai-proxy-advanced' || @.name == 'ai-rag-injector' || @.name == 'ai-rate-limiting-advanced' || @.name == 'ai-request-transformer' || @.name == 'ai-response-transformer' || @.name == 'ai-semantic-cache' || @.name == 'ai-semantic-prompt-guard' || @.name == 'ai-semantic-response-guard' || @.name == 'aws-lambda' || @.name == 'azure-functions' || @.name == 'basic-auth' || @.name == 'confluent' || @.name == 'confluent-consume' || @.name == 'datakit' || @.name == 'forward-proxy' || @.name == 'graphql-proxy-cache-advanced' || @.name == 'graphql-rate-limiting-advanced' || @.name == 'header-cert-auth' || @.name == 'http-log' || @.name == 'jwt-signer' || @.name == 'kafka-consume' || @.name == 'kafka-log' || @.name == 'kafka-upstream' || @.name == 'ldap-auth' || @.name == 'ldap-auth-advanced' || @.name == 'mtls-auth' || @.name == 'opa' || @.name == 'openid-connect' || @.name == 'proxy-cache-advanced' || @.name == 'rate-limiting' || @.name == 'rate-limiting-advanced' || @.name == 'request-callout' || @.name == 'response-ratelimiting' || @.name == 'saml' || @.name == 'service-protection' || @.name == 'tcp-log' || @.name == 'upstream-oauth')].config
+      - $.services[*].routes[*].plugins[?(@.name == 'ace' || @.name == 'acme' || @.name == 'ai-aws-guardrail' || @.name == 'ai-azure-content-safety' || @.name == 'ai-llm-as-judge' || @.name == 'ai-proxy-advanced' || @.name == 'ai-rag-injector' || @.name == 'ai-rate-limiting-advanced' || @.name == 'ai-request-transformer' || @.name == 'ai-response-transformer' || @.name == 'ai-semantic-cache' || @.name == 'ai-semantic-prompt-guard' || @.name == 'ai-semantic-response-guard' || @.name == 'aws-lambda' || @.name == 'azure-functions' || @.name == 'basic-auth' || @.name == 'confluent' || @.name == 'confluent-consume' || @.name == 'datakit' || @.name == 'forward-proxy' || @.name == 'graphql-proxy-cache-advanced' || @.name == 'graphql-rate-limiting-advanced' || @.name == 'header-cert-auth' || @.name == 'http-log' || @.name == 'jwt-signer' || @.name == 'kafka-consume' || @.name == 'kafka-log' || @.name == 'kafka-upstream' || @.name == 'ldap-auth' || @.name == 'ldap-auth-advanced' || @.name == 'mtls-auth' || @.name == 'opa' || @.name == 'openid-connect' || @.name == 'proxy-cache-advanced' || @.name == 'rate-limiting' || @.name == 'rate-limiting-advanced' || @.name == 'request-callout' || @.name == 'response-ratelimiting' || @.name == 'saml' || @.name == 'service-protection' || @.name == 'tcp-log' || @.name == 'upstream-oauth')].config
+      - $.consumers[*].plugins[?(@.name == 'ace' || @.name == 'acme' || @.name == 'ai-aws-guardrail' || @.name == 'ai-azure-content-safety' || @.name == 'ai-llm-as-judge' || @.name == 'ai-proxy-advanced' || @.name == 'ai-rag-injector' || @.name == 'ai-rate-limiting-advanced' || @.name == 'ai-request-transformer' || @.name == 'ai-response-transformer' || @.name == 'ai-semantic-cache' || @.name == 'ai-semantic-prompt-guard' || @.name == 'ai-semantic-response-guard' || @.name == 'aws-lambda' || @.name == 'azure-functions' || @.name == 'basic-auth' || @.name == 'confluent' || @.name == 'confluent-consume' || @.name == 'datakit' || @.name == 'forward-proxy' || @.name == 'graphql-proxy-cache-advanced' || @.name == 'graphql-rate-limiting-advanced' || @.name == 'header-cert-auth' || @.name == 'http-log' || @.name == 'jwt-signer' || @.name == 'kafka-consume' || @.name == 'kafka-log' || @.name == 'kafka-upstream' || @.name == 'ldap-auth' || @.name == 'ldap-auth-advanced' || @.name == 'mtls-auth' || @.name == 'opa' || @.name == 'openid-connect' || @.name == 'proxy-cache-advanced' || @.name == 'rate-limiting' || @.name == 'rate-limiting-advanced' || @.name == 'request-callout' || @.name == 'response-ratelimiting' || @.name == 'saml' || @.name == 'service-protection' || @.name == 'tcp-log' || @.name == 'upstream-oauth')].config
     message: >-
       Kong Gateway 3.14 enables TLS certificate verification by default. Plugins that connect
       to external services over TLS without explicit ssl_verify settings will now verify

go.mod

@@ -14,8 +14,8 @@ require (
 	github.com/fatih/color v1.18.0
 	github.com/google/go-cmp v0.7.0
 	github.com/kong/go-apiops v0.4.0
-	github.com/kong/go-database-reconciler v1.33.0
-	github.com/kong/go-kong v0.73.0
+	github.com/kong/go-database-reconciler v1.35.0
+	github.com/kong/go-kong v0.74.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/pflag v1.0.10

go.sum

@@ -209,8 +209,12 @@ github.com/kong/go-apiops v0.4.0 h1:rXxdcE6VkDTOkhGt/ZytYqp8elw3SfrBQ/+r42lX2J4=
 github.com/kong/go-apiops v0.4.0/go.mod h1:Xt99d90LallLVwYJAGaufiNbBdsK0KKboe7gR4Ryths=
 github.com/kong/go-database-reconciler v1.33.0 h1:EAwap2hSCwHUJfVxbF+uw6UruqC47hNMGRbXhinY9IE=
 github.com/kong/go-database-reconciler v1.33.0/go.mod h1:CTFu9hfOh2L4qPniCUCqLYhKiuVA0UsQe0Iv4jZc58o=
+github.com/kong/go-database-reconciler v1.35.0 h1:36Lmik7eoNMl+B/g6beWqBVgGl8b2vLXwNYNUVOtdRM=
+github.com/kong/go-database-reconciler v1.35.0/go.mod h1:AeQNEbEQ/Wy3OwJiWIo3PwWCWfav25F/FsTESqbelac=
 github.com/kong/go-kong v0.73.0 h1:Ct4EHJsJqoaYod39+gGy/JnU1664jzDbxTS2dk6Y628=
 github.com/kong/go-kong v0.73.0/go.mod h1:Wx5aTcMjyUnIF94M5NYFWb/EnuEkqB5STrWvybFSYYQ=
+github.com/kong/go-kong v0.74.0 h1:iNs3tHeCRWL2lbJ4io0Rpm23u4GZchFY2wJ/BUxNs4w=
+github.com/kong/go-kong v0.74.0/go.mod h1:Wx5aTcMjyUnIF94M5NYFWb/EnuEkqB5STrWvybFSYYQ=
 github.com/kong/go-slugify v1.0.0 h1:vCFAyf2sdoSlBtLcrmDWUFn0ohlpKiKvQfXZkO5vSKY=
 github.com/kong/go-slugify v1.0.0/go.mod h1:dbR2h3J2QKXQ1k0aww6cN7o4cIcwlWflr6RKRdcoaiw=
 github.com/kong/kubernetes-configuration v1.5.3 h1:4Hh2ZAT5zjHHqx07VVMF5lIUZ7c2RAOUpIINpOsduZs=

tests/integration/file_convert_test.go

@@ -74,6 +74,55 @@ func Test_FileConvert(t *testing.T) {
 	}
 }
 
+func Test_FileConvert_NoExpandEnvVars(t *testing.T) {
+	tests := []struct {
+		name               string
+		additionalArgs     []string
+		envVars            map[string]string
+		expectedOutputFile string
+	}{
+		{
+			name:               "env vars are expanded by default",
+			additionalArgs:     []string{},
+			envVars:            map[string]string{"DECK_SVC1_HOST": "mockbin.org"},
+			expectedOutputFile: "testdata/file-convert/003-no-expand-env-vars/expected-expanded.yaml",
+		},
+		{
+			name:               "env vars are not expanded with --no-expand-env-vars",
+			additionalArgs:     []string{"--no-expand-env-vars"},
+			envVars:            map[string]string{"DECK_SVC1_HOST": "mockbin.org"},
+			expectedOutputFile: "testdata/file-convert/003-no-expand-env-vars/expected-no-expand.yaml",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			for k, v := range tc.envVars {
+				t.Setenv(k, v)
+			}
+
+			args := append([]string{
+				"--from", "kong-gateway-2.x",
+				"--to", "kong-gateway-3.x",
+				"--input-file", "testdata/file-convert/003-no-expand-env-vars/input.yaml",
+			}, tc.additionalArgs...)
+			output, err := fileConvert(args...)
+
+			require.NoError(t, err)
+
+			var expectedOutput interface{}
+			var currentOutput interface{}
+			content, err := os.ReadFile(tc.expectedOutputFile)
+			require.NoError(t, err)
+
+			err = yaml.Unmarshal(content, &expectedOutput)
+			require.NoError(t, err)
+			err = yaml.Unmarshal([]byte(output), &currentOutput)
+			require.NoError(t, err)
+			assert.Equal(t, expectedOutput, currentOutput)
+		})
+	}
+}
+
 func Test_FileConvert_28xTo34x(t *testing.T) {
 	originalCwd, err := os.Getwd()
 	require.NoError(t, err)

tests/integration/testdata/file-convert/003-no-expand-env-vars/expected-expanded.yaml

@@ -0,0 +1,9 @@
+_format_version: "3.0"
+services:
+- connect_timeout: 60000
+  host: mockbin.org
+  name: svc1
+  port: 80
+  protocol: http
+  read_timeout: 60000
+  retries: 5

tests/integration/testdata/file-convert/003-no-expand-env-vars/expected-no-expand.yaml

@@ -0,0 +1,9 @@
+_format_version: "3.0"
+services:
+- connect_timeout: 60000
+  host: '${{ env "DECK_SVC1_HOST" }}'
+  name: svc1
+  port: 80
+  protocol: http
+  read_timeout: 60000
+  retries: 5

tests/integration/testdata/file-convert/003-no-expand-env-vars/input.yaml

@@ -0,0 +1,9 @@
+_format_version: "1.1"
+services:
+- connect_timeout: 60000
+  host: ${{ env "DECK_SVC1_HOST" }}
+  name: svc1
+  port: 80
+  protocol: http
+  read_timeout: 60000
+  retries: 5

utils/utils.go

@@ -1,3 +1,4 @@
+//revive:disable:var-naming // package name is established across the codebase.
 package utils
 
 const (