databricks · denik · Jul 9, 2025 · Jul 9, 2025 · Jul 9, 2025 · Jul 9, 2025
diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md
@@ -14,5 +14,6 @@
 
 ### Bundles
 * Fix default search location for whl artifacts ([#3184](https://github.com/databricks/cli/pull/3184)). This was a regression introduced in 0.255.0.
+* The job tasks are now sorted by task key in "bundle validate/summary" output ([#3212](https://github.com/databricks/cli/pull/3212))
 
 ### API Changes
diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_dynamic/out.test.toml
@@ -2,4 +2,4 @@ Local = true
 Cloud = false
 
 [EnvMatrix]
-  DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
diff --git a/acceptance/bundle/artifacts/whl_dynamic/test.toml b/acceptance/bundle/artifacts/whl_dynamic/test.toml
@@ -1,5 +1,3 @@
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # need to sort tasks by key like terraform does
-
 [[Repls]]
 Old = '\\\\'
 New = '/'

diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml
@@ -2,4 +2,4 @@ Local = true
 Cloud = false
 
 [EnvMatrix]
-  DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/test.toml
@@ -1,2 +1 @@
 BundleConfig.default_name = ""
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # need to sort tasks by key
diff --git a/acceptance/bundle/paths/fallback/output.job.json b/acceptance/bundle/paths/fallback/output.job.json
@@ -1,18 +1,4 @@
 [
-  {
-    "job_cluster_key": "default",
-    "notebook_task": {
-      "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/notebook"
-    },
-    "task_key": "notebook_example"
-  },
-  {
-    "job_cluster_key": "default",
-    "spark_python_task": {
-      "python_file": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/file.py"
-    },
-    "task_key": "spark_python_example"
-  },
   {
     "dbt_task": {
       "commands": [
@@ -26,13 +12,10 @@
   },
   {
     "job_cluster_key": "default",
-    "sql_task": {
-      "file": {
-        "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/sql.sql"
-      },
-      "warehouse_id": "cafef00d"
+    "notebook_task": {
+      "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/notebook"
     },
-    "task_key": "sql_example"
+    "task_key": "notebook_example"
   },
   {
     "job_cluster_key": "default",
@@ -63,5 +46,22 @@
       "main_class_name": "com.example.Main"
     },
     "task_key": "spark_jar_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "spark_python_task": {
+      "python_file": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/file.py"
+    },
+    "task_key": "spark_python_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "sql_task": {
+      "file": {
+        "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/sql.sql"
+      },
+      "warehouse_id": "cafef00d"
+    },
+    "task_key": "sql_example"
   }
 ]
diff --git a/acceptance/bundle/paths/nominal/output.job.json b/acceptance/bundle/paths/nominal/output.job.json
@@ -1,18 +1,4 @@
 [
-  {
-    "job_cluster_key": "default",
-    "notebook_task": {
-      "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook"
-    },
-    "task_key": "notebook_example"
-  },
-  {
-    "job_cluster_key": "default",
-    "spark_python_task": {
-      "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py"
-    },
-    "task_key": "spark_python_example"
-  },
   {
     "dbt_task": {
       "commands": [
@@ -25,14 +11,33 @@
     "task_key": "dbt_example"
   },
   {
+    "for_each_task": {
+      "task": {
+        "notebook_task": {
+          "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook"
+        }
+      }
+    },
     "job_cluster_key": "default",
-    "sql_task": {
-      "file": {
-        "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/sql.sql"
-      },
-      "warehouse_id": "cafef00d"
+    "task_key": "for_each_notebook_example"
+  },
+  {
+    "for_each_task": {
+      "task": {
+        "job_cluster_key": "default",
+        "spark_python_task": {
+          "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py"
+        }
+      }
     },
-    "task_key": "sql_example"
+    "task_key": "for_each_spark_python_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "notebook_task": {
+      "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook"
+    },
+    "task_key": "notebook_example"
   },
   {
     "job_cluster_key": "default",
@@ -65,25 +70,20 @@
     "task_key": "spark_jar_example"
   },
   {
-    "for_each_task": {
-      "task": {
-        "notebook_task": {
-          "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook"
-        }
-      }
-    },
     "job_cluster_key": "default",
-    "task_key": "for_each_notebook_example"
+    "spark_python_task": {
+      "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py"
+    },
+    "task_key": "spark_python_example"
   },
   {
-    "for_each_task": {
-      "task": {
-        "job_cluster_key": "default",
-        "spark_python_task": {
-          "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py"
-        }
-      }
+    "job_cluster_key": "default",
+    "sql_task": {
+      "file": {
+        "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/sql.sql"
+      },
+      "warehouse_id": "cafef00d"
     },
-    "task_key": "for_each_spark_python_example"
+    "task_key": "sql_example"
   }
 ]
diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json b/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json
@@ -75,24 +75,6 @@
           "dev": "[USERNAME]"
         },
         "tasks": [
-          {
-            "job_cluster_key": "job_cluster",
-            "notebook_task": {
-              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
-            },
-            "task_key": "notebook_task"
-          },
-          {
-            "depends_on": [
-              {
-                "task_key": "notebook_task"
-              }
-            ],
-            "pipeline_task": {
-              "pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}"
-            },
-            "task_key": "refresh_pipeline"
-          },
           {
             "depends_on": [
               {
@@ -110,6 +92,24 @@
               "package_name": "project_name_[UNIQUE_NAME]"
             },
             "task_key": "main_task"
+          },
+          {
+            "job_cluster_key": "job_cluster",
+            "notebook_task": {
+              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
+            },
+            "task_key": "notebook_task"
+          },
+          {
+            "depends_on": [
+              {
+                "task_key": "notebook_task"
+              }
+            ],
+            "pipeline_task": {
+              "pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}"
+            },
+            "task_key": "refresh_pipeline"
           }
         ],
         "trigger": {

diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt
@@ -177,7 +177,7 @@ Validation OK!
 -          "dev": "[USERNAME]"
          },
          "tasks": [
-@@ -79,5 +66,5 @@
+@@ -97,5 +84,5 @@
              "job_cluster_key": "job_cluster",
              "notebook_task": {
 -              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
@@ -335,7 +335,7 @@ Resources:
 -          "dev": "[USERNAME]"
          },
          "tasks": [
-@@ -80,5 +67,5 @@
+@@ -98,5 +85,5 @@
              "job_cluster_key": "job_cluster",
              "notebook_task": {
 -              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"

diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
@@ -46,13 +46,6 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi
         "dev": "[USERNAME]"
       },
       "tasks": [
-        {
-          "job_cluster_key": "job_cluster",
-          "notebook_task": {
-            "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/files/src/notebook"
-          },
-          "task_key": "notebook_task"
-        },
         {
           "depends_on": [
             {
@@ -70,6 +63,13 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi
             "package_name": "my_jobs_as_code"
           },
           "task_key": "main_task"
+        },
+        {
+          "job_cluster_key": "job_cluster",
+          "notebook_task": {
+            "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/files/src/notebook"
+          },
+          "task_key": "notebook_task"
         }
       ],
       "trigger": {

diff --git a/bundle/config/mutator/resourcemutator/merge_job_tasks.go b/bundle/config/mutator/resourcemutator/merge_job_tasks.go
@@ -37,7 +37,11 @@ func (m *mergeJobTasks) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 		}
 
 		return dyn.Map(v, "resources.jobs", dyn.Foreach(func(_ dyn.Path, job dyn.Value) (dyn.Value, error) {
-			return dyn.Map(job, "tasks", merge.ElementsByKey("task_key", m.taskKeyString))
+			// Sorting keys here since it'll be sorted by TF anyway
+			// https://github.com/databricks/terraform-provider-databricks/blob/0a932c2/jobs/resource_job.go#L343
+			// However, if we don't sort we have a difference between direct and TF and between configs in
+			// "bundle validate" and configs sent to backend.
+			return dyn.Map(job, "tasks", merge.ElementsBySortedKey("task_key", m.taskKeyString))
 		}))
 	})
 

diff --git a/bundle/config/mutator/resourcemutator/merge_job_tasks_test.go b/bundle/config/mutator/resourcemutator/merge_job_tasks_test.go
@@ -65,11 +65,11 @@ func TestMergeJobTasks(t *testing.T) {
 	j := b.Config.Resources.Jobs["foo"]
 
 	assert.Len(t, j.Tasks, 2)
-	assert.Equal(t, "foo", j.Tasks[0].TaskKey)
-	assert.Equal(t, "bar", j.Tasks[1].TaskKey)
+	assert.Equal(t, "foo", j.Tasks[1].TaskKey)
+	assert.Equal(t, "bar", j.Tasks[0].TaskKey)
 
 	// This task was merged with a subsequent one.
-	task0 := j.Tasks[0]
+	task0 := j.Tasks[1]
 	cluster := task0.NewCluster
 	assert.Equal(t, "13.3.x-scala2.12", cluster.SparkVersion)
 	assert.Equal(t, "i3.2xlarge", cluster.NodeTypeId)
@@ -79,7 +79,7 @@ func TestMergeJobTasks(t *testing.T) {
 	assert.Equal(t, "package2", task0.Libraries[1].Pypi.Package)
 
 	// This task was left untouched.
-	task1 := j.Tasks[1].NewCluster
+	task1 := j.Tasks[0].NewCluster
 	assert.Equal(t, "10.4.x-scala2.12", task1.SparkVersion)
 }
 

diff --git a/libs/dyn/merge/elements_by_key.go b/libs/dyn/merge/elements_by_key.go
@@ -1,10 +1,15 @@
 package merge
 
-import "github.com/databricks/cli/libs/dyn"
+import (
+	"sort"
+
+	"github.com/databricks/cli/libs/dyn"
+)
 
 type elementsByKey struct {
-	key     string
-	keyFunc func(dyn.Value) string
+	key      string
+	keyFunc  func(dyn.Value) string
+	sortKeys bool
 }
 
 func (e elementsByKey) doMap(_ dyn.Path, v dyn.Value, mergeFunc func(a, b dyn.Value) (dyn.Value, error)) (dyn.Value, error) {
@@ -42,6 +47,10 @@ func (e elementsByKey) doMap(_ dyn.Path, v dyn.Value, mergeFunc func(a, b dyn.Va
 		seen[key] = nv
 	}
 
+	if e.sortKeys {
+		sort.Strings(keys)
+	}
+
 	// Gather resulting elements in natural order.
 	out := make([]dyn.Value, 0, len(keys))
 	for _, key := range keys {
@@ -83,9 +92,13 @@ func (e elementsByKey) MapWithOverride(p dyn.Path, v dyn.Value) (dyn.Value, erro
 // a parameter. The resulting elements get their key field overwritten
 // with the value as returned by the key function.
 func ElementsByKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
-	return elementsByKey{key, keyFunc}.Map
+	return elementsByKey{key, keyFunc, false}.Map
+}
+
+func ElementsBySortedKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
+	return elementsByKey{key, keyFunc, true}.Map
 }
 
 func ElementsByKeyWithOverride(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
-	return elementsByKey{key, keyFunc}.MapWithOverride
+	return elementsByKey{key, keyFunc, false}.MapWithOverride
 }
Original file line number	Diff line number	Diff line change
		@@ -1,2 +1 @@
		BundleConfig.default_name = ""
		EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # need to sort tasks by key