bundle: add env var to set record_deployment_history in built-in templates

acceptance/bundle/templates/record-deployment-history/input.dbt-sql.json

@@ -0,0 +1,3 @@
+{
+    "project_name": "my_dbt_sql"
+}

acceptance/bundle/templates/record-deployment-history/input.default-python.json

@@ -0,0 +1,3 @@
+{
+    "project_name": "my_default_python"
+}

acceptance/bundle/templates/record-deployment-history/input.default-scala.json

@@ -0,0 +1,4 @@
+{
+    "project_name": "my_default_scala",
+    "artifacts_dest_path": "/Volumes/test-folder"
+}

acceptance/bundle/templates/record-deployment-history/input.default-sql.json

@@ -0,0 +1,3 @@
+{
+    "project_name": "my_default_sql"
+}

acceptance/bundle/templates/record-deployment-history/input.lakeflow-integrations.json

@@ -0,0 +1,3 @@
+{
+    "project_name": "my_lakeflow_integrations"
+}

acceptance/bundle/templates/record-deployment-history/out.default-python.databricks.yml

@@ -0,0 +1,49 @@
+# This is a Declarative Automation Bundle definition for my_default_python.
+# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
+bundle:
+  name: my_default_python
+  uuid: [BUNDLE_UUID]
+
+experimental:
+  record_deployment_history: true
+
+include:
+  - resources/*.yml
+
+artifacts:
+  python_artifact:
+    type: whl
+    build: uv build --wheel
+
+# Variable declarations. These variables are assigned in the dev/prod targets below.
+variables:
+  catalog:
+    description: The catalog to use
+  schema:
+    description: The schema to use
+
+targets:
+  dev:
+    # The default target uses 'mode: development' to create a development copy.
+    # - Deployed resources get prefixed with '[dev my_user_name]'
+    # - Any job schedules and triggers are paused by default.
+    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
+    mode: development
+    default: true
+    workspace:
+      host: [DATABRICKS_URL]
+    variables:
+      catalog: hive_metastore
+      schema: ${workspace.current_user.short_name}
+  prod:
+    mode: production
+    workspace:
+      host: [DATABRICKS_URL]
+      # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy.
+      root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target}
+    variables:
+      catalog: hive_metastore
+      schema: prod
+    permissions:
+      - user_name: [USERNAME]
+        level: CAN_MANAGE

acceptance/bundle/templates/record-deployment-history/output.txt

@@ -0,0 +1,70 @@
+Welcome to the default Python template for Declarative Automation Bundles!
+
+Answer the following questions to customize your project.
+You can always change your configuration in the databricks.yml file later.
+
+Note that [DATABRICKS_URL] is used for initialization.
+(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.)
+
+✨ Your new project has been created in the 'my_default_python' directory!
+
+To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
+
+>>> grep -A1 ^experimental: output/default-python/my_default_python/databricks.yml
+experimental:
+  record_deployment_history: true
+
+Welcome to the default SQL template for Declarative Automation Bundles!
+
+A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
+workspace_host: [DATABRICKS_URL]
+
+✨ Your new project has been created in the 'my_default_sql' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
+
+>>> grep -A1 ^experimental: output/default-sql/my_default_sql/databricks.yml
+experimental:
+  record_deployment_history: true
+
+Welcome to the default-scala template for Declarative Automation Bundles!
+
+A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
+workspace_host: [DATABRICKS_URL]
+
+✨ Your new project has been created in the 'my_default_scala' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
+
+>>> grep -A1 ^experimental: output/default-scala/my_default_scala/databricks.yml
+experimental:
+  record_deployment_history: true
+
+Welcome to the dbt template for Declarative Automation Bundles!
+
+A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
+workspace_host: [DATABRICKS_URL]
+
+📊 Your new project has been created in the 'my_dbt_sql' directory!
+If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started.
+Refer to the README.md file for full "getting started" guide and production setup instructions.
+
+
+>>> grep -A1 ^experimental: output/dbt-sql/my_dbt_sql/databricks.yml
+experimental:
+  record_deployment_history: true
+Welcome to the Lakeflow integrations template!
+
+
+✨ Your new project has been created in the 'my_lakeflow_integrations' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+
+>>> grep -A1 ^experimental: output/lakeflow-integrations/my_lakeflow_integrations/databricks.yml
+experimental:
+  record_deployment_history: true
+
+>>> [CLI] bundle validate -o json
+json.experimental.record_deployment_history = true;

acceptance/bundle/templates/record-deployment-history/script

@@ -0,0 +1,17 @@
+# The env var in test.toml opts every built-in template into recording deployment history.
+
+# Leaving the generated projects behind would add their .gitignore files to this repo, see #2318.
+trap 'rm -rf output' EXIT
+
+for template in default-python default-sql default-scala dbt-sql lakeflow-integrations; do
+    $CLI bundle init "$template" --config-file "input.$template.json" --output-dir "output/$template"
+    trace grep -A1 '^experimental:' output/$template/*/databricks.yml
+done
+
+# Check that a generated project still loads and keeps the field through validation.
+# This applies to the field rather than to the layout, so one template is enough.
+(cd output/default-python/my_default_python && trace $CLI bundle validate -o json | gron.py | grep experimental)
+
+# Keep one file in full to assert where the stanza lands in the document.
+mv output/default-python/my_default_python/databricks.yml out.default-python.databricks.yml
+add_repl.py "$(grep -o 'uuid: .*' out.default-python.databricks.yml | cut -d ' ' -f2)" BUNDLE_UUID

acceptance/bundle/templates/record-deployment-history/test.toml

@@ -0,0 +1,4 @@
+Env.DATABRICKS_BUNDLE_INIT_RECORD_DEPLOYMENT_HISTORY = "true"
+
+# What is under test is what `bundle init` writes, so one engine is enough.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

libs/template/helpers.go

@@ -13,6 +13,7 @@ import (
 	"text/template"
 
 	"github.com/databricks/cli/libs/cmdctx"
+	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/iamutil"
 	"github.com/databricks/databricks-sdk-go/apierr"
 	"github.com/databricks/databricks-sdk-go/service/iam"
@@ -52,6 +53,14 @@ var metastoreDisabledErrorCodes = []string{
 // is run and can be used to attribute DBU revenue to bundle templates.
 var bundleUuid = uuid.New().String()
 
+// recordDeploymentHistoryEnvVar makes the built-in templates set
+// experimental.record_deployment_history in the databricks.yml they generate. It is scoped to
+// `bundle init` because it persists the setting in the generated project rather than applying
+// it to the bundle being run.
+//
+// Temporary: remove it and its callers in the templates once this is the default.
+const recordDeploymentHistoryEnvVar = "DATABRICKS_BUNDLE_INIT_RECORD_DEPLOYMENT_HISTORY"
+
 func loadHelpers(ctx context.Context) template.FuncMap {
 	w := cmdctx.WorkspaceClient(ctx)
 	return template.FuncMap{
@@ -174,6 +183,10 @@ func loadHelpers(ctx context.Context) template.FuncMap {
 			cachedIsServicePrincipal = &result
 			return result, nil
 		},
+		"record_deployment_history": func() bool {
+			v, _ := env.GetBool(ctx, recordDeploymentHistoryEnvVar)
+			return v
+		},
 		"lower": func(s string) string {
 			return strings.ToLower(s)
 		},

libs/template/templates/common/library/experimental.tmpl

@@ -0,0 +1,15 @@
+{{- /*
+     * Temporary opt-in for recording deployment history; remove once it is the default.
+     *
+     * The flag is passed in rather than read from the record_deployment_history helper here,
+     * because this shared library is parsed for every template, including by callers that
+     * register no helpers.
+     */ -}}
+
+{{- define "experimental_record_deployment_history" -}}
+{{- if .}}
+
+experimental:
+  record_deployment_history: true
+{{- end}}
+{{- end}}

libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl

@@ -4,6 +4,7 @@
 bundle:
   name: {{.project_name}}
   uuid: {{bundle_uuid}}
+{{- template "experimental_record_deployment_history" record_deployment_history}}
 
 include:
   - resources/*.yml

libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl

@@ -3,6 +3,7 @@
 bundle:
   name: {{.project_name}}
   uuid: {{bundle_uuid}}
+{{- template "experimental_record_deployment_history" record_deployment_history}}
 
 include:
   - resources/*.yml

libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl

@@ -3,6 +3,7 @@
 bundle:
   name: {{.project_name}}
   uuid: {{bundle_uuid}}
+{{- template "experimental_record_deployment_history" record_deployment_history}}
 
 include:
   - resources/*.yml

libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl

@@ -6,6 +6,7 @@
 bundle:
   name: {{.project_name}}
   uuid: {{bundle_uuid}}
+{{- template "experimental_record_deployment_history" record_deployment_history}}
 {{- if $with_pydabs }}
 
 python:

libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl

@@ -4,6 +4,7 @@ bundle:
   name: {{.project_name}}
   uuid: {{bundle_uuid}}
   engine: direct
+{{- template "experimental_record_deployment_history" record_deployment_history}}
 
 include:
   - resources/*.yml