diff --git a/acceptance/bundle/templates/record-deployment-history/input.dbt-sql.json b/acceptance/bundle/templates/record-deployment-history/input.dbt-sql.json new file mode 100644 index 00000000000..8a0fef4dcd4 --- /dev/null +++ b/acceptance/bundle/templates/record-deployment-history/input.dbt-sql.json @@ -0,0 +1,3 @@ +{ + "project_name": "my_dbt_sql" +} diff --git a/acceptance/bundle/templates/record-deployment-history/input.default-python.json b/acceptance/bundle/templates/record-deployment-history/input.default-python.json new file mode 100644 index 00000000000..697fc31dd1b --- /dev/null +++ b/acceptance/bundle/templates/record-deployment-history/input.default-python.json @@ -0,0 +1,3 @@ +{ + "project_name": "my_default_python" +} diff --git a/acceptance/bundle/templates/record-deployment-history/input.default-scala.json b/acceptance/bundle/templates/record-deployment-history/input.default-scala.json new file mode 100644 index 00000000000..f35206cac89 --- /dev/null +++ b/acceptance/bundle/templates/record-deployment-history/input.default-scala.json @@ -0,0 +1,4 @@ +{ + "project_name": "my_default_scala", + "artifacts_dest_path": "/Volumes/test-folder" +} diff --git a/acceptance/bundle/templates/record-deployment-history/input.default-sql.json b/acceptance/bundle/templates/record-deployment-history/input.default-sql.json new file mode 100644 index 00000000000..5bb0b317d93 --- /dev/null +++ b/acceptance/bundle/templates/record-deployment-history/input.default-sql.json @@ -0,0 +1,3 @@ +{ + "project_name": "my_default_sql" +} diff --git a/acceptance/bundle/templates/record-deployment-history/input.lakeflow-integrations.json b/acceptance/bundle/templates/record-deployment-history/input.lakeflow-integrations.json new file mode 100644 index 00000000000..f88b7d45310 --- /dev/null +++ b/acceptance/bundle/templates/record-deployment-history/input.lakeflow-integrations.json @@ -0,0 +1,3 @@ +{ + "project_name": "my_lakeflow_integrations" +} diff --git a/acceptance/bundle/templates/record-deployment-history/out.default-python.databricks.yml b/acceptance/bundle/templates/record-deployment-history/out.default-python.databricks.yml new file mode 100644 index 00000000000..b9ede45b676 --- /dev/null +++ b/acceptance/bundle/templates/record-deployment-history/out.default-python.databricks.yml @@ -0,0 +1,49 @@ +# This is a Declarative Automation Bundle definition for my_default_python. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: my_default_python + uuid: [BUNDLE_UUID] + +experimental: + record_deployment_history: true + +include: + - resources/*.yml + +artifacts: + python_artifact: + type: whl + build: uv build --wheel + +# Variable declarations. These variables are assigned in the dev/prod targets below. +variables: + catalog: + description: The catalog to use + schema: + description: The schema to use + +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: [DATABRICKS_URL] + variables: + catalog: hive_metastore + schema: ${workspace.current_user.short_name} + prod: + mode: production + workspace: + host: [DATABRICKS_URL] + # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy. + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + variables: + catalog: hive_metastore + schema: prod + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE diff --git a/acceptance/bundle/templates/record-deployment-history/out.test.toml b/acceptance/bundle/templates/record-deployment-history/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/templates/record-deployment-history/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/templates/record-deployment-history/output.txt b/acceptance/bundle/templates/record-deployment-history/output.txt new file mode 100644 index 00000000000..befcd703c14 --- /dev/null +++ b/acceptance/bundle/templates/record-deployment-history/output.txt @@ -0,0 +1,70 @@ +Welcome to the default Python template for Declarative Automation Bundles! + +Answer the following questions to customize your project. +You can always change your configuration in the databricks.yml file later. + +Note that [DATABRICKS_URL] is used for initialization. +(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.) + +✨ Your new project has been created in the 'my_default_python' directory! + +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> grep -A1 ^experimental: output/default-python/my_default_python/databricks.yml +experimental: + record_deployment_history: true + +Welcome to the default SQL template for Declarative Automation Bundles! + +A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html. +workspace_host: [DATABRICKS_URL] + +✨ Your new project has been created in the 'my_default_sql' directory! + +Please refer to the README.md file for "getting started" instructions. +See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> grep -A1 ^experimental: output/default-sql/my_default_sql/databricks.yml +experimental: + record_deployment_history: true + +Welcome to the default-scala template for Declarative Automation Bundles! + +A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html. +workspace_host: [DATABRICKS_URL] + +✨ Your new project has been created in the 'my_default_scala' directory! + +Please refer to the README.md file for "getting started" instructions. +See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> grep -A1 ^experimental: output/default-scala/my_default_scala/databricks.yml +experimental: + record_deployment_history: true + +Welcome to the dbt template for Declarative Automation Bundles! + +A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html. +workspace_host: [DATABRICKS_URL] + +📊 Your new project has been created in the 'my_dbt_sql' directory! +If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started. +Refer to the README.md file for full "getting started" guide and production setup instructions. + + +>>> grep -A1 ^experimental: output/dbt-sql/my_dbt_sql/databricks.yml +experimental: + record_deployment_history: true +Welcome to the Lakeflow integrations template! + + +✨ Your new project has been created in the 'my_lakeflow_integrations' directory! + +Please refer to the README.md file for "getting started" instructions. + +>>> grep -A1 ^experimental: output/lakeflow-integrations/my_lakeflow_integrations/databricks.yml +experimental: + record_deployment_history: true + +>>> [CLI] bundle validate -o json +json.experimental.record_deployment_history = true; diff --git a/acceptance/bundle/templates/record-deployment-history/script b/acceptance/bundle/templates/record-deployment-history/script new file mode 100644 index 00000000000..bad50ccad90 --- /dev/null +++ b/acceptance/bundle/templates/record-deployment-history/script @@ -0,0 +1,17 @@ +# The env var in test.toml opts every built-in template into recording deployment history. + +# Leaving the generated projects behind would add their .gitignore files to this repo, see #2318. +trap 'rm -rf output' EXIT + +for template in default-python default-sql default-scala dbt-sql lakeflow-integrations; do + $CLI bundle init "$template" --config-file "input.$template.json" --output-dir "output/$template" + trace grep -A1 '^experimental:' output/$template/*/databricks.yml +done + +# Check that a generated project still loads and keeps the field through validation. +# This applies to the field rather than to the layout, so one template is enough. +(cd output/default-python/my_default_python && trace $CLI bundle validate -o json | gron.py | grep experimental) + +# Keep one file in full to assert where the stanza lands in the document. +mv output/default-python/my_default_python/databricks.yml out.default-python.databricks.yml +add_repl.py "$(grep -o 'uuid: .*' out.default-python.databricks.yml | cut -d ' ' -f2)" BUNDLE_UUID diff --git a/acceptance/bundle/templates/record-deployment-history/test.toml b/acceptance/bundle/templates/record-deployment-history/test.toml new file mode 100644 index 00000000000..a1d3232167a --- /dev/null +++ b/acceptance/bundle/templates/record-deployment-history/test.toml @@ -0,0 +1,4 @@ +Env.DATABRICKS_BUNDLE_INIT_RECORD_DEPLOYMENT_HISTORY = "true" + +# What is under test is what `bundle init` writes, so one engine is enough. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/libs/template/helpers.go b/libs/template/helpers.go index f32ae86c60b..a02efae3ea9 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -13,6 +13,7 @@ import ( "text/template" "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/iam" @@ -52,6 +53,14 @@ var metastoreDisabledErrorCodes = []string{ // is run and can be used to attribute DBU revenue to bundle templates. var bundleUuid = uuid.New().String() +// recordDeploymentHistoryEnvVar makes the built-in templates set +// experimental.record_deployment_history in the databricks.yml they generate. It is scoped to +// `bundle init` because it persists the setting in the generated project rather than applying +// it to the bundle being run. +// +// Temporary: remove it and its callers in the templates once this is the default. +const recordDeploymentHistoryEnvVar = "DATABRICKS_BUNDLE_INIT_RECORD_DEPLOYMENT_HISTORY" + func loadHelpers(ctx context.Context) template.FuncMap { w := cmdctx.WorkspaceClient(ctx) return template.FuncMap{ @@ -174,6 +183,10 @@ func loadHelpers(ctx context.Context) template.FuncMap { cachedIsServicePrincipal = &result return result, nil }, + "record_deployment_history": func() bool { + v, _ := env.GetBool(ctx, recordDeploymentHistoryEnvVar) + return v + }, "lower": func(s string) string { return strings.ToLower(s) }, diff --git a/libs/template/templates/common/library/experimental.tmpl b/libs/template/templates/common/library/experimental.tmpl new file mode 100644 index 00000000000..d55f4234b02 --- /dev/null +++ b/libs/template/templates/common/library/experimental.tmpl @@ -0,0 +1,15 @@ +{{- /* + * Temporary opt-in for recording deployment history; remove once it is the default. + * + * The flag is passed in rather than read from the record_deployment_history helper here, + * because this shared library is parsed for every template, including by callers that + * register no helpers. + */ -}} + +{{- define "experimental_record_deployment_history" -}} +{{- if .}} + +experimental: + record_deployment_history: true +{{- end}} +{{- end}} diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl index 89e66ef929c..ddc236733bf 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -4,6 +4,7 @@ bundle: name: {{.project_name}} uuid: {{bundle_uuid}} +{{- template "experimental_record_deployment_history" record_deployment_history}} include: - resources/*.yml diff --git a/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl index 9ad15fddf19..ad7a32de3e8 100644 --- a/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl @@ -3,6 +3,7 @@ bundle: name: {{.project_name}} uuid: {{bundle_uuid}} +{{- template "experimental_record_deployment_history" record_deployment_history}} include: - resources/*.yml diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl index f475da8593e..9749477f312 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -3,6 +3,7 @@ bundle: name: {{.project_name}} uuid: {{bundle_uuid}} +{{- template "experimental_record_deployment_history" record_deployment_history}} include: - resources/*.yml diff --git a/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl index 7fb657982aa..6f86e9ae9b1 100644 --- a/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl @@ -6,6 +6,7 @@ bundle: name: {{.project_name}} uuid: {{bundle_uuid}} +{{- template "experimental_record_deployment_history" record_deployment_history}} {{- if $with_pydabs }} python: diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl index da766018c38..0d781d38a8c 100644 --- a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl @@ -4,6 +4,7 @@ bundle: name: {{.project_name}} uuid: {{bundle_uuid}} engine: direct +{{- template "experimental_record_deployment_history" record_deployment_history}} include: - resources/*.yml