From f0673efa12866457321cee27dad46df94d75f85f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 3 Sep 2026 20:36:52 +0000 Subject: [PATCH 001/164] bundle: record and read deployment state via DMS Record each deploy's resource state to the deployment metadata service and read it back, making the service the source of truth for a recorded bundle: the local state file becomes a tombstone and a leftover WAL is discarded. Gated on experimental.record_deployment_history (direct engine only), and supports deploy --plan. Co-authored-by: Isaac --- acceptance/bin/dms_resources.py | 105 +++ acceptance/bin/nostamp | 41 -- acceptance/bin/nostamp.py | 79 +++ acceptance/bin/print_state.py | 31 +- acceptance/bin/read_id.py | 21 +- acceptance/bin/read_state.py | 19 +- acceptance/bin/replace_ids.py | 18 +- .../whl_dynamic/out.plan_create.direct.json | 2 +- .../whl_dynamic/out.plan_update.direct.json | 2 +- .../bundle/artifacts/whl_dynamic/script | 6 +- .../artifacts/whl_implicit_custom_path/script | 2 +- .../artifacts/whl_via_environment_key/script | 2 +- .../whl_via_environment_key_extras/script | 2 +- acceptance/bundle/bundle_tag/id/script | 2 +- acceptance/bundle/bundle_tag/url/script | 2 +- .../bundle/deploy/experimental-python/script | 2 +- acceptance/bundle/deploy/immutable/script | 2 +- .../bundle/deploy/python-notebook/script | 2 +- .../readplan/basic/out.plan_create.json | 2 +- .../deploy/readplan/basic/out.plan_skip.json | 2 +- .../bundle/deploy/readplan/basic/script | 4 +- .../readplan/cli-version-mismatch/plan.json | 2 +- .../readplan/lineage-mismatch/plan.json | 2 +- .../readplan/plan-version-mismatch/output.txt | 2 +- acceptance/bundle/deploy/readplan/test.toml | 4 +- .../readplan/unknown-field/out.plan.json | 2 +- .../deploy/readplan/unknown-field/script | 2 +- .../readplan/whl-not-uploaded/out.plan.json | 2 +- .../deploy/wal/chain-3-jobs/out.test.toml | 2 +- .../bundle/deploy/wal/chain-3-jobs/script | 2 +- .../wal/crash-after-create/out.test.toml | 2 +- .../deploy/wal/crash-after-create/script | 2 +- .../bundle/deploy/wal/empty-wal/out.test.toml | 2 +- .../wal/failed-plan-no-wal/out.test.toml | 2 +- .../wal/future-serial-wal/out.test.toml | 2 +- .../deploy/wal/header-only-wal/out.test.toml | 2 +- .../deploy/wal/header-only-wal/output.txt | 2 +- .../bundle/deploy/wal/header-only-wal/script | 2 +- .../deploy/wal/lineage-mismatch/out.test.toml | 2 +- acceptance/bundle/deploy/wal/test.toml | 6 + .../deploy/wal/wal-with-delete/out.test.toml | 2 +- .../bundle/deployment/bind/experiment/script | 4 +- .../bundle/destroy/jobs-and-pipeline/script | 2 +- .../dms/declined-deploy/databricks.yml.tmpl | 10 + .../dms/declined-deploy/out.requests.txt | 7 + .../bundle/dms/declined-deploy/out.test.toml | 3 + .../bundle/dms/declined-deploy/output.txt | 88 +++ acceptance/bundle/dms/declined-deploy/script | 22 + .../bundle/dms/depends-on/databricks.yml.tmpl | 12 + .../bundle/dms/depends-on/out.test.toml | 3 + acceptance/bundle/dms/depends-on/output.txt | 47 ++ acceptance/bundle/dms/depends-on/script | 8 + .../databricks.yml.tmpl | 22 + .../deployment-metadata-change/out.test.toml | 3 + .../dms/deployment-metadata-change/output.txt | 144 ++++ .../dms/deployment-metadata-change/script | 14 + .../dms/emptied-resource/databricks.yml.tmpl | 11 + .../bundle/dms/emptied-resource/out.test.toml | 3 + .../bundle/dms/emptied-resource/output.txt | 142 ++++ acceptance/bundle/dms/emptied-resource/script | 20 + .../dms/existing-state/databricks.yml.tmpl | 7 + .../bundle/dms/existing-state/out.test.toml | 3 + .../bundle/dms/existing-state/output.txt | 78 +++ acceptance/bundle/dms/existing-state/script | 27 + .../bundle/dms/existing-state/test.toml | 3 + .../dms/failed-delete/databricks.yml.tmpl | 9 + .../bundle/dms/failed-delete/out.test.toml | 3 + .../bundle/dms/failed-delete/output.txt | 149 +++++ acceptance/bundle/dms/failed-delete/script | 16 + acceptance/bundle/dms/failed-delete/test.toml | 3 + .../dms/failed-recreate/databricks.yml.tmpl | 11 + .../dms/failed-recreate/out.requests.txt | 7 + .../bundle/dms/failed-recreate/out.test.toml | 3 + .../bundle/dms/failed-recreate/output.txt | 87 +++ acceptance/bundle/dms/failed-recreate/script | 32 + .../bundle/dms/failed-recreate/test.toml | 5 + .../databricks.yml.tmpl | 12 + .../failed-update-permissions/out.test.toml | 3 + .../dms/failed-update-permissions/output.txt | 29 + .../dms/failed-update-permissions/script | 12 + .../dms/failed-update-permissions/test.toml | 5 + .../dms/failed-update/databricks.yml.tmpl | 11 + .../bundle/dms/failed-update/out.test.toml | 3 + .../bundle/dms/failed-update/output.txt | 86 +++ acceptance/bundle/dms/failed-update/script | 28 + acceptance/bundle/dms/failed-update/test.toml | 3 + .../multiple-resources/databricks.yml.tmpl | 17 + .../dms/multiple-resources/out.test.toml | 3 + .../bundle/dms/multiple-resources/output.txt | 28 + .../bundle/dms/multiple-resources/script | 9 + .../bundle/dms/no-drift/databricks.yml.tmpl | 20 + acceptance/bundle/dms/no-drift/out.test.toml | 3 + acceptance/bundle/dms/no-drift/output.txt | 126 ++++ acceptance/bundle/dms/no-drift/script | 26 + acceptance/bundle/dms/no-drift/transform.py | 1 + .../dms/no-resources/databricks.yml.tmpl | 4 + .../bundle/dms/no-resources/out.test.toml | 3 + acceptance/bundle/dms/no-resources/output.txt | 80 +++ acceptance/bundle/dms/no-resources/script | 10 + .../databricks.yml.tmpl | 11 + .../out.requests.txt | 7 + .../operation-upload-fails-once/out.test.toml | 3 + .../operation-upload-fails-once/output.txt | 79 +++ .../dms/operation-upload-fails-once/script | 15 + .../dms/operation-upload-fails-once/test.toml | 3 + .../databricks.yml.tmpl | 23 + .../dms/operation-upload-fails/out.test.toml | 3 + .../dms/operation-upload-fails/output.txt | 4 + .../bundle/dms/operation-upload-fails/script | 9 + .../dms/operation-upload-fails/test.toml | 10 + .../databricks.yml.tmpl | 9 + .../operation-upload-message/out.test.toml | 3 + .../dms/operation-upload-message/output.txt | 16 + .../dms/operation-upload-message/script | 5 + .../dms/operation-upload-message/test.toml | 8 + .../bundle/dms/provenance/databricks.yml.tmpl | 14 + .../bundle/dms/provenance/out.test.toml | 3 + acceptance/bundle/dms/provenance/output.txt | 68 ++ acceptance/bundle/dms/provenance/script | 19 + acceptance/bundle/dms/provenance/test.toml | 4 + .../bundle/dms/readplan/databricks.yml.tmpl | 19 + acceptance/bundle/dms/readplan/out.test.toml | 3 + acceptance/bundle/dms/readplan/output.txt | 22 + acceptance/bundle/dms/readplan/script | 13 + acceptance/bundle/dms/readplan/transform.py | 1 + .../dms/record-failure/databricks.yml.tmpl | 9 + .../bundle/dms/record-failure/out.test.toml | 3 + .../bundle/dms/record-failure/output.txt | 75 +++ acceptance/bundle/dms/record-failure/script | 21 + .../bundle/dms/record-failure/test.toml | 5 + .../bundle/dms/record/databricks.yml.tmpl | 9 + acceptance/bundle/dms/record/out.test.toml | 3 + acceptance/bundle/dms/record/output.txt | 84 +++ acceptance/bundle/dms/record/script | 16 + .../dms/serialized-plan/databricks.yml.tmpl | 9 + .../bundle/dms/serialized-plan/out.test.toml | 3 + .../bundle/dms/serialized-plan/output.txt | 41 ++ acceptance/bundle/dms/serialized-plan/script | 20 + .../bundle/dms/stale-plan/databricks.yml.tmpl | 9 + .../bundle/dms/stale-plan/out.test.toml | 3 + acceptance/bundle/dms/stale-plan/output.txt | 21 + acceptance/bundle/dms/stale-plan/script | 13 + .../successful-recreate/databricks.yml.tmpl | 11 + .../dms/successful-recreate/out.requests.txt | 7 + .../dms/successful-recreate/out.test.toml | 3 + .../bundle/dms/successful-recreate/output.txt | 180 +++++ .../bundle/dms/successful-recreate/script | 20 + .../bundle/dms/summary/databricks.yml.tmpl | 9 + acceptance/bundle/dms/summary/out.test.toml | 3 + acceptance/bundle/dms/summary/output.txt | 37 ++ acceptance/bundle/dms/summary/script | 17 + acceptance/bundle/dms/test.toml | 19 + .../dms/version-never-created/databricks.yml | 10 + .../dms/version-never-created/out.test.toml | 3 + .../dms/version-never-created/output.txt | 38 ++ .../bundle/dms/version-never-created/script | 8 + .../dms/version-never-created/test.toml | 12 + acceptance/bundle/empty_string_dropped/script | 2 +- .../bundle/environments/dependencies/script | 8 +- .../bundle/escaped_refs/out.plan.direct.json | 2 +- acceptance/bundle/escaped_refs/out.test.toml | 2 +- acceptance/bundle/escaped_refs/output.txt | 2 +- acceptance/bundle/escaped_refs/script | 6 +- acceptance/bundle/escaped_refs/test.toml | 4 + .../generate/dashboard-inplace/out.test.toml | 2 +- .../generate/dashboard-inplace/test.toml | 4 + .../genie_space_inplace/out.test.toml | 2 +- .../generate/genie_space_inplace/test.toml | 3 + .../out.plan_update.direct.json | 2 +- .../bundle/migrate/auto-migrate-clean/script | 2 +- .../migrate/auto-migrate-clean/test.toml | 4 + .../migrate/auto-migrate-direct-only/script | 2 +- .../bundle/migrate/basic/out.plan_update.json | 2 +- acceptance/bundle/migrate/basic/script | 2 +- .../dashboards/out.plan_after_migrate.json | 2 +- acceptance/bundle/migrate/dashboards/script | 4 +- .../default-python/out.plan_after_deploy.json | 2 +- .../out.plan_after_migration.json | 2 +- .../bundle/migrate/default-python/script | 4 +- acceptance/bundle/migrate/runas/out.plan.json | 2 +- acceptance/bundle/migrate/runas/script | 2 +- .../resource_deps/escaped_ref/output.txt | 2 +- .../bundle/resource_deps/escaped_ref/script | 4 +- .../grant_ref/out.plan_create.direct.json | 2 +- .../bundle/resource_deps/grant_ref/script | 2 +- .../id_chain/out.plan_create.direct.json | 2 +- .../id_chain/out.plan_skip.direct.json | 2 +- .../id_chain/out.plan_update.direct.json | 2 +- .../bundle/resource_deps/id_chain/script | 6 +- .../id_star/out.plan_create.direct.json | 2 +- .../bundle/resource_deps/id_star/script | 2 +- .../job_id/out.plan_create.direct.json | 2 +- .../job_id/out.plan_delete.direct.json | 2 +- acceptance/bundle/resource_deps/job_id/script | 4 +- .../out.plan_delete.direct.json | 2 +- .../resource_deps/job_id_delete_bar/script | 2 +- .../out.plan_delete.direct.json | 2 +- .../resource_deps/job_id_delete_foo/script | 2 +- .../bundle/resource_deps/jobs_update/script | 4 +- .../out.plan_create.direct.json | 2 +- .../out.plan_update.direct.json | 2 +- .../resource_deps/jobs_update_remote/script | 4 +- .../resource_deps/missing_string_field/script | 2 +- .../out.plan_create.direct.json | 2 +- .../resource_deps/permission_ref/script | 2 +- .../out.plan_create.direct.json | 2 +- .../out.plan_noop.direct.json | 2 +- .../out.plan_update.direct.json | 2 +- .../resource_deps/pipelines_recreate/script | 10 +- .../out.plan_create.direct.json | 2 +- .../remote_pipeline/out.plan_skip.direct.json | 2 +- .../resource_deps/remote_pipeline/script | 4 +- .../resources_var/out.plan.direct.json | 2 +- .../bundle/resource_deps/resources_var/script | 2 +- .../out.plan.direct.json | 2 +- .../script | 2 +- .../tf_path_renames/out.plan.direct.json | 2 +- .../resource_deps/tf_path_renames/script | 2 +- .../out.plan.direct.json | 2 +- .../volume_path_contains_id/script | 2 +- .../volume_path_job_ref/out.plan.direct.json | 2 +- .../volume_path_job_ref/out.test.toml | 2 +- .../resource_deps/volume_path_job_ref/script | 2 +- .../volume_path_job_ref/test.toml | 4 + .../apps/config-drift-stopped/script | 4 +- .../bundle/resources/apps/config-drift/script | 6 +- .../apps/config-no-deployment/script | 2 +- .../apps/git-source-no-deployment/script | 2 +- .../apps/lifecycle-started-omitted/script | 2 +- .../resources/apps/uc-securable-drift/script | 2 +- .../catalogs/comment_out_of_band/script | 2 +- .../catalogs/drift/managed_properties/script | 2 +- .../out.plan_.direct.json | 10 +- .../deploy/update-and-resize-autoscale/script | 12 +- .../update-and-resize/out.plan_.direct.json | 8 +- .../clusters/deploy/update-and-resize/script | 10 +- .../out.plan.direct.json | 2 +- .../change-serialized-inline/out.test.toml | 2 +- .../change-serialized-inline/script | 2 +- .../change-serialized-inline/test.toml | 4 + .../out.plan.direct.json | 2 +- .../dashboards/dataset-catalog-schema/script | 2 +- .../out.plan.direct.json | 2 +- .../delete-trashed-out-of-band/script | 2 +- .../detect-change/out.plan.direct.json | 2 +- .../resources/dashboards/detect-change/script | 2 +- .../dashboards/generate_inplace/out.test.toml | 2 +- .../dashboards/generate_inplace/test.toml | 3 + .../publish-failure-stale-content/script | 4 +- .../republish-after-draft-update/script | 4 +- .../dashboards/simple/out.plan.direct.json | 2 +- .../bundle/resources/dashboards/simple/script | 2 +- .../out.plan.direct.json | 2 +- .../out.plan_initial.direct.json | 2 +- .../dashboards/unpublish-out-of-band/script | 4 +- .../genie_spaces/inline/out.plan.json | 2 +- .../resources/genie_spaces/inline/script | 2 +- .../genie_spaces/version_migration/script | 2 +- .../grants/catalogs/out.plan1.direct.json | 2 +- .../grants/catalogs/out.plan2.direct.json | 2 +- .../bundle/resources/grants/catalogs/script | 4 +- .../registered_models/out.plan1.direct.json | 2 +- .../resources/grants/registered_models/script | 2 +- .../change_privilege/out.plan1.direct.json | 2 +- .../change_privilege/out.plan2.direct.json | 2 +- .../grants/schemas/change_privilege/script | 4 +- .../duplicate_principals/out.plan.direct.json | 2 +- .../schemas/duplicate_principals/script | 2 +- .../out.plan.direct.json | 2 +- .../out.plan2.direct.json | 2 +- .../schemas/out_of_band_principal/script | 4 +- .../grants/volumes/out.plan1.direct.json | 2 +- .../grants/volumes/out.plan2.direct.json | 2 +- .../bundle/resources/grants/volumes/script | 4 +- .../resources/job_runs/failed_run/script | 2 +- .../bundle/resources/job_runs/redeploy/script | 2 +- .../bundle/resources/job_runs/wait/script | 2 +- .../jobs/big_id/out.plan.direct.json | 2 +- .../bundle/resources/jobs/big_id/script | 2 +- .../bundle/resources/jobs/big_id/test.toml | 4 +- .../jobs/delete_job/out.plan.direct.json | 2 +- .../bundle/resources/jobs/delete_job/script | 2 +- .../delete_task/out.plan_create.direct.json | 2 +- .../delete_task/out.plan_update.direct.json | 2 +- .../bundle/resources/jobs/delete_task/script | 4 +- .../resources/jobs/delete_task/test.toml | 4 +- .../jobs/on_failure_empty_slice/script | 2 +- .../out.plan_post_update.direct.json | 2 +- .../resources/jobs/remote_add_tag/script | 2 +- .../remote_delete/deploy/out.plan.direct.json | 2 +- .../jobs/remote_delete/deploy/test.toml | 4 +- .../out.plan.direct.json | 2 +- .../jobs/remote_matches_config/script | 2 +- .../resources/jobs/tags_empty_map/script | 2 +- .../jobs/update/out.plan_create.direct.json | 2 +- .../jobs/update/out.plan_skip.direct.json | 2 +- .../jobs/update/out.plan_update.direct.json | 2 +- .../bundle/resources/jobs/update/script | 6 +- .../bundle/resources/jobs/update/test.toml | 4 +- .../out.plan_create.direct.json | 2 +- .../out.plan_skip.direct.json | 2 +- .../out.plan_update.direct.json | 2 +- .../resources/jobs/update_single_node/script | 6 +- .../jobs/webhook-reorder-remote/script | 2 +- .../basic/out.first-plan.direct.txt | 2 +- .../basic/out.second-plan.direct.txt | 2 +- .../model_serving_endpoints/basic/script | 4 +- .../drift/telemetry_config_unmanaged/script | 2 +- .../drift/write_only/script | 2 +- .../catalog-name/out.first-plan.direct.json | 2 +- .../catalog-name/out.second-plan.direct.json | 2 +- .../recreate/catalog-name/script | 4 +- .../name-change/out.first-plan.direct.json | 2 +- .../name-change/out.second-plan.direct.json | 2 +- .../recreate/name-change/script | 4 +- .../out.first-plan.direct.json | 2 +- .../out.second-plan.direct.json | 2 +- .../recreate/route-optimized/script | 4 +- .../schema-name/out.first-plan.direct.json | 2 +- .../schema-name/out.second-plan.direct.json | 2 +- .../recreate/schema-name/script | 4 +- .../table-prefix/out.first-plan.direct.json | 2 +- .../table-prefix/out.second-plan.direct.json | 2 +- .../recreate/table-prefix/script | 4 +- .../running-endpoint/out.plan.direct.txt | 2 +- .../running-endpoint/script | 2 +- .../update/ai-gateway/out.plan.direct.json | 2 +- .../update/ai-gateway/script | 2 +- .../out.plan.direct.json | 2 +- .../update/both_gateway_and_tags/script | 2 +- .../update/config/out.plan.direct.json | 2 +- .../update/config/script | 2 +- .../email-notifications/out.plan.direct.json | 2 +- .../update/email-notifications/script | 2 +- .../update/tags/out.plan.direct.json | 2 +- .../update/tags/script | 2 +- .../bundle/resources/permissions/_script | 4 +- .../current_can_manage/out.plan.direct.json | 2 +- .../other_can_manage/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../dashboards/create/out.plan.direct.json | 2 +- .../permissions/dashboards/create/script | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../genie_spaces/current_can_manage/script | 2 +- .../jobs/added_remotely/out.plan.direct.json | 2 +- .../permissions/jobs/added_remotely/script | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../out.plan.direct.txt | 2 +- .../jobs/current_can_manage_run/script | 4 +- .../current_is_owner/out.plan.direct.json | 2 +- .../delete_one/out.plan_create.direct.json | 2 +- .../permissions/jobs/delete_one/script | 2 +- .../out.plan_create.direct.json | 2 +- .../out.plan_restore.direct.json | 2 +- .../permissions/jobs/deleted_remotely/script | 4 +- .../jobs/destroy_without_mgmtperms/test.toml | 5 +- .../jobs/empty_list/out.plan.direct.json | 2 +- .../other_can_manage/out.plan.direct.json | 2 +- .../jobs/other_is_owner/out.plan.direct.json | 2 +- .../jobs/update/out.plan_create.direct.json | 2 +- .../update/out.plan_delete_all.direct.json | 2 +- .../update/out.plan_delete_one.direct.json | 2 +- .../update/out.plan_post_create.direct.json | 2 +- .../jobs/update/out.plan_restore.direct.json | 2 +- .../update/out.plan_set_empty.direct.json | 2 +- .../jobs/update/out.plan_update.direct.json | 2 +- .../resources/permissions/jobs/update/script | 14 +- .../jobs/viewers/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../models/current_can_manage/script | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../current_is_owner/out.plan.direct.json | 2 +- .../pipelines/empty_list/out.plan.direct.json | 2 +- .../other_can_manage/out.plan.direct.json | 2 +- .../other_is_owner/out.plan.direct.json | 2 +- .../update/out.plan_create.direct.json | 2 +- .../update/out.plan_delete_all.direct.json | 2 +- .../update/out.plan_delete_one.direct.json | 2 +- .../update/out.plan_restore.direct.json | 2 +- .../update/out.plan_update.direct.json | 2 +- .../permissions/pipelines/update/script | 10 +- .../current_can_manage/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../target_permissions/out.plan.direct.json | 2 +- .../permissions/target_permissions/script | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../resources/pipelines/auto-approve/script | 8 +- .../pipelines/lakeflow-pipeline/script | 2 +- .../pipelines/num-workers-zero/script | 2 +- .../resources/pipelines/recreate-keys/_script | 8 +- .../out.plan_create.direct.json | 2 +- .../out.plan_recreate.direct.json | 2 +- .../out.plan_create.direct.json | 2 +- .../out.plan_recreate.direct.json | 2 +- .../resources/pipelines/recreate/script | 4 +- .../out.plan.direct.json | 2 +- .../pipelines/remote_matches_config/script | 2 +- .../bundle/resources/pipelines/update/script | 2 +- .../purge_on_delete_transitions/script | 2 +- .../recreate_source_branch_time/script | 2 +- .../update_expire_time/script | 2 +- .../postgres_branches/update_protected/script | 8 +- .../postgres_branches/update_ttl/script | 2 +- .../postgres_databases/update/script | 8 +- .../remove_suspend_timeout/script | 4 +- .../update_autoscaling/script | 8 +- .../update_pg_settings/script | 2 +- .../update_suspend_timeout/script | 2 +- .../purge_on_delete_transitions/script | 2 +- .../remove_history_retention/script | 4 +- .../update_default_endpoint_suspend/script | 2 +- .../update_display_name/script | 8 +- .../update_history_retention/script | 2 +- .../inherited-role-bind/out.test.toml | 2 +- .../inherited-role-bind/test.toml | 4 + .../resources/postgres_roles/update/script | 8 +- .../change_assets_dir/out.plan.direct.json | 2 +- .../change_assets_dir/out.test.toml | 2 +- .../change_assets_dir/output.txt | 2 +- .../change_assets_dir/test.toml | 3 + .../out.plan.direct.json | 2 +- .../change_output_schema_name/script | 2 +- .../change_table_name/out.plan.direct.json | 2 +- .../change_table_name/out.test.toml | 2 +- .../change_table_name/test.toml | 4 + .../create/out.plan_create.direct.json | 2 +- .../create/out.plan_noop.direct.json | 2 +- .../resources/quality_monitors/create/script | 4 +- .../registered_models/aliases_converge/script | 2 +- .../resources/registered_models/basic/script | 2 +- .../drift/browse_only/script | 2 +- .../resources/schemas/auto-approve/script | 4 +- .../schemas/comment_out_of_band/script | 2 +- .../schemas/drift/managed_properties/script | 2 +- .../secret_scopes/basic/out.plan1.direct.txt | 2 +- .../secret_scopes/basic/out.plan2.direct.txt | 2 +- .../basic/out.plan_verify_no_drift.direct.txt | 2 +- .../resources/secret_scopes/basic/script | 6 +- .../secrets/remove-expire-time/script | 4 +- .../secrets/update-expire-time/script | 2 +- .../resources/secrets/update-value/output.txt | 26 +- .../resources/secrets/update-value/script | 6 +- .../drift/target_qps/script | 2 +- .../recreate/with_endpoint/output.txt | 10 +- .../recreate/with_endpoint/script | 2 +- .../volumes/catalog-var-ref/output.txt | 2 +- .../resources/volumes/catalog-var-ref/script | 2 +- .../volumes/change-name/out.plan.direct.json | 2 +- .../resources/volumes/change-name/script | 2 +- .../volumes/comment_out_of_band/script | 2 +- .../out.plan_create.direct.json | 2 +- .../volumes/remote-change-name/script | 2 +- .../resources/volumes/uppercase-name/script | 2 +- acceptance/bundle/run_as/pipelines/_script | 4 +- ....plan_t_service_principal_name.direct.json | 2 +- ...rvice_principal_name_different.direct.json | 2 +- .../out.plan_t_user_name.direct.json | 2 +- ...out.plan_t_user_name_different.direct.json | 2 +- ....plan_t_service_principal_name.direct.json | 2 +- ...rvice_principal_name_different.direct.json | 2 +- .../out.plan_t_user_name.direct.json | 2 +- ...out.plan_t_user_name_different.direct.json | 2 +- acceptance/bundle/select/basic/out.test.toml | 2 +- acceptance/bundle/select/basic/script | 4 +- acceptance/bundle/select/basic/test.toml | 4 + .../select/grants_permissions/out.test.toml | 2 +- .../select/grants_permissions/test.toml | 4 + .../bundle/state/feature_flags/out.test.toml | 2 +- .../bundle/state/feature_flags/output.txt | 8 +- .../resources.empty_features.json | 2 +- .../feature_flags/resources.with_feature.json | 2 +- acceptance/bundle/state/feature_flags/script | 4 +- .../bundle/state/feature_flags/test.toml | 3 + .../bundle/state/future_version/output.txt | 2 +- .../out.plan.direct.json | 2 +- .../state/permission_level_migration/script | 4 +- .../bundle/summary/modified_status/script | 8 +- .../out.plan_after_deploy_dev.direct.json | 2 +- .../out.plan_after_deploy_prod.direct.json | 2 +- .../classic/out.plan_dev.direct.json | 2 +- .../classic/out.plan_prod.direct.json | 2 +- .../templates/default-python/classic/script | 8 +- .../out.plan_dev.direct.json | 2 +- .../out.plan_prod.direct.json | 2 +- .../default-python/integration_classic/script | 4 +- .../out.plan_after_deploy_dev.direct.json | 2 +- .../out.plan_after_deploy_prod.direct.json | 2 +- .../serverless/out.plan_dev.direct.json | 2 +- .../serverless/out.plan_prod.direct.json | 2 +- .../default-python/serverless/script | 8 +- .../record-deployment-history/out.test.toml | 2 +- .../record-deployment-history/test.toml | 4 + acceptance/bundle/test.toml | 8 - bundle/bundle.go | 8 + bundle/config/deployment.go | 16 + bundle/config/experimental.go | 3 + bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- .../metadata/annotate_deployment_version.go | 65 ++ .../annotate_deployment_version_test.go | 50 ++ bundle/deployplan/plan.go | 21 +- bundle/direct/apply.go | 28 +- bundle/direct/bind.go | 22 +- bundle/direct/bundle_apply.go | 28 +- bundle/direct/bundle_plan.go | 59 +- bundle/direct/dstate/dms.go | 58 ++ bundle/direct/dstate/dms_test.go | 100 +++ bundle/direct/dstate/migrate.go | 20 - bundle/direct/dstate/state.go | 321 +++++++-- bundle/direct/dstate/state_test.go | 80 +-- bundle/direct/pkg.go | 8 + bundle/env/dms.go | 22 + bundle/env/dms_test.go | 41 ++ bundle/migrate/build_state.go | 2 +- bundle/phases/bind.go | 16 +- bundle/phases/deploy.go | 78 ++- bundle/phases/destroy.go | 48 +- bundle/phases/dms.go | 249 +++++++ bundle/phases/dms_test.go | 81 +++ bundle/statemgmt/direct_migration.go | 3 +- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 116 +++- libs/dms/client.go | 250 +++++++ libs/dms/client_test.go | 67 ++ libs/dms/fields.go | 59 ++ libs/dms/fields_test.go | 23 + libs/dms/metadata.go | 95 +++ libs/dms/metadata_test.go | 25 + libs/dms/operation.go | 139 ++++ libs/dms/operation_buffer.go | 197 ++++++ libs/dms/operation_buffer_test.go | 116 ++++ libs/dms/operation_test.go | 83 +++ libs/dms/resources.go | 41 ++ libs/testserver/bundledeployments.go | 615 ++++++++++++++++++ libs/testserver/fake_workspace.go | 11 + libs/testserver/handlers.go | 29 + libs/workspaceurls/urls.go | 17 + libs/workspaceurls/urls_test.go | 49 ++ 541 files changed, 6619 insertions(+), 748 deletions(-) create mode 100755 acceptance/bin/dms_resources.py delete mode 100755 acceptance/bin/nostamp create mode 100755 acceptance/bin/nostamp.py create mode 100644 acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/declined-deploy/out.requests.txt create mode 100644 acceptance/bundle/dms/declined-deploy/out.test.toml create mode 100644 acceptance/bundle/dms/declined-deploy/output.txt create mode 100644 acceptance/bundle/dms/declined-deploy/script create mode 100644 acceptance/bundle/dms/depends-on/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/depends-on/out.test.toml create mode 100644 acceptance/bundle/dms/depends-on/output.txt create mode 100644 acceptance/bundle/dms/depends-on/script create mode 100644 acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/deployment-metadata-change/out.test.toml create mode 100644 acceptance/bundle/dms/deployment-metadata-change/output.txt create mode 100644 acceptance/bundle/dms/deployment-metadata-change/script create mode 100644 acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/emptied-resource/out.test.toml create mode 100644 acceptance/bundle/dms/emptied-resource/output.txt create mode 100644 acceptance/bundle/dms/emptied-resource/script create mode 100644 acceptance/bundle/dms/existing-state/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/existing-state/out.test.toml create mode 100644 acceptance/bundle/dms/existing-state/output.txt create mode 100644 acceptance/bundle/dms/existing-state/script create mode 100644 acceptance/bundle/dms/existing-state/test.toml create mode 100644 acceptance/bundle/dms/failed-delete/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/failed-delete/out.test.toml create mode 100644 acceptance/bundle/dms/failed-delete/output.txt create mode 100644 acceptance/bundle/dms/failed-delete/script create mode 100644 acceptance/bundle/dms/failed-delete/test.toml create mode 100644 acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/failed-recreate/out.requests.txt create mode 100644 acceptance/bundle/dms/failed-recreate/out.test.toml create mode 100644 acceptance/bundle/dms/failed-recreate/output.txt create mode 100644 acceptance/bundle/dms/failed-recreate/script create mode 100644 acceptance/bundle/dms/failed-recreate/test.toml create mode 100644 acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/failed-update-permissions/out.test.toml create mode 100644 acceptance/bundle/dms/failed-update-permissions/output.txt create mode 100644 acceptance/bundle/dms/failed-update-permissions/script create mode 100644 acceptance/bundle/dms/failed-update-permissions/test.toml create mode 100644 acceptance/bundle/dms/failed-update/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/failed-update/out.test.toml create mode 100644 acceptance/bundle/dms/failed-update/output.txt create mode 100644 acceptance/bundle/dms/failed-update/script create mode 100644 acceptance/bundle/dms/failed-update/test.toml create mode 100644 acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/multiple-resources/out.test.toml create mode 100644 acceptance/bundle/dms/multiple-resources/output.txt create mode 100644 acceptance/bundle/dms/multiple-resources/script create mode 100644 acceptance/bundle/dms/no-drift/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/no-drift/out.test.toml create mode 100644 acceptance/bundle/dms/no-drift/output.txt create mode 100644 acceptance/bundle/dms/no-drift/script create mode 100644 acceptance/bundle/dms/no-drift/transform.py create mode 100644 acceptance/bundle/dms/no-resources/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/no-resources/out.test.toml create mode 100644 acceptance/bundle/dms/no-resources/output.txt create mode 100644 acceptance/bundle/dms/no-resources/script create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/out.requests.txt create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/out.test.toml create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/output.txt create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/script create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/test.toml create mode 100644 acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/operation-upload-fails/out.test.toml create mode 100644 acceptance/bundle/dms/operation-upload-fails/output.txt create mode 100644 acceptance/bundle/dms/operation-upload-fails/script create mode 100644 acceptance/bundle/dms/operation-upload-fails/test.toml create mode 100644 acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/operation-upload-message/out.test.toml create mode 100644 acceptance/bundle/dms/operation-upload-message/output.txt create mode 100644 acceptance/bundle/dms/operation-upload-message/script create mode 100644 acceptance/bundle/dms/operation-upload-message/test.toml create mode 100644 acceptance/bundle/dms/provenance/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/provenance/out.test.toml create mode 100644 acceptance/bundle/dms/provenance/output.txt create mode 100644 acceptance/bundle/dms/provenance/script create mode 100644 acceptance/bundle/dms/provenance/test.toml create mode 100644 acceptance/bundle/dms/readplan/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/readplan/out.test.toml create mode 100644 acceptance/bundle/dms/readplan/output.txt create mode 100644 acceptance/bundle/dms/readplan/script create mode 100644 acceptance/bundle/dms/readplan/transform.py create mode 100644 acceptance/bundle/dms/record-failure/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/record-failure/out.test.toml create mode 100644 acceptance/bundle/dms/record-failure/output.txt create mode 100644 acceptance/bundle/dms/record-failure/script create mode 100644 acceptance/bundle/dms/record-failure/test.toml create mode 100644 acceptance/bundle/dms/record/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/record/out.test.toml create mode 100644 acceptance/bundle/dms/record/output.txt create mode 100644 acceptance/bundle/dms/record/script create mode 100644 acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/serialized-plan/out.test.toml create mode 100644 acceptance/bundle/dms/serialized-plan/output.txt create mode 100644 acceptance/bundle/dms/serialized-plan/script create mode 100644 acceptance/bundle/dms/stale-plan/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/stale-plan/out.test.toml create mode 100644 acceptance/bundle/dms/stale-plan/output.txt create mode 100644 acceptance/bundle/dms/stale-plan/script create mode 100644 acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/successful-recreate/out.requests.txt create mode 100644 acceptance/bundle/dms/successful-recreate/out.test.toml create mode 100644 acceptance/bundle/dms/successful-recreate/output.txt create mode 100644 acceptance/bundle/dms/successful-recreate/script create mode 100644 acceptance/bundle/dms/summary/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/summary/out.test.toml create mode 100644 acceptance/bundle/dms/summary/output.txt create mode 100644 acceptance/bundle/dms/summary/script create mode 100644 acceptance/bundle/dms/test.toml create mode 100644 acceptance/bundle/dms/version-never-created/databricks.yml create mode 100644 acceptance/bundle/dms/version-never-created/out.test.toml create mode 100644 acceptance/bundle/dms/version-never-created/output.txt create mode 100644 acceptance/bundle/dms/version-never-created/script create mode 100644 acceptance/bundle/dms/version-never-created/test.toml create mode 100644 acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml create mode 100644 bundle/deploy/metadata/annotate_deployment_version.go create mode 100644 bundle/deploy/metadata/annotate_deployment_version_test.go create mode 100644 bundle/direct/dstate/dms.go create mode 100644 bundle/direct/dstate/dms_test.go create mode 100644 bundle/env/dms.go create mode 100644 bundle/env/dms_test.go create mode 100644 bundle/phases/dms.go create mode 100644 bundle/phases/dms_test.go create mode 100644 libs/dms/client.go create mode 100644 libs/dms/client_test.go create mode 100644 libs/dms/fields.go create mode 100644 libs/dms/fields_test.go create mode 100644 libs/dms/metadata.go create mode 100644 libs/dms/metadata_test.go create mode 100644 libs/dms/operation.go create mode 100644 libs/dms/operation_buffer.go create mode 100644 libs/dms/operation_buffer_test.go create mode 100644 libs/dms/operation_test.go create mode 100644 libs/dms/resources.go create mode 100644 libs/testserver/bundledeployments.go diff --git a/acceptance/bin/dms_resources.py b/acceptance/bin/dms_resources.py new file mode 100755 index 00000000000..25e469c5bad --- /dev/null +++ b/acceptance/bin/dms_resources.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +""" +Read resource ids and state from the deployment metadata service. + +While a bundle records deployment history the service owns the resource set, so the state file is +not where ids and state come from. The service is asked instead, which takes two lookups: the CLI +keeps no deployment id locally, and the id is the object id of the workspace node the service +registers under /resources.deployment.json (see libs/dms/resolve.go). +""" + +import functools +import glob +import json +import os +import posixpath +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(__file__)) +from print_state import get_state_file + +CLI = os.environ.get("CLI", "databricks") + +# Must match dms.DeploymentNodeName. +DEPLOYMENT_NODE_NAME = "resources.deployment.json" + + +def run_json(cmd, allow_failure=False): + """Run cmd and parse its stdout, or return None if it fails and allow_failure is set. stderr is + captured rather than inherited: these lookups are plumbing, and a CLI warning like "no files to + sync" would otherwise land in the test output.""" + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") + if result.returncode != 0: + if allow_failure: + return None + raise SystemExit(f"{cmd} failed with code {result.returncode}\n{result.stdout}{result.stderr}".strip()) + return json.loads(result.stdout) + + +def records_deployment_history(): + """Whether this run records deployment history, so the service is what to ask.""" + return os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true" + + +def get_remote_state_path(target): + """The bundle's remote state directory. + + Preferred source is the sync snapshot, because it needs no CLI call: re-running the config + load would need whatever --var and flags the test deployed with, which a helper cannot know. + A bundle with no files to sync writes no snapshot, so fall back to asking the CLI - those + bundles are the ones with nothing to parameterize.""" + target_dir = os.path.dirname(get_state_file(target, False)) + snapshots = glob.glob(f"{target_dir}/sync-snapshots/*.json") + if snapshots: + # One snapshot per remote path, so a test that moved its root leaves several: the newest + # is the one the last deploy used. + newest = max(snapshots, key=os.path.getmtime) + remote_path = json.loads(open(newest).read())["remote_path"] + # state and files are siblings under the bundle root. + return posixpath.join(posixpath.dirname(remote_path), "state") + + args = [CLI, "bundle", "validate", "--output", "json"] + if target: + args += ["-t", target] + return run_json(args)["workspace"]["state_path"] + + +@functools.cache +def get_resources(target): + """Map every recorded resource key ("jobs.foo") to its {"id", "state"}. + + Empty when the bundle has no deployment recorded yet. Cached because a lookup costs three + round trips and a script asks for one resource at a time. + """ + state_path = get_remote_state_path(target) + if not state_path: + return {} + + # No node means nothing has been recorded, the conclusion dms.resolveDeploymentID also draws + # from a 404 - the deployment is gone once the bundle is destroyed. + node = run_json([CLI, "workspace", "get-status", f"{state_path}/{DEPLOYMENT_NODE_NAME}"], allow_failure=True) + if not node or not node.get("object_id"): + return {} + deployment_id = node["object_id"] + + result = {} + # The service pages at 50 resources; the local fake returns everything at once. + page_token = None + while True: + url = f"/api/2.0/bundle/deployments/{deployment_id}/resources" + if page_token: + url += f"?page_token={page_token}" + listed = run_json([CLI, "api", "get", url]) + for resource in listed.get("resources") or []: + # The service stores state as the opaque envelope the CLI wrote (dstate.RecordedState), + # so unwrap it to the resource state itself. + envelope = json.loads(resource["state"]) if resource.get("state") else {} + result[resource["resource_key"]] = { + "id": resource.get("resource_id"), + "state": envelope.get("state") or {}, + "depends_on": envelope.get("depends_on") or [], + } + page_token = listed.get("next_page_token") + if not page_token: + return result diff --git a/acceptance/bin/nostamp b/acceptance/bin/nostamp deleted file mode 100755 index 1827400bece..00000000000 --- a/acceptance/bin/nostamp +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# Reads JSON on stdin, writes it back with the deployment stamp removed. -# -# Deployment history recording adds deployment_id and version_id to every job and -# pipeline. Acceptance tests compare output byte for byte, so those two extra fields -# would fail every test in the DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true run -# (see bundle/test.toml). -# Pipe a plan, a state dump, or a resource payload through this and the test asserts one -# golden file either way. Tests under bundle/dms assert the stamp itself and must not. -# -# Three passes, because the stamp shows up in three shapes: -# -# 1. Nested in a deployment block, as printed by `jobs get` / `pipelines get`: -# "deployment": {"deployment_id": "87..", "kind": "BUNDLE", -# "metadata_file_path": "/x", "version_id": "1"} -# -> both keys dropped, "kind" and "metadata_file_path" kept. -# -# 2. Flat in a plan's "changes", keyed by field path: -# "changes": {"deployment.version_id": {"action": "skip", ...}, "name": {...}} -# -> the stamp entries dropped, real changes kept. -# -# 3. A "changes" object that pass 2 emptied to {} - it existed only because of -# recording, so the key goes too. -# -# Two things it deliberately keeps: -# -# - A deployment_id anywhere else. Pass 1 requires "kind" and "metadata_file_path" as -# neighbours, a pair unique to the deployment block, so an unrelated field of the -# same name is untouched. -# - "version_id": "". A terraform state dump carries that for a job it never stamped, -# and it is the test's own expected output, so `.value == ""` keeps it. -# -# Arguments are passed to jq, for callers whose input is not formatted the way jq -# formats by default: a state dump is printed verbatim from disk, so it needs -# --indent 1 to come back out unchanged. -jq "$@" '((.. | objects | select(has("kind") and has("metadata_file_path"))) - |= with_entries(select((.key | IN("deployment_id", "version_id")) == false or .value == ""))) - | ((.. | objects | .changes? | objects) - |= with_entries(select(.key | IN("deployment.deployment_id", "deployment.version_id") | not))) - | del(.. | objects | select(.changes == {}) | .changes)' diff --git a/acceptance/bin/nostamp.py b/acceptance/bin/nostamp.py new file mode 100755 index 00000000000..a4306fe401a --- /dev/null +++ b/acceptance/bin/nostamp.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Read JSON on stdin, write it back with the DMS deployment stamp removed. + +Deployment history recording (DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true; see +bundle/test.toml) adds a deployment stamp to plans, states, and resource payloads. Pipe any of +those through this so an acceptance golden compares equal whether or not recording is on. Output +is a 2-space indent, keys in input order, <>& left unescaped, and integers at full precision. +Tests under bundle/dms assert the stamp itself and must not use it. + +Removed in three shapes plus the plan header: + 1. deployment_id/version_id nested in a deployment block, recognized by the + neighbouring "kind" and "metadata_file_path", which are kept; version_id + == "" is kept (a Terraform state dump carries that for an unstamped job). + 2. changes entries keyed "deployment.deployment_id" / "deployment.version_id". + 3. a changes object emptied by (2), or already empty. + 4. the recorded plan header fields deployment_id, next_version_id, last_version_id. +""" + +import argparse +import json +import sys + +_STAMP_KEYS = ("deployment_id", "version_id") +_CHANGE_KEYS = ("deployment.deployment_id", "deployment.version_id") +_PLAN_HEADER_KEYS = ("deployment_id", "next_version_id", "last_version_id") + + +def scrub(node): + if isinstance(node, dict): + # 1. A deployment block is the only object carrying both of these. + if "kind" in node and "metadata_file_path" in node: + for k in _STAMP_KEYS: + if node.get(k, "") != "": + node.pop(k, None) + out = {} + for k, v in node.items(): + if k == "changes" and isinstance(v, dict): + v = {ck: scrub(cv) for ck, cv in v.items() if ck not in _CHANGE_KEYS} + if not v: + continue # 3. drop a changes object left (or already) empty + out[k] = v + else: + out[k] = scrub(v) + return out + if isinstance(node, list): + return [scrub(x) for x in node] + return node + + +def render(data, indent): + data = scrub(data) + if isinstance(data, dict): + for k in _PLAN_HEADER_KEYS: + data.pop(k, None) # 4. plan header, present only at the root + return json.dumps(data, indent=indent, ensure_ascii=False, separators=(",", ": ")) + + +def main(): + parser = argparse.ArgumentParser() + # A state dump is printed with a single-space indent; plans use two. + parser.add_argument("--indent", type=int, default=2) + args = parser.parse_args() + + # Input is one JSON value (a plan or state dump) or a whitespace-separated stream of them (each + # request emitted by a print_requests filter) - jq accepted both here, so this must too. + text = sys.stdin.read() + decoder = json.JSONDecoder() + idx, n = 0, len(text) + while idx < n: + while idx < n and text[idx].isspace(): + idx += 1 + if idx >= n: + break + data, idx = decoder.raw_decode(text, idx) + sys.stdout.write(render(data, args.indent) + "\n") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 7e82f7b3d16..2959a76f81b 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -8,9 +8,14 @@ import argparse import glob +import json import os +def records_deployment_history(): + return os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true" + + def print_file(filename): data = open(filename).read() print(data, end="") @@ -53,6 +58,26 @@ def get_state_file(target, backup): return filtered[0] if filtered else result[0] +def print_recorded_state(filename, target): + """Print the state file with its resources filled in from the deployment metadata service. + + While recording, the file itself carries only the header - the service holds the resources - so + printing it raw would show an empty state and differ from the same test's non-recording run. + """ + # Imported here rather than at module level: dms_resources reads get_state_file from this module. + from dms_resources import get_resources + + data = json.loads(open(filename).read()) + state = {} + for key, value in sorted(get_resources(target).items()): + entry = {"__id__": value["id"], "state": value["state"]} + if value["depends_on"]: + entry["depends_on"] = value["depends_on"] + state[f"resources.{key}"] = entry + data["state"] = state + print(json.dumps(data, indent=1)) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("-t", "--target") @@ -60,7 +85,11 @@ def main(): args = parser.parse_args() for filename in get_state_files(args.target, args.backup): - if os.path.exists(filename): + if not os.path.exists(filename): + continue + if filename.endswith("resources.json") and records_deployment_history(): + print_recorded_state(filename, args.target) + else: print_file(filename) diff --git a/acceptance/bin/read_id.py b/acceptance/bin/read_id.py index 87cd2954bdf..06e7b69281e 100755 --- a/acceptance/bin/read_id.py +++ b/acceptance/bin/read_id.py @@ -15,6 +15,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from add_repl import add_repl +from dms_resources import get_resources, records_deployment_history from print_state import get_state_file @@ -34,6 +35,15 @@ def get_id_terraform(filename, name): print(f"Cannot find resource with {name=}. Available: {available}", file=sys.stderr) +def get_id_recorded(target, name): + resources = get_resources(target) + for key, value in resources.items(): + if key.split(".")[1] == name: + return value["id"] + + print(f"Cannot find recorded resource with {name=}. Available: {list(resources)}", file=sys.stderr) + + def get_id_direct(filename, name): raw = open(filename).read() data = json.loads(raw) @@ -53,11 +63,14 @@ def main(): parser.add_argument("name") args = parser.parse_args() - filename = get_state_file(args.target, args.backup) - if filename.endswith(".tfstate"): - id = get_id_terraform(filename, args.name) + if records_deployment_history(): + id = get_id_recorded(args.target, args.name) else: - id = get_id_direct(filename, args.name) + filename = get_state_file(args.target, args.backup) + if filename.endswith(".tfstate"): + id = get_id_terraform(filename, args.name) + else: + id = get_id_direct(filename, args.name) if id: print(id) diff --git a/acceptance/bin/read_state.py b/acceptance/bin/read_state.py index 0166bf9abbb..4d9bf84186e 100755 --- a/acceptance/bin/read_state.py +++ b/acceptance/bin/read_state.py @@ -9,6 +9,9 @@ import os import sys +sys.path.insert(0, os.path.dirname(__file__)) +from dms_resources import get_resources, records_deployment_history + def print_resource_terraform(group, name, *attrs): resource_type = "databricks_" + group[:-1] @@ -50,7 +53,21 @@ def print_resource_direct(group, name, *attrs): print(group, name, " ".join(values)) -if os.environ.get("DATABRICKS_BUNDLE_ENGINE", "").startswith("direct"): +def print_resource_recorded(group, name, *attrs): + result = get_resources(None).get(f"{group}.{name}") + if result is None: + print(f"State not found for {group}.{name}") + return + + state = dict(result["state"]) + state.setdefault("id", result["id"]) + values = [f"{x}={state.get(x)!r}" for x in attrs] + print(group, name, " ".join(values)) + + +if records_deployment_history(): + print_resource_recorded(*sys.argv[1:]) +elif os.environ.get("DATABRICKS_BUNDLE_ENGINE", "").startswith("direct"): print_resource_direct(*sys.argv[1:]) else: print_resource_terraform(*sys.argv[1:]) diff --git a/acceptance/bin/replace_ids.py b/acceptance/bin/replace_ids.py index e0ebf1cd0e6..2f5d8165032 100755 --- a/acceptance/bin/replace_ids.py +++ b/acceptance/bin/replace_ids.py @@ -10,6 +10,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from add_repl import add_repl +from dms_resources import get_resources, records_deployment_history from print_state import get_state_file @@ -26,6 +27,12 @@ def iter_ids_terraform(filename): yield r_name, id +def iter_ids_recorded(target): + for key, value in get_resources(target).items(): + if value["id"]: + yield key.split(".")[1], value["id"] + + def iter_ids_direct(filename): raw = open(filename).read() data = json.loads(raw) @@ -44,11 +51,14 @@ def main(): parser.add_argument("--backup", action="store_true") args = parser.parse_args() - filename = get_state_file(args.target, args.backup) - if filename.endswith(".tfstate"): - it = iter_ids_terraform(filename) + if records_deployment_history(): + it = iter_ids_recorded(args.target) else: - it = iter_ids_direct(filename) + filename = get_state_file(args.target, args.backup) + if filename.endswith(".tfstate"): + it = iter_ids_terraform(filename) + else: + it = iter_ids_direct(filename) for name, id in it: add_repl(id, name.upper() + "_ID") diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json index c881a8a76b2..804ba480a90 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.test_job": { diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json index b8d5ada22e7..ca1198e6750 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index ddb6fcd55ec..68d328cc67d 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -4,9 +4,9 @@ cp -r $TESTDIR/../whl_explicit/my_test_code . mkdir prebuilt cp -r $TESTDIR/../whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl prebuilt -trace $CLI bundle validate -o json | nostamp | jq .artifacts +trace $CLI bundle validate -o json | nostamp.py | jq .artifacts -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "There are 2 original wheels and 2 patched ones" @@ -25,7 +25,7 @@ rm out.requests.txt title "Updating the local wheel and deploying again\n" touch my_test_code/src/new_module.py -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Verify contents, it should now have new_module.py" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index 623849d3bb5..21bc2c9addc 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index f82c5d7eccf..d62de02530b 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in environments section in /jobs/create" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script index ccc0d9ed7f8..59268fdeb0c 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting the environments dependency to keep its [train] extras suffix" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py title "Expecting 1 wheel to be uploaded under its bare filename (no extras)" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/bundle_tag/id/script b/acceptance/bundle/bundle_tag/id/script index 759cc89eba1..200c5a7d2e6 100644 --- a/acceptance/bundle/bundle_tag/id/script +++ b/acceptance/bundle/bundle_tag/id/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -trace $CLI bundle validate -o json | nostamp | jq .resources +trace $CLI bundle validate -o json | nostamp.py | jq .resources trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/bundle_tag/url/script b/acceptance/bundle/bundle_tag/url/script index 759cc89eba1..200c5a7d2e6 100644 --- a/acceptance/bundle/bundle_tag/url/script +++ b/acceptance/bundle/bundle_tag/url/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -trace $CLI bundle validate -o json | nostamp | jq .resources +trace $CLI bundle validate -o json | nostamp.py | jq .resources trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/deploy/experimental-python/script b/acceptance/bundle/deploy/experimental-python/script index 09b90b28e2d..09bf4b14ef6 100644 --- a/acceptance/bundle/deploy/experimental-python/script +++ b/acceptance/bundle/deploy/experimental-python/script @@ -1,3 +1,3 @@ trace uv run --quiet --with $DATABRICKS_BUNDLES_WHEEL -- $CLI bundle deploy -trace $CLI jobs list --output json | nostamp +trace $CLI jobs list --output json | nostamp.py diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/script index 85e9a908d1d..83136e53447 100644 --- a/acceptance/bundle/deploy/immutable/script +++ b/acceptance/bundle/deploy/immutable/script @@ -5,7 +5,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle validate -trace $CLI bundle plan -o json | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' trace $CLI bundle deploy diff --git a/acceptance/bundle/deploy/python-notebook/script b/acceptance/bundle/deploy/python-notebook/script index 21318c58b66..8b12727b59e 100644 --- a/acceptance/bundle/deploy/python-notebook/script +++ b/acceptance/bundle/deploy/python-notebook/script @@ -1,3 +1,3 @@ trace $CLI bundle deploy -trace $CLI jobs list --output json | nostamp +trace $CLI jobs list --output json | nostamp.py diff --git a/acceptance/bundle/deploy/readplan/basic/out.plan_create.json b/acceptance/bundle/deploy/readplan/basic/out.plan_create.json index 4e8f66bafe4..c093dad07ab 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.plan_create.json +++ b/acceptance/bundle/deploy/readplan/basic/out.plan_create.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json b/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json index a6b4e835bb9..542dc55d6ef 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json +++ b/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/deploy/readplan/basic/script b/acceptance/bundle/deploy/readplan/basic/script index 89bf2487b86..e01d5c5f3e7 100644 --- a/acceptance/bundle/deploy/readplan/basic/script +++ b/acceptance/bundle/deploy/readplan/basic/script @@ -1,5 +1,5 @@ # Generate initial plan -trace $CLI bundle plan -o json > out.plan_create.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.json trace print_requests.py --get //jobs | contains.py "!GET" "!POST" # First deploy to create the resource @@ -7,7 +7,7 @@ trace $CLI bundle deploy --plan out.plan_create.json trace print_requests.py --get //jobs | contains.py "!GET" "POST" # Generate a plan (should show "skip" since nothing changed) -trace $CLI bundle plan -o json > out.plan_skip.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_skip.json trace print_requests.py --get //jobs | contains.py "GET" trace $CLI bundle deploy --plan out.plan_skip.json diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json b/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json index 932edf74f84..bd9561c9ec8 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "0.0.0-test", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json index 42cce15cbfc..80310520edb 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "0.0.0-test", "lineage": "different-lineage", "serial": 0, diff --git a/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt b/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt index 387aac2335c..3ec4497a23e 100644 --- a/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt +++ b/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt @@ -1,4 +1,4 @@ -Error: plan version mismatch: plan has version 999 (generated with CLI "0.0.0"), but current version is 2 +Error: plan version mismatch: plan has version 999 (generated with CLI "0.0.0"), but current version is 3 Exit code: 1 diff --git a/acceptance/bundle/deploy/readplan/test.toml b/acceptance/bundle/deploy/readplan/test.toml index e6ef3fcdcfb..576ff1ab001 100644 --- a/acceptance/bundle/deploy/readplan/test.toml +++ b/acceptance/bundle/deploy/readplan/test.toml @@ -1,3 +1,3 @@ -# Saved plans don't carry the deployment stamp. Applying one plans an update on the next run. -# See dms_no_readplan in acceptance/bundle/test.toml. +# Dumps post-deploy state/plan, which carry the DMS deployment stamp under recording (applied at +# deploy, not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json b/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json index fa91f3d0bf9..84df30af6d8 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json +++ b/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/deploy/readplan/unknown-field/script b/acceptance/bundle/deploy/readplan/unknown-field/script index 6032a3ae34e..ea17245ace1 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/script +++ b/acceptance/bundle/deploy/readplan/unknown-field/script @@ -1,5 +1,5 @@ # Generate a plan -trace $CLI bundle plan -o json > out.plan.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.json # Modify the plan to rename "action" field to "action_renamed" # This will create an unknown field that the strict decoder should reject diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json index da6e6cae939..09a1645c786 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/script b/acceptance/bundle/deploy/wal/chain-3-jobs/script index c41e8e7f995..5cc77faa786 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/script +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/script @@ -7,7 +7,7 @@ trace errcode $CLI bundle deploy echo "" echo "=== WAL content after crash ===" -jq -S . .databricks/bundle/default/resources.json.wal 2>/dev/null | nostamp || echo "No WAL file" +jq -S . .databricks/bundle/default/resources.json.wal 2>/dev/null | nostamp.py || echo "No WAL file" echo "" echo "=== Number of jobs saved in WAL ===" diff --git a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml index b0851b242b8..9a7cc99433c 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml +++ b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.COMMAND = ["plan", "deploy --force-lock", "summary"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/crash-after-create/script b/acceptance/bundle/deploy/wal/crash-after-create/script index 5647c83e387..22f7587e52e 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/script +++ b/acceptance/bundle/deploy/wal/crash-after-create/script @@ -8,7 +8,7 @@ trace errcode $CLI bundle deploy trace assert_exists.py .databricks/bundle/default/resources.json.wal trace assert_not_exists.py .databricks/bundle/default/resources.json -trace cat .databricks/bundle/default/resources.json.wal | jq | nostamp +trace cat .databricks/bundle/default/resources.json.wal | jq | nostamp.py title "Any other command recovers state" $CLI bundle $COMMAND &> LOG.COMMAND.txt diff --git a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/header-only-wal/output.txt b/acceptance/bundle/deploy/wal/header-only-wal/output.txt index 807fa926044..da8c1bdc1f3 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/output.txt +++ b/acceptance/bundle/deploy/wal/header-only-wal/output.txt @@ -28,7 +28,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> errcode assert_not_exists.py .databricks/bundle/default/resources.json.wal ->>> errcode cat .databricks/bundle/default/resources.json +>>> errcode print_state.py { "serial": 1, "state_keys": [ diff --git a/acceptance/bundle/deploy/wal/header-only-wal/script b/acceptance/bundle/deploy/wal/header-only-wal/script index 89da7e1a277..557fc7d4f47 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/script +++ b/acceptance/bundle/deploy/wal/header-only-wal/script @@ -19,4 +19,4 @@ title "Third deploy (must recover and succeed, not blocked by the leftover WAL)" trace errcode $CLI bundle deploy --force-lock trace errcode assert_not_exists.py .databricks/bundle/default/resources.json.wal -trace errcode cat .databricks/bundle/default/resources.json | jq -S '{serial: .serial, state_keys: (.state | keys)}' +trace errcode print_state.py | jq -S '{serial: .serial, state_keys: (.state | keys)}' diff --git a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml index dc89c55cf90..2845c70cfef 100644 --- a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.COMMAND = ["deploy", "plan", "summary"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/test.toml b/acceptance/bundle/deploy/wal/test.toml index 5218ccbbd10..e4be659d2e1 100644 --- a/acceptance/bundle/deploy/wal/test.toml +++ b/acceptance/bundle/deploy/wal/test.toml @@ -1,10 +1,16 @@ # WAL (Write-Ahead Log) tests verify crash recovery during bundle deployment. # These tests simulate process crashes using KillCaller and verify state recovery. # Only runs with direct engine since WAL is a direct-engine feature. +# +# Not run under deployment history recording, and not planned to be: WAL recovery is a +# non-recording concept - under recording the service is the source of truth and a leftover WAL is +# discarded, not replayed (see dstate.Open; the DMS side is covered by bundle/dms) - so there is +# nothing here a DMS run would exercise. Env.DATABRICKS_CLI_TEST_PID = "1" EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] [[Repls]] Old = 'script: line \d+:\s+\d+ Killed(: 9)?\s+"\$@"' diff --git a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml +++ b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deployment/bind/experiment/script b/acceptance/bundle/deployment/bind/experiment/script index 70cf1933811..e31a17f7eaa 100644 --- a/acceptance/bundle/deployment/bind/experiment/script +++ b/acceptance/bundle/deployment/bind/experiment/script @@ -21,7 +21,7 @@ trace $CLI bundle deploy --force-lock --auto-approve # Use --output json to verify the change details including skip reason. # Write per-engine output since direct includes change details that terraform doesn't. # Filter to only the name change (tags/artifact_location may differ between local and cloud). -$CLI bundle plan --output json | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment.name, lifecycle_stage: .experiment.lifecycle_stage}' @@ -29,7 +29,7 @@ trace $CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment. # Plan should still show no changes because the diff is suppressed. EXPERIMENT_NAME="//Users/${CURRENT_USER_NAME}/test-experiment$UNIQUE_NAME" envsubst < databricks.yml.tmpl > databricks.yml -$CLI bundle plan --output json | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deployment unbind experiment1 diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/script b/acceptance/bundle/destroy/jobs-and-pipeline/script index b02d93bcecb..2391f9d44e0 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/script +++ b/acceptance/bundle/destroy/jobs-and-pipeline/script @@ -45,7 +45,7 @@ trace $CLI workspace get-status "${DEPLOYMENT_PATH}" | jq '{path, object_type}' title "Assert the pipeline is created" PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.bar.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp | jq "{spec}" +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py | jq "{spec}" title "Assert the job is created:\n" JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.foo.id') diff --git a/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl b/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl new file mode 100644 index 00000000000..e909afaab5c --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: dms-declined-deploy-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_declined_deploy_schema_$UNIQUE_NAME + catalog_name: main diff --git a/acceptance/bundle/dms/declined-deploy/out.requests.txt b/acceptance/bundle/dms/declined-deploy/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/declined-deploy/out.test.toml b/acceptance/bundle/dms/declined-deploy/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt new file mode 100644 index 00000000000..d09e8d18743 --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -0,0 +1,88 @@ + +=== Deploy a schema, so the deployment and its first version exist +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-declined-deploy-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== A destructive change without --auto-approve is declined: this console cannot prompt +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + +Files: 4 uploaded, 0 deleted + +=== Nothing was recorded for the declined deploy - no version, so none to abort +>>> print_requests.py --dms //api/2.0/bundle + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script new file mode 100644 index 00000000000..e67ff786b19 --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/script @@ -0,0 +1,22 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy a schema, so the deployment and its first version exist" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "A destructive change without --auto-approve is declined: this console cannot prompt" +# Changing the catalog recreates the schema, which needs approval. +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +trace musterr $CLI bundle deploy + +title "Nothing was recorded for the declined deploy - no version, so none to abort" +# The version number it would have used is left for the next deploy to take, so the +# history has no entry that reads like a deploy which failed or did nothing. +trace print_requests.py --dms //api/2.0/bundle + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/depends-on/databricks.yml.tmpl b/acceptance/bundle/dms/depends-on/databricks.yml.tmpl new file mode 100644 index 00000000000..1a60d215ecc --- /dev/null +++ b/acceptance/bundle/dms/depends-on/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: dms-depends-on-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + parent: + name: parent + child: + name: child + description: depends on ${resources.jobs.parent.id} diff --git a/acceptance/bundle/dms/depends-on/out.test.toml b/acceptance/bundle/dms/depends-on/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/depends-on/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt new file mode 100644 index 00000000000..4807497d63c --- /dev/null +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -0,0 +1,47 @@ + +=== Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/files... +Created jobs.child +Created jobs.parent +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //versions/1/operations +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.parent", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.child", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.child + delete resources.jobs.parent + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script new file mode 100644 index 00000000000..7f5576ce72b --- /dev/null +++ b/acceptance/bundle/dms/depends-on/script @@ -0,0 +1,8 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" +trace $CLI bundle deploy +trace print_requests.py --dms //versions/1/operations + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl b/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl new file mode 100644 index 00000000000..07f434b2e7a --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: dms-metadata-before-$UNIQUE_NAME +experimental: + record_deployment_history: true + +workspace: + # Pinned, so renaming the bundle or switching target below keeps the same deployment: the + # deployment is the workspace node under state_path, and a different path would be a + # different deployment. + state_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/dms-metadata-pinned-$UNIQUE_NAME/state + +resources: + schemas: + foo: + name: dms_metadata_schema_$UNIQUE_NAME + catalog_name: main + +targets: + dev: + default: true + prod: + mode: production diff --git a/acceptance/bundle/dms/deployment-metadata-change/out.test.toml b/acceptance/bundle/dms/deployment-metadata-change/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt new file mode 100644 index 00000000000..0b0798eee7f --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -0,0 +1,144 @@ + +=== Deploy: the deployment is created carrying this bundle's metadata +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-metadata-before-[UNIQUE_NAME]/dev/files... +Created schemas.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-metadata-before-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-pinned-[UNIQUE_NAME]/state", + "target_name": "dev", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-before-[UNIQUE_NAME]/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-before-[UNIQUE_NAME]/dev" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_metadata_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_metadata_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed +>>> update_file.py databricks.yml dms-metadata-before dms-metadata-after + +>>> [CLI] bundle deploy -t prod +Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed + +A common practice is to use a username or principal name in this path, i.e. use + + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod/files... +Files: 5 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]", + "q": { + "update_mask": "display_name,target_name,deployment_mode,workspace_info" + }, + "body": { + "deployment_mode": "DEPLOYMENT_MODE_PRODUCTION", + "display_name": "dms-metadata-after-[UNIQUE_NAME]", + "target_name": "prod", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Deploy again without touching anything: the record already says this, so no deployment update +>>> [CLI] bundle deploy -t prod +Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed + +A common practice is to use a username or principal name in this path, i.e. use + + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "2" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script new file mode 100644 index 00000000000..03f0c593999 --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -0,0 +1,14 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy: the deployment is created carrying this bundle's metadata" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed" +trace update_file.py databricks.yml dms-metadata-before dms-metadata-after +trace $CLI bundle deploy -t prod +trace print_requests.py --dms //api/2.0/bundle + +title "Deploy again without touching anything: the record already says this, so no deployment update" +trace $CLI bundle deploy -t prod +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl b/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl new file mode 100644 index 00000000000..b8c21a3cc96 --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-emptied-resource-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_emptied_resource_$UNIQUE_NAME + catalog_name: main + grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}] diff --git a/acceptance/bundle/dms/emptied-resource/out.test.toml b/acceptance/bundle/dms/emptied-resource/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt new file mode 100644 index 00000000000..483b3d8249d --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -0,0 +1,142 @@ + +=== Deploy a schema with one grant: the grants node is recorded with its state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files... +Created schemas.foo +Created schemas.foo.grants +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-emptied-resource-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + }, + { + "resource_key": "schemas.foo.grants", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_emptied_resource_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_emptied_resource_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo.grants", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "schema/main.dms_emptied_resource_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_emptied_resource_[UNIQUE_NAME]\",\"__embed__\":[{\"principal\":\"deco-test-user@databricks.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Revoke the grant, so the grants node empties out +>>> update_file.py databricks.yml grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}] grants: [] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files... +Updated schemas.foo.grants +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo.grants", + "action_type": "OPERATION_ACTION_TYPE_UPDATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo.grants", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Plan again: reading state back from the service works and reports no work +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script new file mode 100644 index 00000000000..b38be294ce1 --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy a schema with one grant: the grants node is recorded with its state" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "Revoke the grant, so the grants node empties out" +trace update_file.py databricks.yml 'grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}]' 'grants: []' +trace $CLI bundle deploy + +# The action type stays update: it is staged from the plan, before apply knows the state +# comes back empty. What drops the resource from the deployment is the update naming state +# with no value. +trace print_requests.py --dms //api/2.0/bundle + +title "Plan again: reading state back from the service works and reports no work" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/existing-state/databricks.yml.tmpl b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl new file mode 100644 index 00000000000..29addd6433e --- /dev/null +++ b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: dms-existing-state-$UNIQUE_NAME + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/existing-state/out.test.toml b/acceptance/bundle/dms/existing-state/out.test.toml new file mode 100644 index 00000000000..d73c45e3119 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt new file mode 100644 index 00000000000..9a7439eef8a --- /dev/null +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -0,0 +1,78 @@ + +=== Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Created jobs.one +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --oneline + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 2, + "features": null +} + +=== Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + + +>>> print_requests.py --dms //api/2.0/bundle --oneline + +=== The check is against the remote state, not the local cache: wiping .databricks still refuses, because the deploy pulls the state back from the workspace +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + + +=== A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + + +=== Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.one + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Created jobs.one +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.one", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +>>> find.py resources[.]json[.]wal --expect 0 + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 2, + "features": { + "record_deployment_history": {} + } +} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script new file mode 100644 index 00000000000..e0784bb406b --- /dev/null +++ b/acceptance/bundle/dms/existing-state/script @@ -0,0 +1,27 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --oneline +trace jq '{state_version, features}' .databricks/bundle/default/resources.json + +title "Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time" +musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --oneline + +title "The check is against the remote state, not the local cache: wiping .databricks still refuses, because the deploy pulls the state back from the workspace" +rm -rf .databricks +musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy + +title "A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands" +mkdir -p .databricks/bundle/default +printf '{"lineage":"stale","serial":99,"state_version":2}\n' > .databricks/bundle/default/resources.json.wal +musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy + +title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed" +trace $CLI bundle destroy --auto-approve +printf '{"lineage":"stale","serial":99,"state_version":2}\n' > .databricks/bundle/default/resources.json.wal +trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --oneline +trace find.py 'resources[.]json[.]wal' --expect 0 +trace jq '{state_version, features}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/existing-state/test.toml b/acceptance/bundle/dms/existing-state/test.toml new file mode 100644 index 00000000000..bc95f7e86c4 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/test.toml @@ -0,0 +1,3 @@ +# This test deploys without recording first, then turns it on per-command, so it opts out of +# the parent's DMS=true and drives DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY inline instead. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl b/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl new file mode 100644 index 00000000000..abdb86103ff --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-failed-delete-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + stuck: + name: stuck diff --git a/acceptance/bundle/dms/failed-delete/out.test.toml b/acceptance/bundle/dms/failed-delete/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt new file mode 100644 index 00000000000..701b2051c14 --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -0,0 +1,149 @@ + +=== Deploy: the job is recorded +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/files... +Created jobs.stuck +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-failed-delete-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "jobs.stuck", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.stuck", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"stuck\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone +>>> fault.py POST /api/2.2/jobs/delete 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.stuck + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default + +Error: cannot delete resources.jobs.stuck: deleting id=[NUMID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/delete +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "jobs.stuck", + "action_type": "OPERATION_ACTION_TYPE_DELETE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.stuck", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "deleting id=[NUMID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"stuck\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.stuck": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "stuck", + "queue": { + "enabled": true + } + } + } + } +} diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script new file mode 100644 index 00000000000..fff56460c21 --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/script @@ -0,0 +1,16 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy: the job is recorded" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone" +# The state entry survives too, so the next deploy still knows about the resource. +trace fault.py "POST /api/2.2/jobs/delete" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle destroy --auto-approve +trace print_requests.py --dms //api/2.0/bundle +trace print_state.py + +# print_state.py asks the service for the resources while recording, so drain those reads instead +# of leaving them behind for the harness to diff. +print_requests.py //api/2.0 > /dev/null diff --git a/acceptance/bundle/dms/failed-delete/test.toml b/acceptance/bundle/dms/failed-delete/test.toml new file mode 100644 index 00000000000..1a22ff04ef4 --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/test.toml @@ -0,0 +1,3 @@ +# Local only: this case needs an injected failure - no job delete that the Jobs API refuses - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false diff --git a/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl b/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..a2937bf9a41 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-failed-recreate-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_failed_recreate_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/failed-recreate/out.requests.txt b/acceptance/bundle/dms/failed-recreate/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/failed-recreate/out.test.toml b/acceptance/bundle/dms/failed-recreate/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt new file mode 100644 index 00000000000..718b8108771 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -0,0 +1,87 @@ + +=== Deploy the schema, so there is a resource to recreate +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> fault.py POST /api/2.1/unity-catalog/schemas 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Error: cannot recreate resources.schemas.foo: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/schemas +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Files: 4 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_PENDING" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "sequence_id": "1", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +=== The resource is not listed: state is what projects a resource, and the failed recreate left none +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{} + +=== Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists +>>> [CLI] bundle plan +create schemas.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script new file mode 100644 index 00000000000..e2d4a68f508 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/script @@ -0,0 +1,32 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy the schema, so there is a resource to recreate" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy" +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +# Fail only the create that follows the delete; the delete itself still goes through. +trace fault.py "POST /api/2.1/unity-catalog/schemas" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy --auto-approve +# Not sorted: the point of this test is the order the operations are recorded in. +trace print_requests.py --dms //api/2.0/bundle + +title "The resource is not listed: state is what projects a resource, and the failed recreate left none" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-recreate/test.toml b/acceptance/bundle/dms/failed-recreate/test.toml new file mode 100644 index 00000000000..8a850ba07c5 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/test.toml @@ -0,0 +1,5 @@ +# Local only: the failure is injected, and fault.py registers its rule on the fake, which +# a cloud run does not have. Simulating it for real instead - a catalog or principal that +# does not exist - does not work either: the fake accepts both, so the case would stop +# failing locally. Making it portable needs the fake to model that existence. +Cloud = false diff --git a/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl b/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl new file mode 100644 index 00000000000..622598450cd --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: dms-failed-update-permissions-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + permissions: + - level: CAN_VIEW + user_name: viewer@example.com diff --git a/acceptance/bundle/dms/failed-update-permissions/out.test.toml b/acceptance/bundle/dms/failed-update-permissions/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt new file mode 100644 index 00000000000..080f4b6c8be --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -0,0 +1,29 @@ + +=== Deploy the job and its permissions +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/files... +Created jobs.foo +Created jobs.foo.permissions +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not +>>> update_file.py databricks.yml CAN_VIEW CAN_MANAGE + +>>> fault.py PUT /api/2.0/permissions/jobs/* 403 0 1 PERMISSION_DENIED + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/files... +Error: cannot update resources.jobs.foo.permissions: updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED) + +Endpoint: PUT [DATABRICKS_URL]/api/2.0/permissions/jobs/[NUMID] +HTTP Status: 403 Forbidden +API error_code: PERMISSION_DENIED +API message: Fault injected by test. + +Files: 3 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo.permissions", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED)", "resource_id": "/jobs/[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", "status": "OPERATION_STATUS_FAILED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}} diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script new file mode 100644 index 00000000000..56141a30f4c --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -0,0 +1,12 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy the job and its permissions" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not" +trace update_file.py databricks.yml "CAN_VIEW" "CAN_MANAGE" +# A 403 is what the API returns when the caller cannot change a job's owner. +trace fault.py "PUT /api/2.0/permissions/jobs/*" 403 0 1 PERMISSION_DENIED +trace musterr $CLI bundle deploy --auto-approve +trace print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/failed-update-permissions/test.toml b/acceptance/bundle/dms/failed-update-permissions/test.toml new file mode 100644 index 00000000000..8a850ba07c5 --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/test.toml @@ -0,0 +1,5 @@ +# Local only: the failure is injected, and fault.py registers its rule on the fake, which +# a cloud run does not have. Simulating it for real instead - a catalog or principal that +# does not exist - does not work either: the fake accepts both, so the case would stop +# failing locally. Making it portable needs the fake to model that existence. +Cloud = false diff --git a/acceptance/bundle/dms/failed-update/databricks.yml.tmpl b/acceptance/bundle/dms/failed-update/databricks.yml.tmpl new file mode 100644 index 00000000000..87cfcff9f36 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-failed-update-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_failed_update_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/failed-update/out.test.toml b/acceptance/bundle/dms/failed-update/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt new file mode 100644 index 00000000000..125c16608b0 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -0,0 +1,86 @@ + +=== Deploy the schema, so there is an existing resource to update +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it +>>> update_file.py databricks.yml comment: v1 comment: v2 + +>>> fault.py PATCH /api/2.1/unity-catalog/schemas/* 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files... +Error: cannot update resources.schemas.foo: updating id=main.dms_failed_update_schema_[UNIQUE_NAME]: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.1/unity-catalog/schemas/main.dms_failed_update_schema_[UNIQUE_NAME] +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Files: 3 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_UPDATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "updating id=main.dms_failed_update_schema_[UNIQUE_NAME]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +=== The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{ + "resources": [ + { + "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", + "last_version_id": "2", + "name": "deployments/[NUMID]/resources/schemas.foo", + "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "resource_key": "schemas.foo", + "resource_type": "", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}" + } + ] +} + +=== Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys +>>> [CLI] bundle plan +update schemas.foo + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script new file mode 100644 index 00000000000..91640bd7631 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy the schema, so there is an existing resource to update" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it" +trace update_file.py databricks.yml "comment: v1" "comment: v2" +# Fail the update call itself, so the deploy never writes state for the schema. +trace fault.py "PATCH /api/2.1/unity-catalog/schemas/*" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy --auto-approve +# Not sorted: the point of this test is the order the operations are recorded in. +trace print_requests.py --dms //api/2.0/bundle + +title "The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-update/test.toml b/acceptance/bundle/dms/failed-update/test.toml new file mode 100644 index 00000000000..2b4eeb1bb04 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/test.toml @@ -0,0 +1,3 @@ +# Local only: this case needs an injected failure - no schema update that Unity Catalog rejects outright - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false diff --git a/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl b/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl new file mode 100644 index 00000000000..faa6ef5e63c --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: dms-multiple-resources-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five diff --git a/acceptance/bundle/dms/multiple-resources/out.test.toml b/acceptance/bundle/dms/multiple-resources/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt new file mode 100644 index 00000000000..ecc2092225a --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -0,0 +1,28 @@ + +=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it. +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... +Created jobs.five +Created jobs.four +Created jobs.one +Created jobs.three +Created jobs.two +Files: 5 uploaded, 0 deleted +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //versions/1/operations --sort --del-body state --oneline +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.five", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.four", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.three", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.two", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} + +=== Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 5 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script new file mode 100644 index 00000000000..584423a4d5e --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/script @@ -0,0 +1,9 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it." +trace $CLI bundle deploy +trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline + +title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/no-drift/databricks.yml.tmpl b/acceptance/bundle/dms/no-drift/databricks.yml.tmpl new file mode 100644 index 00000000000..e9892fa79a5 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/databricks.yml.tmpl @@ -0,0 +1,20 @@ +bundle: + name: dms-no-drift-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + pipelines: + bar: + # Unique: the Pipelines API refuses a duplicate name, so two overlapping runs on the + # same workspace would collide. Jobs allow duplicates, which is why foo above does not. + name: bar-$UNIQUE_NAME + catalog: main + schema: default + serverless: true + libraries: + - file: + path: ./transform.py diff --git a/acceptance/bundle/dms/no-drift/out.test.toml b/acceptance/bundle/dms/no-drift/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/no-drift/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt new file mode 100644 index 00000000000..7b23792edcd --- /dev/null +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -0,0 +1,126 @@ + +=== Deploy job foo and pipeline bar; both are recorded at version 1 +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files... +Created jobs.foo +Created pipelines.bar +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "pipelines.bar" + } +] + +=== Plan without touching anything: the deployment stamp must not show as drift +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== Update only the pipeline (not the job), then redeploy: only the pipeline's recorded version bumps to 2 +>>> update_file.py databricks.yml serverless: true serverless: true + development: true + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files... +Updated pipelines.bar +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } +} +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[UUID]", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "2" + }, + "development": true, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } +} + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", + "last_version_id": "2", + "resource_key": "pipelines.bar" + } +] + +=== Plan again: no residual drift, and the version stamps are ignored +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script new file mode 100644 index 00000000000..8a18d13f258 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/script @@ -0,0 +1,26 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy job foo and pipeline bar; both are recorded at version 1" +trace $CLI bundle deploy + +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-drift-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +# Only deploy stamps deployment.deployment_id. Plan sees it in state but not in config, +# so it must ignore it rather than report spurious changes. +title "Plan without touching anything: the deployment stamp must not show as drift" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" + +title "Update only the pipeline (not the job), then redeploy: only the pipeline's recorded version bumps to 2" +trace update_file.py databricks.yml 'serverless: true' 'serverless: true + development: true' +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "Plan again: no residual drift, and the version stamps are ignored" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" + +# The list-resources and final plan above record requests we do not assert on; drop them so +# out.requests.txt does not become a nondeterministic golden. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/no-drift/transform.py b/acceptance/bundle/dms/no-drift/transform.py new file mode 100644 index 00000000000..c07007592a6 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/transform.py @@ -0,0 +1 @@ +# Minimal source so the pipeline has a library, which the API requires. diff --git a/acceptance/bundle/dms/no-resources/databricks.yml.tmpl b/acceptance/bundle/dms/no-resources/databricks.yml.tmpl new file mode 100644 index 00000000000..973189b1603 --- /dev/null +++ b/acceptance/bundle/dms/no-resources/databricks.yml.tmpl @@ -0,0 +1,4 @@ +bundle: + name: dms-no-resources-$UNIQUE_NAME +experimental: + record_deployment_history: true diff --git a/acceptance/bundle/dms/no-resources/out.test.toml b/acceptance/bundle/dms/no-resources/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/no-resources/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt new file mode 100644 index 00000000000..a4f993a04e5 --- /dev/null +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -0,0 +1,80 @@ + +=== First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files... +Files: 5 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-no-resources-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state/resources.deployment.json +{ + "object_type": "BUNDLE_DEPLOYMENT", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state/resources.deployment.json" +} + +=== Redeploy: the deployment is resolved from that node, so no second deployment is created +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script new file mode 100644 index 00000000000..13924f3664f --- /dev/null +++ b/acceptance/bundle/dms/no-resources/script @@ -0,0 +1,10 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' + +title "Redeploy: the deployment is resolved from that node, so no second deployment is created" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl new file mode 100644 index 00000000000..c46cfb03d73 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-upload-fails-once-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_upload_fails_once_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/operation-upload-fails-once/out.requests.txt b/acceptance/bundle/dms/operation-upload-fails-once/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/operation-upload-fails-once/out.test.toml b/acceptance/bundle/dms/operation-upload-fails-once/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt new file mode 100644 index 00000000000..abdeedeb959 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -0,0 +1,79 @@ + +=== Deploy so there is a recorded resource to recreate +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one +>>> fault.py PATCH /api/2.0/bundle/* 500 0 1 + +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Error: recording operation for resources.schemas.foo: Fault injected by test. (500 INJECTED) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo?update_mask=state%2Cerror_message%2Cstatus +HTTP Status: 500 Internal Server Error +API error_code: INJECTED +API message: Fault injected by test. + +Files: 4 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_PENDING" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_upload_fails_once_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script new file mode 100644 index 00000000000..5b609b1b8ee --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -0,0 +1,15 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy so there is a recorded resource to recreate" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one" +trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +trace musterr $CLI bundle deploy --auto-approve +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/operation-upload-fails-once/test.toml b/acceptance/bundle/dms/operation-upload-fails-once/test.toml new file mode 100644 index 00000000000..5baa9c9d923 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/test.toml @@ -0,0 +1,3 @@ +# Local only: this case needs an injected failure - DMS itself must reject the operation upload, once - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false diff --git a/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl new file mode 100644 index 00000000000..b6aa93265df --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl @@ -0,0 +1,23 @@ +bundle: + name: dms-operation-upload-fails-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five + six: + name: six + seven: + name: seven + eight: + name: eight diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-fails/output.txt b/acceptance/bundle/dms/operation-upload-fails/output.txt new file mode 100644 index 00000000000..2b273972417 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/output.txt @@ -0,0 +1,4 @@ + +=== An operation upload failure fails the deploy instead of reporting only at the end +>>> grep -c ^Error: LOG.deploy +deploy reported errors diff --git a/acceptance/bundle/dms/operation-upload-fails/script b/acceptance/bundle/dms/operation-upload-fails/script new file mode 100644 index 00000000000..cc2ead92c6f --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/script @@ -0,0 +1,9 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "An operation upload failure fails the deploy instead of reporting only at the end" +# Which resources get refused depends on how far apply got before a background +# upload failed, so the per-resource errors go to a LOG file rather than the diff. +fault.py "PATCH /api/2.0/bundle/*" 500 0 99 INTERNAL_ERROR +errcode $CLI bundle deploy &> LOG.deploy +contains.py 'recording operation for' '!panic' < LOG.deploy > /dev/null +trace grep -c "^Error:" LOG.deploy > /dev/null && echo "deploy reported errors" diff --git a/acceptance/bundle/dms/operation-upload-fails/test.toml b/acceptance/bundle/dms/operation-upload-fails/test.toml new file mode 100644 index 00000000000..b5502a2516c --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/test.toml @@ -0,0 +1,10 @@ +# Local only: this case needs an injected failure - DMS itself must reject the operation upload - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false + +# Which requests are made depends on how far apply got before a background upload +# failed, so recording them would make the output nondeterministic. +RecordRequests = false + +# Completed versions make DMS the source of truth; unrecorded resources get recreated. +# Deploy must stop rather than continue creating. diff --git a/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl new file mode 100644 index 00000000000..bc5b4ec97ad --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-operation-upload-message-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/operation-upload-message/out.test.toml b/acceptance/bundle/dms/operation-upload-message/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-message/output.txt b/acceptance/bundle/dms/operation-upload-message/output.txt new file mode 100644 index 00000000000..0791010b1f1 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/output.txt @@ -0,0 +1,16 @@ + +=== What a user sees when an operation cannot be recorded +>>> fault.py PATCH /api/2.0/bundle/* 500 0 1 INTERNAL_ERROR + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-operation-upload-message-[UNIQUE_NAME]/default/files... +Error: recording operation for resources.jobs.foo: Fault injected by test. (500 INTERNAL_ERROR) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Fault injected by test. + +Files: 5 uploaded, 0 deleted + +Exit code: 1 diff --git a/acceptance/bundle/dms/operation-upload-message/script b/acceptance/bundle/dms/operation-upload-message/script new file mode 100644 index 00000000000..c6b796e4149 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/script @@ -0,0 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "What a user sees when an operation cannot be recorded" +trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 INTERNAL_ERROR +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/dms/operation-upload-message/test.toml b/acceptance/bundle/dms/operation-upload-message/test.toml new file mode 100644 index 00000000000..1dbd55da1f0 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/test.toml @@ -0,0 +1,8 @@ +# Local only: this case needs an injected failure - DMS itself must reject the operation upload - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false + +# One resource, so which operation is refused is fixed and the error a user sees can be +# asserted. operation-upload-fails covers a deploy of several resources, where which ones +# were reached before the failure landed is not. +RecordRequests = false diff --git a/acceptance/bundle/dms/provenance/databricks.yml.tmpl b/acceptance/bundle/dms/provenance/databricks.yml.tmpl new file mode 100644 index 00000000000..0a3fa1656ae --- /dev/null +++ b/acceptance/bundle/dms/provenance/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: dms-provenance-$UNIQUE_NAME +experimental: + record_deployment_history: true + +targets: + dev: + default: true + mode: development + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/provenance/out.test.toml b/acceptance/bundle/dms/provenance/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/provenance/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt new file mode 100644 index 00000000000..2142dabb700 --- /dev/null +++ b/acceptance/bundle/dms/provenance/output.txt @@ -0,0 +1,68 @@ + +=== Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/files... +Created jobs.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //versions +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID] +{ + "target_name": "dev", + "deployment_mode": "DEPLOYMENT_MODE_DEVELOPMENT", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev" + } +} diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script new file mode 100644 index 00000000000..8b15822fa4b --- /dev/null +++ b/acceptance/bundle/dms/provenance/script @@ -0,0 +1,19 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode" +git-repo-init +git remote add origin https://github.com/databricks/bundle-examples.git +trace $CLI bundle deploy +# The commit SHA changes every run, so assert it is a 40-char hex string and drop it. +add_repl.py "$(git rev-parse HEAD)" COMMIT +trace print_requests.py --dms //versions + +title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance-${UNIQUE_NAME}/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}" | jq '{target_name, deployment_mode, git_info, workspace_info}' + +# The deploy uploads files in a nondeterministic order; only the requests above are +# asserted. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/provenance/test.toml b/acceptance/bundle/dms/provenance/test.toml new file mode 100644 index 00000000000..0a47bfb1b91 --- /dev/null +++ b/acceptance/bundle/dms/provenance/test.toml @@ -0,0 +1,4 @@ +# git-repo-init creates a repo in the test directory so the deploy has git provenance. +Ignore = [ + '.git', +] diff --git a/acceptance/bundle/dms/readplan/databricks.yml.tmpl b/acceptance/bundle/dms/readplan/databricks.yml.tmpl new file mode 100644 index 00000000000..d11621ab1e5 --- /dev/null +++ b/acceptance/bundle/dms/readplan/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: dms-readplan-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + pipelines: + bar: + # Unique: the Pipelines API refuses a duplicate name across overlapping runs. + name: bar-$UNIQUE_NAME + catalog: main + schema: default + serverless: true + libraries: + - file: + path: ./transform.py diff --git a/acceptance/bundle/dms/readplan/out.test.toml b/acceptance/bundle/dms/readplan/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/readplan/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/readplan/output.txt b/acceptance/bundle/dms/readplan/output.txt new file mode 100644 index 00000000000..371855dbcc0 --- /dev/null +++ b/acceptance/bundle/dms/readplan/output.txt @@ -0,0 +1,22 @@ + +=== Save a plan, then deploy from it: the deploy stamps the id and version onto the job and pipeline the plan predates +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy --plan tmp.plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files... +Created jobs.foo +Created pipelines.bar +Files: 7 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Re-plan: no drift, the deployed job and pipeline carry the deployment stamp +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== Both recorded operations carry deployment_id and version_id (lineage), not just kind +>>> print_requests.py --dms //api/2.0/bundle --sort --oneline +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.bar", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[UUID]", "sequence_id": "0", "state": "{\"state\":{\"catalog\":\"main\",\"channel\":\"CURRENT\",\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edition\":\"ADVANCED\",\"libraries\":[{\"file\":{\"path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files/transform.py\"}}],\"name\":\"bar-[UNIQUE_NAME]\",\"schema\":\"default\",\"serverless\":true}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-readplan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}, {"resource_key": "pipelines.bar", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/readplan/script b/acceptance/bundle/dms/readplan/script new file mode 100644 index 00000000000..40bd110bf00 --- /dev/null +++ b/acceptance/bundle/dms/readplan/script @@ -0,0 +1,13 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Save a plan, then deploy from it: the deploy stamps the id and version onto the job and pipeline the plan predates" +trace $CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle deploy --plan tmp.plan.json + +title "Re-plan: no drift, the deployed job and pipeline carry the deployment stamp" +trace $CLI bundle plan + +title "Both recorded operations carry deployment_id and version_id (lineage), not just kind" +trace print_requests.py --dms //api/2.0/bundle --sort --oneline | contains.py deployment_id version_id + +rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/readplan/transform.py b/acceptance/bundle/dms/readplan/transform.py new file mode 100644 index 00000000000..c07007592a6 --- /dev/null +++ b/acceptance/bundle/dms/readplan/transform.py @@ -0,0 +1 @@ +# Minimal source so the pipeline has a library, which the API requires. diff --git a/acceptance/bundle/dms/record-failure/databricks.yml.tmpl b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl new file mode 100644 index 00000000000..87f0eb4d1a9 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-record-failure-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + doomed: + name: doomed diff --git a/acceptance/bundle/dms/record-failure/out.test.toml b/acceptance/bundle/dms/record-failure/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt new file mode 100644 index 00000000000..479acca7a84 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -0,0 +1,75 @@ + +=== A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource +>>> fault.py POST /api/2.2/jobs/create 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/files... +Error: cannot create resources.jobs.doomed: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Files: 6 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-record-failure-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "jobs.doomed", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.doomed", + "q": { + "update_mask": "error_message,status" + }, + "body": { + "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "sequence_id": "0", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +=== The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{} + +=== Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged +>>> [CLI] bundle plan +create jobs.doomed + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script new file mode 100644 index 00000000000..bc0947a1d91 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/script @@ -0,0 +1,21 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" +trace fault.py "POST /api/2.2/jobs/create" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/record-failure/test.toml b/acceptance/bundle/dms/record-failure/test.toml new file mode 100644 index 00000000000..8a850ba07c5 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/test.toml @@ -0,0 +1,5 @@ +# Local only: the failure is injected, and fault.py registers its rule on the fake, which +# a cloud run does not have. Simulating it for real instead - a catalog or principal that +# does not exist - does not work either: the fake accepts both, so the case would stop +# failing locally. Making it portable needs the fake to model that existence. +Cloud = false diff --git a/acceptance/bundle/dms/record/databricks.yml.tmpl b/acceptance/bundle/dms/record/databricks.yml.tmpl new file mode 100644 index 00000000000..214793eca38 --- /dev/null +++ b/acceptance/bundle/dms/record/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-record-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/record/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt new file mode 100644 index 00000000000..7e63506bf79 --- /dev/null +++ b/acceptance/bundle/dms/record/output.txt @@ -0,0 +1,84 @@ + +=== Deploy, then destroy: set up a deployment and delete it, so the deploy below starts from nothing +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +=== Deploy again: the destroy took the workspace node with it, so there is nothing to resolve and a fresh deployment starts at version 1, not a continuation of the deleted one +>>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json) doesn't exist. + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json +{ + "object_type": "BUNDLE_DEPLOYMENT", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json" +} + +>>> print_requests.py --get --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-record-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script new file mode 100644 index 00000000000..c521416b158 --- /dev/null +++ b/acceptance/bundle/dms/record/script @@ -0,0 +1,16 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy, then destroy: set up a deployment and delete it, so the deploy below starts from nothing" +trace $CLI bundle deploy +trace $CLI bundle destroy --auto-approve + +# Discard the deploy+destroy requests above; this test is only about the deploy that FOLLOWS a +# destroy. The individual deploy and destroy request sequences are covered by no-resources, +# successful-recreate, and multiple-resources. +rm -f out.requests.txt + +title "Deploy again: the destroy took the workspace node with it, so there is nothing to resolve and a fresh deployment starts at version 1, not a continuation of the deleted one" +trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" +trace $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' +trace print_requests.py --get --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl b/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl new file mode 100644 index 00000000000..01d891c3880 --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-serialized-plan-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/serialized-plan/out.test.toml b/acceptance/bundle/dms/serialized-plan/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/serialized-plan/output.txt b/acceptance/bundle/dms/serialized-plan/output.txt new file mode 100644 index 00000000000..fc84deb9f4c --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/output.txt @@ -0,0 +1,41 @@ + +=== Deploy once to create the deployment (records version 1) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A saved plan carries the deployment id and the versions it targets (last 1, next 2) +>>> [CLI] bundle plan -o json + +>>> jq {deployment_id, last_version_id, next_version_id} tmp.plan.json +{ + "deployment_id": "[NUMID]", + "last_version_id": "1", + "next_version_id": "2" +} + +=== Change the job and deploy again: advances the deployment to version 2 +>>> update_file.py databricks.yml name: foo name: foo-v2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 4 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +=== Replaying the earlier plan is rejected: it predates the deployment's current version +>>> musterr [CLI] bundle deploy --plan tmp.plan.json +Error: this plan predates the deployment's current version 2; run 'bundle plan' again + + +=== Only versions 1 and 2 were recorded; the rejected replay created none +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-serialized-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"2\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo-v2\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/serialized-plan/script b/acceptance/bundle/dms/serialized-plan/script new file mode 100644 index 00000000000..0800e1a8265 --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy once to create the deployment (records version 1)" +trace $CLI bundle deploy + +title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" +trace $CLI bundle plan -o json > tmp.plan.json +trace jq '{deployment_id, last_version_id, next_version_id}' tmp.plan.json + +title "Change the job and deploy again: advances the deployment to version 2" +trace update_file.py databricks.yml "name: foo" "name: foo-v2" +trace $CLI bundle deploy + +title "Replaying the earlier plan is rejected: it predates the deployment's current version" +trace musterr $CLI bundle deploy --plan tmp.plan.json + +title "Only versions 1 and 2 were recorded; the rejected replay created none" +trace print_requests.py --dms //api/2.0/bundle --oneline + +rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl b/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl new file mode 100644 index 00000000000..dea8fbd9a8d --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-stale-plan-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/stale-plan/out.test.toml b/acceptance/bundle/dms/stale-plan/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt new file mode 100644 index 00000000000..2ce3b598d1c --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -0,0 +1,21 @@ + +=== Save a plan, then deploy it: records version 1 +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy --plan tmp.plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Replay: re-deploying the same plan is rejected - it predates the recorded version +>>> musterr [CLI] bundle deploy --plan tmp.plan.json +Error: this plan predates the deployment's current version 1; run 'bundle plan' again + + +=== Only the first deploy recorded a version; the rejected replay created none +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-stale-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script new file mode 100644 index 00000000000..1f9e792f30f --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/script @@ -0,0 +1,13 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Save a plan, then deploy it: records version 1" +trace $CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle deploy --plan tmp.plan.json + +title "Replay: re-deploying the same plan is rejected - it predates the recorded version" +trace musterr $CLI bundle deploy --plan tmp.plan.json + +title "Only the first deploy recorded a version; the rejected replay created none" +trace print_requests.py --dms //api/2.0/bundle --oneline + +rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl b/acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..8a40cabeda6 --- /dev/null +++ b/acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-successful-recreate-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_successful_recreate_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/successful-recreate/out.requests.txt b/acceptance/bundle/dms/successful-recreate/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/successful-recreate/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/successful-recreate/out.test.toml b/acceptance/bundle/dms/successful-recreate/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/successful-recreate/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt new file mode 100644 index 00000000000..4d497d81f2b --- /dev/null +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -0,0 +1,180 @@ + +=== Deploy: the state write records the resource +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-successful-recreate-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Recreate writes state twice - the entry is dropped, then the new resource is saved +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Recreated schemas.foo +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_PENDING" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", + "sequence_id": "1", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Destroy: the delete is recorded with the id and no state +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "previous_version_id": "2", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_DELETE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script new file mode 100644 index 00000000000..43050cc7c8f --- /dev/null +++ b/acceptance/bundle/dms/successful-recreate/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy: the state write records the resource" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "Recreate writes state twice - the entry is dropped, then the new resource is saved" +# One operation per resource per version; both writes land on the same one. +# The drop opens it IN_PROGRESS, the save patches it to SUCCEEDED. +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +trace $CLI bundle deploy --auto-approve +trace print_requests.py --dms //api/2.0/bundle + +title "Destroy: the delete is recorded with the id and no state" +trace $CLI bundle destroy --auto-approve +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/summary/databricks.yml.tmpl b/acceptance/bundle/dms/summary/databricks.yml.tmpl new file mode 100644 index 00000000000..1c2fa3c7f20 --- /dev/null +++ b/acceptance/bundle/dms/summary/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-summary-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/summary/out.test.toml b/acceptance/bundle/dms/summary/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/summary/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt new file mode 100644 index 00000000000..233f2416dc3 --- /dev/null +++ b/acceptance/bundle/dms/summary/output.txt @@ -0,0 +1,37 @@ + +=== Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "1" +} + +=== Redeploying advances the version the summary reports +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "2" +} + +=== After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> [CLI] bundle summary -o json +false diff --git a/acceptance/bundle/dms/summary/script b/acceptance/bundle/dms/summary/script new file mode 100644 index 00000000000..75ec8612ffe --- /dev/null +++ b/acceptance/bundle/dms/summary/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "Redeploying advances the version the summary reports" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID" +trace $CLI bundle destroy --auto-approve +trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' + +# This test asserts the summary output, not the requests behind it; the file uploads +# recorded here are ordered nondeterministically. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml new file mode 100644 index 00000000000..32a52823ff7 --- /dev/null +++ b/acceptance/bundle/dms/test.toml @@ -0,0 +1,19 @@ +Cloud = true + +# Deployment Metadata Service (DMS) recording is only supported by the direct +# engine; it is a no-op on terraform. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + + +RecordRequests = true + +Ignore = [ + '.databricks', + 'databricks.yml', +] + +# These tests set experimental.record_deployment_history in databricks.yml; the DMS env var +# both records and permits that otherwise-gated field. bundle/dms/not-supported covers the +# rejection path. Pinned to "true" (not the suite's ["", "true"]) since the field already +# records - the empty variant would just duplicate it. +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/version-never-created/databricks.yml b/acceptance/bundle/dms/version-never-created/databricks.yml new file mode 100644 index 00000000000..a7077f18b1f --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-version-never-created + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt new file mode 100644 index 00000000000..8a7e5fe8506 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -0,0 +1,38 @@ + +=== The first version fails, so no deployment record exists - only the node naming its ID +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + +Files: 5 uploaded, 0 deleted + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json +{ + "object_type": "BUNDLE_DEPLOYMENT" +} + +>>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-version-never-created", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} + +=== The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + +Files: 2 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script new file mode 100644 index 00000000000..58178feaf53 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/script @@ -0,0 +1,8 @@ +title "The first version fails, so no deployment record exists - only the node naming its ID" +trace musterr $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' +trace print_requests.py --dms //api/2.0/bundle --get --oneline + +title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" +trace musterr $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get --oneline diff --git a/acceptance/bundle/dms/version-never-created/test.toml b/acceptance/bundle/dms/version-never-created/test.toml new file mode 100644 index 00000000000..6553e7c09bb --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/test.toml @@ -0,0 +1,12 @@ +# Local only: the failure has to land on CreateVersion and not on the CreateDeployment +# that precedes it, which fault.py's trailing-* pattern cannot express, so this case +# keeps a [[Server]] stub - and the cloud harness ignores stubs. +Cloud = false + +# The first version fails, so the deployment record is never created - only the +# workspace node CreateDeployment registered. The next deploy resolves the ID from +# that node and has to cope with a deployment that has no record yet. +[[Server]] +Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/empty_string_dropped/script b/acceptance/bundle/empty_string_dropped/script index a03c479c071..01ed00ce42f 100644 --- a/acceptance/bundle/empty_string_dropped/script +++ b/acceptance/bundle/empty_string_dropped/script @@ -8,7 +8,7 @@ # Resolved config the engines see. Today every "" field is still present here; a # fix in the initialize phase would drop them, and this golden would show that. -$CLI bundle validate -o json -t direct | nostamp | jq .resources > out.validate.json +$CLI bundle validate -o json -t direct | nostamp.py | jq .resources > out.validate.json # Exclude non-create traffic: workspace file ops, telemetry (nondeterministic), and the # deployment history calls the DMS run adds. diff --git a/acceptance/bundle/environments/dependencies/script b/acceptance/bundle/environments/dependencies/script index 00e5fc846cc..77c215da892 100644 --- a/acceptance/bundle/environments/dependencies/script +++ b/acceptance/bundle/environments/dependencies/script @@ -4,11 +4,11 @@ set -euo pipefail trace $CLI bundle validate trace $CLI bundle deploy -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt | nostamp.py -trace jq -s '.[] | select(.path=="/api/2.0/pipelines")' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.0/pipelines")' out.requests.txt | nostamp.py -trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs.test_job.environments' -trace $CLI bundle validate -o json | nostamp | jq '.resources.pipelines.test_pipeline.environment' +trace $CLI bundle validate -o json | nostamp.py | jq '.resources.jobs.test_job.environments' +trace $CLI bundle validate -o json | nostamp.py | jq '.resources.pipelines.test_pipeline.environment' rm out.requests.txt diff --git a/acceptance/bundle/escaped_refs/out.plan.direct.json b/acceptance/bundle/escaped_refs/out.plan.direct.json index ae4fd743d26..a64f09b477b 100644 --- a/acceptance/bundle/escaped_refs/out.plan.direct.json +++ b/acceptance/bundle/escaped_refs/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.example_ingestion_job": { diff --git a/acceptance/bundle/escaped_refs/out.test.toml b/acceptance/bundle/escaped_refs/out.test.toml index bf4f0e7d3c6..d5b9a6f4a16 100644 --- a/acceptance/bundle/escaped_refs/out.test.toml +++ b/acceptance/bundle/escaped_refs/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/escaped_refs/output.txt b/acceptance/bundle/escaped_refs/output.txt index 6e12dbfd8fc..342535d0037 100644 --- a/acceptance/bundle/escaped_refs/output.txt +++ b/acceptance/bundle/escaped_refs/output.txt @@ -15,7 +15,7 @@ Created jobs.example_ingestion_job Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //jobs +>>> print_requests.py //jobs --nostamp { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/escaped_refs/script b/acceptance/bundle/escaped_refs/script index 34640492656..1b8986539c6 100644 --- a/acceptance/bundle/escaped_refs/script +++ b/acceptance/bundle/escaped_refs/script @@ -12,10 +12,12 @@ title "Validate: escapes are still escaped in the config" trace $CLI bundle validate -o json | jq '.resources.jobs.example_ingestion_job.tasks[0].notebook_task.base_parameters' title "Plan (json)" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Not traced: the READPLAN variant adds "--plan ...", and the point of that variant is # that the payload below is identical either way. title "Deploy: payload sent to the jobs API\n" $CLI bundle deploy $(readplanarg out.plan.$DATABRICKS_BUNDLE_ENGINE.json) -trace print_requests.py //jobs +# --nostamp: under deployment-history recording the job payload carries the DMS deployment +# stamp; drop it so the assertion is the same whether or not recording is on. +trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/escaped_refs/test.toml b/acceptance/bundle/escaped_refs/test.toml index e7b9cac3bf1..94b36638c0e 100644 --- a/acceptance/bundle/escaped_refs/test.toml +++ b/acceptance/bundle/escaped_refs/test.toml @@ -8,3 +8,7 @@ Ignore = [ ".databricks", "databricks.yml", ] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/generate/dashboard-inplace/out.test.toml b/acceptance/bundle/generate/dashboard-inplace/out.test.toml index e1af1a235ad..ae800809893 100644 --- a/acceptance/bundle/generate/dashboard-inplace/out.test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/generate/dashboard-inplace/test.toml b/acceptance/bundle/generate/dashboard-inplace/test.toml index d5511c15564..10eef9f96c4 100644 --- a/acceptance/bundle/generate/dashboard-inplace/test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/test.toml @@ -1,3 +1,7 @@ +# bundle generate reading recorded state from DMS is a fast followup; until then this +# deploy-then-generate flow can't run under recording (local state is a tombstone). +EnvMatrix.DMS = [""] + [[Repls]] Old = "[0-9a-f]{32}" New = "[DASHBOARD_ID]" diff --git a/acceptance/bundle/generate/genie_space_inplace/out.test.toml b/acceptance/bundle/generate/genie_space_inplace/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/generate/genie_space_inplace/out.test.toml +++ b/acceptance/bundle/generate/genie_space_inplace/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/generate/genie_space_inplace/test.toml b/acceptance/bundle/generate/genie_space_inplace/test.toml index 723c9907976..91944fc581e 100644 --- a/acceptance/bundle/generate/genie_space_inplace/test.toml +++ b/acceptance/bundle/generate/genie_space_inplace/test.toml @@ -1,5 +1,8 @@ # Genie spaces are only deployed via the direct deployment engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +# bundle generate reading recorded state from DMS is a fast followup; until then this +# deploy-then-generate flow can't run under recording (local state is a tombstone). +EnvMatrix.DMS = [""] [[Repls]] Old = "[0-9a-f]{32}" diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json b/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json index cbaa60b7ace..e564ae92773 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json +++ b/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/auto-migrate-clean/script b/acceptance/bundle/migrate/auto-migrate-clean/script index 0f2eb7c1ce1..5afad5a39c5 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/script +++ b/acceptance/bundle/migrate/auto-migrate-clean/script @@ -31,7 +31,7 @@ rm -f out.requests.txt title "Modify the config — plan shows the pending update" trace update_file.py databricks.yml "Test Auto-Migrate Job" "Test Auto-Migrate Job (renamed)" trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_update.direct.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.direct.json title "Deploy picks up the update (with and without --plan)\n" $CLI bundle deploy $(readplanarg out.plan_update.direct.json) diff --git a/acceptance/bundle/migrate/auto-migrate-clean/test.toml b/acceptance/bundle/migrate/auto-migrate-clean/test.toml index ff8a66c196e..32db4804fbf 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/test.toml +++ b/acceptance/bundle/migrate/auto-migrate-clean/test.toml @@ -1 +1,5 @@ EnvMatrix.READPLAN = ["", "1"] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/script b/acceptance/bundle/migrate/auto-migrate-direct-only/script index 3cb22cb053a..7f9c18cac46 100644 --- a/acceptance/bundle/migrate/auto-migrate-direct-only/script +++ b/acceptance/bundle/migrate/auto-migrate-direct-only/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml "name: test-bundle" $'name: test-bundle\n e title "Plan skips the direct-only resource, since this run still uses terraform" trace $CLI bundle plan -trace $CLI bundle plan -o json | gron.py | grep instance_pools +trace $CLI bundle plan -o json | nostamp.py | gron.py | grep instance_pools title "Deploy migrates the state, skipping the direct-only resource" trace $CLI bundle deploy diff --git a/acceptance/bundle/migrate/basic/out.plan_update.json b/acceptance/bundle/migrate/basic/out.plan_update.json index 7b9967a085d..fec804b95b3 100644 --- a/acceptance/bundle/migrate/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/basic/out.plan_update.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/basic/script b/acceptance/bundle/migrate/basic/script index 3247e9fad8d..64c42e6d641 100755 --- a/acceptance/bundle/migrate/basic/script +++ b/acceptance/bundle/migrate/basic/script @@ -43,7 +43,7 @@ rm out.requests.txt title "Update databricks.yml, run plan & update" trace update_file.py databricks.yml Migration Migrated trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan_update.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_update.json trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy title "Different target, still on terraform" diff --git a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json index cbdc8b8203b..66f20559a9a 100644 --- a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json +++ b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/dashboards/script b/acceptance/bundle/migrate/dashboards/script index bd599144f94..b2c9639a0fd 100755 --- a/acceptance/bundle/migrate/dashboards/script +++ b/acceptance/bundle/migrate/dashboards/script @@ -15,7 +15,7 @@ rm out.requests.txt # Badness: known issue with dashboards, will show up as 'recreate' see https://github.com/databricks/cli/pull/3966 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 unchanged" trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan | contains.py "1 unchanged" -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan_after_migrate.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_after_migrate.json trace print_requests.py --get //dashboards | contains.py 'engine/direct' > /dev/null trace DATABRICKS_BUNDLE_ENGINE="" $CLI bundle deploy --auto-approve @@ -32,7 +32,7 @@ rm out.requests.txt #title "Update databricks.yml, run plan & update" #trace update_file.py databricks.yml dashboard DASHBOARD #trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -#trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan_update.json +#trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_update.json #trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy #rm out.requests.txt diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json index 3e01413959f..84de0f4ffdc 100644 --- a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json +++ b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json index e21c3b33069..a54d3357cb1 100644 --- a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/migrate/default-python/script b/acceptance/bundle/migrate/default-python/script index e3a0b9b3e05..6e05ab85f7f 100755 --- a/acceptance/bundle/migrate/default-python/script +++ b/acceptance/bundle/migrate/default-python/script @@ -11,7 +11,7 @@ trace print_state.py > ../out.state_after_migration.json trace jq '.. | .libraries? | select(.)' ../out.state_after_migration.json trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json > ../out.plan_after_migration.json +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json | nostamp.py > ../out.plan_after_migration.json trace jq '.plan[] | .changes' ../out.plan_after_migration.json # Badness: contains changes for whl and num_workers trace DATABRICKS_BUNDLE_ENGINE="" $CLI bundle deploy @@ -21,5 +21,5 @@ rm ../out.state_after_deploy.json title "Extra plan: should have no drift" trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json > ../out.plan_after_deploy.json +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json | nostamp.py > ../out.plan_after_deploy.json trace jq '.plan[] | .changes' ../out.plan_after_migration.json # Badness: contains changes for whl diff --git a/acceptance/bundle/migrate/runas/out.plan.json b/acceptance/bundle/migrate/runas/out.plan.json index 0888cdc22ef..56992a01bc0 100644 --- a/acceptance/bundle/migrate/runas/out.plan.json +++ b/acceptance/bundle/migrate/runas/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/migrate/runas/script b/acceptance/bundle/migrate/runas/script index f3675f4d9bb..4affba0a373 100644 --- a/acceptance/bundle/migrate/runas/script +++ b/acceptance/bundle/migrate/runas/script @@ -23,4 +23,4 @@ trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deployment migrate trace print_state.py > out.new_state.json trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan.json diff --git a/acceptance/bundle/resource_deps/escaped_ref/output.txt b/acceptance/bundle/resource_deps/escaped_ref/output.txt index 2d7df920e29..5370251a14d 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/output.txt +++ b/acceptance/bundle/resource_deps/escaped_ref/output.txt @@ -18,7 +18,7 @@ Created jobs.foo Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //jobs --sort +>>> print_requests.py //jobs --sort --nostamp { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resource_deps/escaped_ref/script b/acceptance/bundle/resource_deps/escaped_ref/script index 413394d8b7d..c53b9b85671 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/script +++ b/acceptance/bundle/resource_deps/escaped_ref/script @@ -12,4 +12,6 @@ trace jq '.plan | map_values(.depends_on)' plan.json # that the payload below is identical either way. title "Payload: escaped stays literal, real is resolved\n" $CLI bundle deploy $(readplanarg plan.json) -trace print_requests.py //jobs --sort +# --nostamp: under deployment-history recording the job payload carries the DMS deployment +# stamp; drop it so the assertion is the same whether or not recording is on. +trace print_requests.py //jobs --sort --nostamp diff --git a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json index 3bb4c8310c8..424b73dc67b 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.schemas.schema_a": { diff --git a/acceptance/bundle/resource_deps/grant_ref/script b/acceptance/bundle/resource_deps/grant_ref/script index 82800b1654e..ad897f76987 100644 --- a/acceptance/bundle/resource_deps/grant_ref/script +++ b/acceptance/bundle/resource_deps/grant_ref/script @@ -1,4 +1,4 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json b/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json index f004dcdfcde..1c0e5115a32 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.a": { diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json b/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json index fa72eb2700b..343ca061a3b 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json b/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json index 0c9904aec79..192e2dd8c2b 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/id_chain/script b/acceptance/bundle/resource_deps/id_chain/script index bcd103ac2a5..eada43ba1c9 100644 --- a/acceptance/bundle/resource_deps/id_chain/script +++ b/acceptance/bundle/resource_deps/id_chain/script @@ -16,7 +16,7 @@ print_requests_short() { trace $CLI bundle plan trace print_requests -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests trace $CLI bundle deploy @@ -25,7 +25,7 @@ trace print_requests_short trace update_file.py databricks.yml aa_desc aa_new_desc trace update_file.py databricks.yml prefix new_prefix -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests trace $CLI bundle plan @@ -34,5 +34,5 @@ trace print_requests trace $CLI bundle deploy trace print_requests_short -$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests diff --git a/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json b/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json index 7c32ae3fe4c..b955e17cf46 100644 --- a/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.a": { diff --git a/acceptance/bundle/resource_deps/id_star/script b/acceptance/bundle/resource_deps/id_star/script index 6b67f59b7b8..46366e7706a 100644 --- a/acceptance/bundle/resource_deps/id_star/script +++ b/acceptance/bundle/resource_deps/id_star/script @@ -1 +1 @@ -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json b/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json index 43c1928ebb8..4cdbdd7e630 100644 --- a/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.bar": { diff --git a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json index 1bfbc8fe053..c6a1b111aca 100644 --- a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/job_id/script b/acceptance/bundle/resource_deps/job_id/script index f8d30917d1c..96d8aed011f 100644 --- a/acceptance/bundle/resource_deps/job_id/script +++ b/acceptance/bundle/resource_deps/job_id/script @@ -1,6 +1,6 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs @@ -8,7 +8,7 @@ foo_id=`read_id.py foo` bar_id=`read_id.py bar` cp empty.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json index 64087a8c63c..8aa9e5e2544 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/script b/acceptance/bundle/resource_deps/job_id_delete_bar/script index 03b9a087810..8fd0af69437 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/script +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/script @@ -8,7 +8,7 @@ bar_id=`read_id.py bar` title "Delete bar, keep foo (foo config updated to not depend on bar)" cp only_foo.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy # Sort requests because foo's update and bar's delete have no dependency edge # (foo's dependency on bar was removed in only_foo.yml), so execution order is non-deterministic. diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json index 6c6c9577e8f..83132ceb0b4 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/script b/acceptance/bundle/resource_deps/job_id_delete_foo/script index f5efbd22609..30e1194c4cd 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/script +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/script @@ -8,7 +8,7 @@ bar_id=`read_id.py bar` title "Delete foo, keep bar" cp only_bar.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/jobs_update/script b/acceptance/bundle/resource_deps/jobs_update/script index f806f2db846..b1fc7332355 100644 --- a/acceptance/bundle/resource_deps/jobs_update/script +++ b/acceptance/bundle/resource_deps/jobs_update/script @@ -21,8 +21,8 @@ trace $CLI bundle plan title "Fetch job ID and verify remote state" # Badness: output should not be different per engine; investigate if it's the same on cloud -trace $CLI jobs get $foo_id | nostamp > out.get_foo.$DATABRICKS_BUNDLE_ENGINE.json -trace $CLI jobs get $bar_id | nostamp +trace $CLI jobs get $foo_id | nostamp.py > out.get_foo.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI jobs get $bar_id | nostamp.py rm out.requests.txt trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json index d768b766d36..be2fb1b17e7 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.bar": { diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json index 77e77c19eb1..deb96f123e6 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/script b/acceptance/bundle/resource_deps/jobs_update_remote/script index d456ed8b023..89a3fc4c36f 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/script +++ b/acceptance/bundle/resource_deps/jobs_update_remote/script @@ -1,5 +1,5 @@ echo "*" > .gitignore -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs @@ -13,7 +13,7 @@ bar_id=`read_id.py bar` title "Update trigger.periodic.unit remotely and re-deploy; jobs.bar is unchanged" trace envsubst < job_update.json > tmp.json && mv tmp.json job_update.json trace $CLI jobs reset --json @job_update.json -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle destroy --auto-approve trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/missing_string_field/script b/acceptance/bundle/resource_deps/missing_string_field/script index c526e980444..d96e2f30b0a 100644 --- a/acceptance/bundle/resource_deps/missing_string_field/script +++ b/acceptance/bundle/resource_deps/missing_string_field/script @@ -1,4 +1,4 @@ -trace $CLI bundle validate -o json | nostamp | jq .resources +trace $CLI bundle validate -o json | nostamp.py | jq .resources errcode $CLI bundle plan trace print_requests.py --nostamp //pipeline diff --git a/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json index d7c0cc7b12f..8dea931a43e 100644 --- a/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_a": { diff --git a/acceptance/bundle/resource_deps/permission_ref/script b/acceptance/bundle/resource_deps/permission_ref/script index 82800b1654e..ad897f76987 100644 --- a/acceptance/bundle/resource_deps/permission_ref/script +++ b/acceptance/bundle/resource_deps/permission_ref/script @@ -1,4 +1,4 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json index 84d1505045e..60da159159b 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.bar": { diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json index 7b3f986f02c..34c1baf3067 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json index 4f7329005fb..222f87d5afb 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/script b/acceptance/bundle/resource_deps/pipelines_recreate/script index 691e24279b2..b1a6741f081 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/script +++ b/acceptance/bundle/resource_deps/pipelines_recreate/script @@ -1,7 +1,7 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' > out.create.requests.json @@ -14,7 +14,7 @@ trace $CLI bundle plan # empty title "Update storage, triggering recreate for pipeline; this means updating downstream deps" trace update_file.py databricks.yml "storage: dbfs:/my-storage" "storage: dbfs:/my-new-storage" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --auto-approve trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -23,13 +23,13 @@ foo_id_2=`read_id.py foo` title "Fetch resource IDs and verify remote state" trace musterr $CLI pipelines get $foo_id -trace $CLI pipelines get $foo_id_2 | nostamp -trace $CLI jobs get $bar_id | nostamp| jq 'del(.settings.run_as)' +trace $CLI pipelines get $foo_id_2 | nostamp.py +trace $CLI jobs get $bar_id | nostamp.py| jq 'del(.settings.run_as)' rm out.requests.txt title "Follow up plan & deploy do nothing" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json index ef8c827e130..869b15ad6f7 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo1": { diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json index 0bb3be878f6..ac8af43c069 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/remote_pipeline/script b/acceptance/bundle/resource_deps/remote_pipeline/script index b8683429aac..4df5a844e73 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/script +++ b/acceptance/bundle/resource_deps/remote_pipeline/script @@ -3,8 +3,8 @@ print_requests() { rm out.requests.txt } -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests -$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests diff --git a/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json b/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json index bfa6735a725..6f4ff29a254 100644 --- a/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.mypipeline": { diff --git a/acceptance/bundle/resource_deps/resources_var/script b/acceptance/bundle/resource_deps/resources_var/script index cc0db790b5c..c04b243240b 100644 --- a/acceptance/bundle/resource_deps/resources_var/script +++ b/acceptance/bundle/resource_deps/resources_var/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -t dev -o json | jq .resources -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy -t dev &> out.deploy.txt trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt trace print_telemetry_bool_values | grep -v direct_drymigrate diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json index 748830efbe4..cb7acf0c2d8 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.mypipeline": { diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script index 707ca1f347f..2fb906b4945 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -t dev -o json | jq .resources -$CLI bundle plan -o json -t dev > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t dev trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json b/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json index 37b90504345..fed77573273 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.dst_git": { diff --git a/acceptance/bundle/resource_deps/tf_path_renames/script b/acceptance/bundle/resource_deps/tf_path_renames/script index 568894d3ba9..dc491ff0f5e 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/script +++ b/acceptance/bundle/resource_deps/tf_path_renames/script @@ -1,3 +1,3 @@ -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs | jq -rs '[.[].body.name] | sort[]' diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json index 1fecd0a7b5a..081c3abd05d 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.volumes.bar": { diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/script b/acceptance/bundle/resource_deps/volume_path_contains_id/script index 914f75443d2..21d91ccd863 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/script +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/script @@ -5,7 +5,7 @@ cleanup() { trace $CLI bundle validate -o json | jq .resources trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trap cleanup EXIT trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json index a6cab2ee51e..9bca8261612 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json @@ -1,7 +1,7 @@ >>> errcode [CLI] bundle plan -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script index 6c7250cc564..21a045fa49c 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/script +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -1,5 +1,5 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs, .resources.volumes' +trace $CLI bundle validate -o json | nostamp.py | jq '.resources.jobs, .resources.volumes' # The job's data_path default references the volume's computed volume_path. # Record the JSON plan per engine to confirm the reference is resolved into the diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml index b53e5e7fbe8..10ce9d961a6 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml @@ -6,3 +6,7 @@ RecordRequests = true # so the case is unsupported there (see CHANGELOG.md). Direct is the # default engine, so the motivating use case works out of the box. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py +# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/script b/acceptance/bundle/resources/apps/config-drift-stopped/script index c449a3099a3..50e046d9442 100644 --- a/acceptance/bundle/resources/apps/config-drift-stopped/script +++ b/acceptance/bundle/resources/apps/config-drift-stopped/script @@ -10,10 +10,10 @@ trace $CLI bundle deploy title "Change config while running: drift is detected (active deployment present)" trace update_file.py databricks.yml original_value changed_value -trace $CLI bundle plan -o json | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' title "Stop the app: the backend clears the active deployment" trace $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state' title "Same config change is now skipped: no active deployment to compare against" -trace $CLI bundle plan -o json | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' diff --git a/acceptance/bundle/resources/apps/config-drift/script b/acceptance/bundle/resources/apps/config-drift/script index 4728255dc8f..23154a3fdf9 100644 --- a/acceptance/bundle/resources/apps/config-drift/script +++ b/acceptance/bundle/resources/apps/config-drift/script @@ -9,7 +9,7 @@ trap cleanup EXIT trace $CLI bundle deploy title "Verify no drift after deploy" -trace $CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' > out.plan.direct.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' > out.plan.direct.json SOURCE_CODE_PATH=$(trace $CLI apps get $UNIQUE_NAME --output json | jq -r '.active_deployment.source_code_path') @@ -27,13 +27,13 @@ $CLI apps deploy $UNIQUE_NAME --no-wait --json '{ title "Plan should detect config drift" trace $CLI bundle plan # Skip entries with action "skip" -$CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json +$CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json title "Redeploy to fix drift" trace $CLI bundle deploy title "Verify no drift after fix" -trace $CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json # TODO: add test for git_source drift when git_source is supported in the Deploy API # Currently it fails with the error: Git source reference is required diff --git a/acceptance/bundle/resources/apps/config-no-deployment/script b/acceptance/bundle/resources/apps/config-no-deployment/script index f1970999645..3b7e38e3493 100644 --- a/acceptance/bundle/resources/apps/config-no-deployment/script +++ b/acceptance/bundle/resources/apps/config-no-deployment/script @@ -7,4 +7,4 @@ title "Plan is a no-op: config/source_code_path drift is skipped while the app h trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "Both deploy-only fields are skipped with reason \"no active deployment\"" -trace $CLI bundle plan -o json | jq '.plan[].changes | {config, source_code_path}' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | {config, source_code_path}' diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/script b/acceptance/bundle/resources/apps/git-source-no-deployment/script index 63cce6312c3..28e12809b2e 100644 --- a/acceptance/bundle/resources/apps/git-source-no-deployment/script +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/script @@ -7,4 +7,4 @@ title "Plan is a no-op: git_source drift is skipped while the app has no active trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "git_source is skipped with reason \"no active deployment\"" -trace $CLI bundle plan -o json | jq '.plan[].changes | {git_source}' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | {git_source}' diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script index dd08f33fff5..8bdbb4dfd9c 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script @@ -79,6 +79,6 @@ trace $CLI bundle deploy trace print_app_requests title "(started omitted, app running) -> bundle plan shows no drift" -$CLI bundle plan -o json | nostamp > LOG.planjson +$CLI bundle plan -o json | nostamp.py > LOG.planjson verify_no_drift.py LOG.planjson echo "Plan: no drift detected" diff --git a/acceptance/bundle/resources/apps/uc-securable-drift/script b/acceptance/bundle/resources/apps/uc-securable-drift/script index 04438951475..7f3022a2bb1 100644 --- a/acceptance/bundle/resources/apps/uc-securable-drift/script +++ b/acceptance/bundle/resources/apps/uc-securable-drift/script @@ -10,4 +10,4 @@ title "Re-plan is clean: the TABLE securable's server-computed securable_kind is trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "securable_kind is classified output-only (skip); the VOLUME securable has none" -trace $CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes // {} | with_entries(select(.key | test("uc_securable")))' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes // {} | with_entries(select(.key | test("uc_securable")))' diff --git a/acceptance/bundle/resources/catalogs/comment_out_of_band/script b/acceptance/bundle/resources/catalogs/comment_out_of_band/script index fe406882746..c0f21c30082 100644 --- a/acceptance/bundle/resources/catalogs/comment_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/catalogs/$CATALOG" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the catalog" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/script b/acceptance/bundle/resources/catalogs/drift/managed_properties/script index 9413632bb9c..25664d4eefd 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/script +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite UC auto-populating managed properties" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only properties map is skipped as a backend default (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy is a no-op (no UpdateCatalog call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json index c7af4b9a333..cdf7731c99b 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.clusters.test_cluster": { @@ -17,7 +17,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -97,7 +97,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, @@ -180,7 +180,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, @@ -265,7 +265,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 4, diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index 71d7b6860af..00614a0ec10 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -10,11 +10,11 @@ trap cleanup EXIT # resize, terraform update), so the per-resource line would diverge. The Resources: # summary agrees, so it is kept rather than using -qq. $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q title "Cluster should exist with num_workers after bundle deployment:\n" -CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') +CLUSTER_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.clusters.test_cluster.id') add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' @@ -26,7 +26,7 @@ update_file.py databricks.yml " num_workers: 2" " autoscale: min_workers: 2 max_workers: 4" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -38,7 +38,7 @@ title "Changing autoscale should call update API on stopped cluster\n" update_file.py databricks.yml "min_workers: 2" "min_workers: 3" update_file.py databricks.yml "max_workers: 4" "max_workers: 5" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -53,7 +53,7 @@ title "Changing autoscale should call resize API on running cluster\n" update_file.py databricks.yml "min_workers: 3" "min_workers: 4" update_file.py databricks.yml "max_workers: 5" "max_workers: 6" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt @@ -66,7 +66,7 @@ update_file.py databricks.yml " autoscale: min_workers: 4 max_workers: 6" " num_workers: 3" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json index b4ba1c76152..9d3e514ecfe 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.clusters.test_cluster": { @@ -20,7 +20,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -92,7 +92,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, @@ -164,7 +164,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index 14799220348..7643752ef88 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -10,18 +10,18 @@ trap cleanup EXIT # resize, terraform update), so the per-resource line would diverge. The Resources: # summary agrees, so it is kept rather than using -qq. $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q title "Cluster should exist after bundle deployment:\n" -CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') +CLUSTER_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.clusters.test_cluster.id') add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' title "Changing num_workers should call update API on stopped cluster\n" update_file.py databricks.yml "num_workers: 2" "num_workers: 3" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -35,7 +35,7 @@ $CLI clusters start "${CLUSTER_ID}" title "Changing num_workers should call resize API on running cluster\n" update_file.py databricks.yml "num_workers: 3" "num_workers: 4" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt @@ -47,7 +47,7 @@ title "Changing num_workers and spark_conf should call update API\n" update_file.py databricks.yml "num_workers: 4" "num_workers: 5" update_file.py databricks.yml '"spark.executor.memory": "2g"' '"spark.executor.memory": "4g"' $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json index e14ec2fba2a..8b496224317 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml index bf4f0e7d3c6..d5b9a6f4a16 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/script b/acceptance/bundle/resources/dashboards/change-serialized-inline/script index 03b8c2ac46b..41629784e3f 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/script +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/script @@ -23,7 +23,7 @@ update_file.py databricks.yml "Page 1" "Page One" trace $CLI bundle plan | contains.py "update" "!recreate" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json title "deploy the change\n" $CLI bundle deploy $(readplanarg out.plan.$DATABRICKS_BUNDLE_ENGINE.json) diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml b/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml index 5cab5faff69..fae8d4e5d0e 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml @@ -8,3 +8,7 @@ Ignore = [ ] EnvMatrix.READPLAN = ["", "1"] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json index 01e4ab17548..7686609f862 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script index aa3aa4026b7..e5a1a51185a 100755 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script @@ -29,7 +29,7 @@ echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path}' echo "$DASHBOARD" | jq '.serialized_dashboard | fromjson | .datasets[] | {catalog, schema}' # Verify that there is no drift right after deploy. -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Modify the direct plan to replace "serialized_dashboard" with a fixture. # It is normalized on the backend so we cannot compare reliably across local and cloud. diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json index 6cedd195b0f..c4d8bc38fd9 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script index 2c1396ac618..b8928da1198 100755 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script @@ -25,7 +25,7 @@ trace $CLI lakeview trash $DASHBOARD_ID trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Run destroy - should succeed even though dashboard is already trashed. # -qq drops the summary: terraform refreshes the trashed dashboard away and reports diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json index c177eb0b450..e11fc5079b5 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/detect-change/script b/acceptance/bundle/resources/dashboards/detect-change/script index 2b4966fd87d..d7c2d18037b 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/script +++ b/acceptance/bundle/resources/dashboards/detect-change/script @@ -38,7 +38,7 @@ add_repl "$(echo "$UPDATE_RESP" | jq -r '.etag')" ETAG_2 title "Try to redeploy the bundle and confirm that the out of band modification is detected:" trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy title "Redeploy the bundle with the --force flag and confirm that the out of band modification is ignored:" diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml index aad5ccf5498..bd29c0328af 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/test.toml index 96b263c004a..ca3a5884557 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/test.toml @@ -3,3 +3,6 @@ RecordRequests = false RunsOnDbr = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + +# bundle generate reading recorded state from DMS is a fast followup; opt out of recording. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script index 41022465670..4c455c0c3f5 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script @@ -16,7 +16,7 @@ ETAG_1=$(echo "$DASHBOARD" | jq -r '.etag') add_repl "$ETAG_1" ETAG_1 echo "$DASHBOARD" | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' -trace $CLI bundle plan -o json | gron.py | grep -E "etag|published" +trace $CLI bundle plan -o json | nostamp.py | gron.py | grep -E "etag|published" rm out.requests.txt # Inject a single publish failure for the update below. @@ -34,7 +34,7 @@ ETAG_2=$(retry --until-not "$ETAG_1" $CLI lakeview get $DASHBOARD_ID | jq -r '.e add_repl "$ETAG_2" ETAG_2 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' -trace $CLI bundle plan -o json | gron.py | grep -E "etag|published" +trace $CLI bundle plan -o json | nostamp.py | gron.py | grep -E "etag|published" # Bug: re-running deploy fails with "modified remotely" because the stored etag # (pre-PATCH) no longer matches the remote etag (bumped by the PATCH above). diff --git a/acceptance/bundle/resources/dashboards/republish-after-draft-update/script b/acceptance/bundle/resources/dashboards/republish-after-draft-update/script index 9cd9a12869f..cd278bab258 100644 --- a/acceptance/bundle/resources/dashboards/republish-after-draft-update/script +++ b/acceptance/bundle/resources/dashboards/republish-after-draft-update/script @@ -16,7 +16,7 @@ DASHBOARD_ID=$(read_id.py dashboard1) # A clean re-plan is a no-op: the published content matches the draft. title "Plan right after deploy -- published is current, no change:" -trace $CLI bundle plan -o json | jq '.plan["resources.dashboards.dashboard1"].changes.published' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.dashboards.dashboard1"].changes.published' # Make an out-of-band DRAFT update without republishing. This advances the draft's # update_time past the published revision_create_time, so the published content is now @@ -28,4 +28,4 @@ DASHBOARD_JSON="{\"serialized_dashboard\": \"{}\", \"warehouse_id\": \"$TEST_DEF $CLI lakeview update "$DASHBOARD_ID" --json "${DASHBOARD_JSON}" > /dev/null title "Plan after out-of-band draft update -- a republish is owed:" -trace $CLI bundle plan -o json | jq '.plan["resources.dashboards.dashboard1"].changes.published' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.dashboards.dashboard1"].changes.published' diff --git a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json index 5e6ad9eb0a6..ad1171713f2 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/simple/script b/acceptance/bundle/resources/dashboards/simple/script index 4c8b9b55ace..6e2b14529f1 100644 --- a/acceptance/bundle/resources/dashboards/simple/script +++ b/acceptance/bundle/resources/dashboards/simple/script @@ -24,4 +24,4 @@ echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path, serialized_dashboar # Verify that there is no drift right after deploy. We use |= to update the serialized_dashboard field in place to account # for formatting that the cloud server applies to the serialized_dashboard field. -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json index 852cc492879..cd96018988d 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json index b3efa15d5da..42a8d2983bf 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.dashboards.dashboard1": { diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script index c3af11af2f7..ddb2b9e8b34 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.plan_initial.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_initial.$DATABRICKS_BUNDLE_ENGINE.json # Deploy the dashboard trace $CLI bundle deploy -qq @@ -25,7 +25,7 @@ trace $CLI lakeview unpublish $DASHBOARD_ID # Direct: shows "update" because Published field changes from false to true trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json json_in_json_normalize.py out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -qq diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json index 02b174d3869..60aedcde18e 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/genie_spaces/inline/script b/acceptance/bundle/resources/genie_spaces/inline/script index 812cfb9abd4..9e5892a1416 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/script +++ b/acceptance/bundle/resources/genie_spaces/inline/script @@ -15,4 +15,4 @@ add_repl "$GENIE_SPACE_ID" GENIE_SPACE_ID # Without normalization the inline serialized_space leaves a map in the # config struct while state holds a string, and structdiff reports false # drift on every plan. -trace $CLI bundle plan -o json > out.plan.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/version_migration/script b/acceptance/bundle/resources/genie_spaces/version_migration/script index 141efd5bef6..5615bbb7034 100644 --- a/acceptance/bundle/resources/genie_spaces/version_migration/script +++ b/acceptance/bundle/resources/genie_spaces/version_migration/script @@ -19,7 +19,7 @@ trace $CLI bundle plan # (and the etag guarding it) is version 2. The serialized_space change is # recorded but skipped via the etag_based rule. title "serialized_space: local version 1, remote version 2, skipped (etag_based)" -$CLI bundle plan -o json \ +$CLI bundle plan -o json | nostamp.py \ | jq '.plan["resources.genie_spaces.foo"].changes.serialized_space | {action, reason, local_version: (.new | fromjson | .version), diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json index 51f6d220e56..008571de908 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.catalogs.grants_catalog": { diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json index fa674375af2..afefaf2f192 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/catalogs/script b/acceptance/bundle/resources/grants/catalogs/script index 067710b705e..aa47ee0ce94 100755 --- a/acceptance/bundle/resources/grants/catalogs/script +++ b/acceptance/bundle/resources/grants/catalogs/script @@ -1,7 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -13,7 +13,7 @@ trace $CLI grants get catalog catalog_grants_$UNIQUE_NAME | jq --sort-keys update_file.py databricks.yml CREATE_SCHEMA USE_SCHEMA -trace $CLI bundle plan -o json | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json index ab6c291b239..66689f09124 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.registered_models.my_registered_model": { diff --git a/acceptance/bundle/resources/grants/registered_models/script b/acceptance/bundle/resources/grants/registered_models/script index 98d73773afc..43ec2d3b87a 100644 --- a/acceptance/bundle/resources/grants/registered_models/script +++ b/acceptance/bundle/resources/grants/registered_models/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json index 534d0328aa1..429898e1b67 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.schemas.grants_schema": { diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json index d8bbf97ffd6..46444558c43 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/script b/acceptance/bundle/resources/grants/schemas/change_privilege/script index 737f7491d3a..ec907ef8887 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/script +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/script @@ -1,7 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -13,7 +13,7 @@ trace $CLI grants get schema main.schema_grants_$UNIQUE_NAME | jq --sort-keys update_file.py databricks.yml USE_SCHEMA APPLY_TAG -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json index ff320b18213..f05a65c1a5d 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/script b/acceptance/bundle/resources/grants/schemas/duplicate_principals/script index 12a255773b1..1056a114bfb 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/script +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/script @@ -11,4 +11,4 @@ trace $CLI bundle deploy print_requests.py --get //permissions --keep > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt # effective_predictive_optimization_flag is inherited from the metastore and backend-controlled; drop it so the test does not depend on metastore settings. # updated_at/updated_by reflect UC's post-create system write that populates managed defaults, so on managed-defaults clouds they show a system principal rather than the creator; drop them. -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json index cc1ae0e9cf3..bc969eb2f1c 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json index bf38acd93fe..8021983f5b5 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script index 6f8965cfe1a..fe3b395156d 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script @@ -12,7 +12,7 @@ trap cleanup EXIT trace $CLI bundle deploy trace $CLI grants update schema "$SCHEMA_FULL_NAME" --json @update.json > /dev/null $CLI grants get schema "$SCHEMA_FULL_NAME" | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com' > /dev/null -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # remote_state.__embed__ order is non-deterministic; extract and sort separately into per-engine file jq '.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | gron.py --noindex | sort_lines.py --repl > out.embed.$DATABRICKS_BUNDLE_ENGINE.txt tmp=$(mktemp) @@ -21,4 +21,4 @@ tmp=$(mktemp) # updated_at/updated_by reflect UC's post-create system write that populates managed defaults, so on managed-defaults clouds they show a system principal rather than the creator; drop them. jq 'del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__)' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > "$tmp" && mv "$tmp" out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json index f49832eddc9..4e059e5634b 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.schemas.grants_schema": { diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json index 1daacee25b4..9042bde70e8 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/volumes/script b/acceptance/bundle/resources/grants/volumes/script index 725049cda8e..9b8854c8062 100644 --- a/acceptance/bundle/resources/grants/volumes/script +++ b/acceptance/bundle/resources/grants/volumes/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -10,7 +10,7 @@ trace $CLI grants get volume main.schema_grants_$UNIQUE_NAME.volume_name | jq -- update_file.py databricks.yml WRITE_VOLUME MANAGE # different on cloud vs local due to remote state -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by volume_id > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by volume_id > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/job_runs/failed_run/script b/acceptance/bundle/resources/job_runs/failed_run/script index 0fe24556182..2c1ee0fa812 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/script +++ b/acceptance/bundle/resources/job_runs/failed_run/script @@ -15,7 +15,7 @@ musterr trace $CLI bundle deploy # The framework saves the run id before WaitAfterCreate, so FAILED vs SUCCESS is drift. title "the failed run is recorded, and not having succeeded is drift" trace read_id.py my_run -trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"]' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.job_runs.my_run"]' # Name the job too, so the run-now bodies below say which number it is. read_id.py my_job > /dev/null diff --git a/acceptance/bundle/resources/job_runs/redeploy/script b/acceptance/bundle/resources/job_runs/redeploy/script index 8ceb3a88052..2c969f0a969 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/script +++ b/acceptance/bundle/resources/job_runs/redeploy/script @@ -17,7 +17,7 @@ trace print_requests.py //jobs/run-now title "change the run configuration and redeploy" trace update_file.py databricks.yml "env: dev" "env: prod" -trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"]' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.job_runs.my_run"]' trace $CLI bundle deploy read_id.py my_run > /dev/null trace $CLI bundle summary diff --git a/acceptance/bundle/resources/job_runs/wait/script b/acceptance/bundle/resources/job_runs/wait/script index 70cee1c9082..3a68150df3b 100644 --- a/acceptance/bundle/resources/job_runs/wait/script +++ b/acceptance/bundle/resources/job_runs/wait/script @@ -15,5 +15,5 @@ downstream_id=$(read_id.py downstream_job) trace $CLI jobs get $downstream_id -o json | jq -r ".settings.tags.run_result" title "nothing the run resolved is drift, so a redeploy starts no new run" -trace $CLI bundle plan -o json | jq -r '.plan | to_entries[] | "\(.key) \(.value.action)"' +trace $CLI bundle plan -o json | nostamp.py | jq -r '.plan | to_entries[] | "\(.key) \(.value.action)"' trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json b/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json index 6328c4f6dd7..aaf9e6c7458 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script index b5823022572..00188cda82d 100644 --- a/acceptance/bundle/resources/jobs/big_id/script +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -o json | jq .resources > out.validate.json -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan.direct.json) trace print_requests.py //jobs --nostamp print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/big_id/test.toml b/acceptance/bundle/resources/jobs/big_id/test.toml index b15d8f9e757..8957ed9ce06 100644 --- a/acceptance/bundle/resources/jobs/big_id/test.toml +++ b/acceptance/bundle/resources/jobs/big_id/test.toml @@ -3,8 +3,8 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ['direct'] EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] [[Repls]] diff --git a/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json b/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json index f5f1717411e..126e0cbda1b 100644 --- a/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index 1622b3e11cb..f1bf01f596d 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy cp empty.yml databricks.yml -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json b/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json index a881f097c7a..0539fa4c2de 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json +++ b/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json index 7a64f0a18d6..7e37e4e09e1 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/delete_task/script b/acceptance/bundle/resources/jobs/delete_task/script index 29b856e5ceb..89cf79be7e3 100644 --- a/acceptance/bundle/resources/jobs/delete_task/script +++ b/acceptance/bundle/resources/jobs/delete_task/script @@ -1,6 +1,6 @@ -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) grep -v TO_DELETE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) diff --git a/acceptance/bundle/resources/jobs/delete_task/test.toml b/acceptance/bundle/resources/jobs/delete_task/test.toml index 46a4d91b848..04d6ea1e5b3 100644 --- a/acceptance/bundle/resources/jobs/delete_task/test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/test.toml @@ -1,5 +1,5 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/script b/acceptance/bundle/resources/jobs/on_failure_empty_slice/script index 6f0606bc346..55c9b9bbf82 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/script +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/script @@ -1,3 +1,3 @@ trace $CLI bundle plan trace $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.refresh_usage_logs".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.refresh_usage_logs".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json b/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json index 693492859fc..052acba00b3 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json +++ b/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/script b/acceptance/bundle/resources/jobs/remote_add_tag/script index 3a651737979..b2b2fefd0ba 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/script +++ b/acceptance/bundle/resources/jobs/remote_add_tag/script @@ -8,4 +8,4 @@ r["tags"]["new_tag"] = "new_value" EOF $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json index c9493e8a21c..796dbf0528b 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml index 46a4d91b848..04d6ea1e5b3 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml @@ -1,5 +1,5 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json index 3af240a0ed0..94e8d4554e1 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index 9ca6a0655b6..b6ed653c0ef 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -13,7 +13,7 @@ r["max_concurrent_runs"] = 2 EOF trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt # XXX READPLAN diff --git a/acceptance/bundle/resources/jobs/tags_empty_map/script b/acceptance/bundle/resources/jobs/tags_empty_map/script index 4a232723bd0..d3dd04214cf 100644 --- a/acceptance/bundle/resources/jobs/tags_empty_map/script +++ b/acceptance/bundle/resources/jobs/tags_empty_map/script @@ -1,3 +1,3 @@ trace $CLI bundle plan trace $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.test_job".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.test_job".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json b/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json index e13d1b307a5..4a7b31e1946 100644 --- a/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json b/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json index 7aecad7102e..87975ef1bc0 100644 --- a/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json index 56eb1211e99..e81f442dbab 100644 --- a/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/script b/acceptance/bundle/resources/jobs/update/script index 1be869aa509..7a90cf02255 100644 --- a/acceptance/bundle/resources/jobs/update/script +++ b/acceptance/bundle/resources/jobs/update/script @@ -1,12 +1,12 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) trace print_requests.py //jobs --nostamp > out.create.requests.json print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_skip.direct.json) trace print_requests.py //jobs --nostamp @@ -14,7 +14,7 @@ trace print_requests.py //jobs --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) trace print_requests.py //jobs --nostamp | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json diff --git a/acceptance/bundle/resources/jobs/update/test.toml b/acceptance/bundle/resources/jobs/update/test.toml index 20c8733641d..a8a43da097a 100644 --- a/acceptance/bundle/resources/jobs/update/test.toml +++ b/acceptance/bundle/resources/jobs/update/test.toml @@ -1,4 +1,4 @@ EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json b/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json index e42c7b24c5f..622487a2539 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json +++ b/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json b/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json index bed35fb5c8c..7e6179fd905 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json +++ b/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json index 330ae5d5c17..3f616a3754e 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update_single_node/script b/acceptance/bundle/resources/jobs/update_single_node/script index 4deefe32bbc..bfeb03571f8 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/script +++ b/acceptance/bundle/resources/jobs/update_single_node/script @@ -1,15 +1,15 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py //jobs > out.create.requests.txt --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --nostamp diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index c0257db6330..b3dbd3b782c 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -14,6 +14,6 @@ EOF # The reordered remote must not produce a phantom diff: on_* lists are diffed by id. trace $CLI bundle plan -$CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt b/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt index c1d4fe55d6e..f8f72b2d600 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.plan.resources.model_serving_endpoints.my_endpoint.action = "create"; json.plan.resources.model_serving_endpoints.my_endpoint.new_state.value.name = "[ENDPOINT_NAME_1]"; diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt b/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt index 7b4362bacbe..79919714725 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.lineage = "[UUID]"; json.serial = 1; diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/script b/acceptance/bundle/resources/model_serving_endpoints/basic/script index 972439f6813..48b1f935b0c 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/script +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/script @@ -14,7 +14,7 @@ trap cleanup EXIT # bundle/direct/dresources/permissions.go), so sorting keeps the plan golden # stable across runs. Same approach as secret_scopes/basic and the requests # sorting below. -trace $CLI bundle plan -o json | gron.py --sort-arrays __embed__ > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays __embed__ > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq # Sort access_control_list: the terraform engine stores it as a set whose order # depends on the (unmasked) principal value, so it differs between the local fake @@ -33,7 +33,7 @@ echo "$get_output" | jq '{name, creator}' export ENDPOINT_NAME="test-endpoint-$UNIQUE_NAME-2" envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json | gron.py --sort-arrays __embed__ > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays __embed__ > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq print_requests.py //serving-endpoints | gron.py --sort-arrays access_control_list > out.second-requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script index 5e30339c21f..7faa6fd31f3 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script @@ -7,7 +7,7 @@ title "Configure telemetry out of band" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.0/serving-endpoints/test-endpoint/telemetry-config" --json '{"telemetry_config": {"table_names": {"logs_table": "main.default.other_logs"}}}' > /dev/null title "Remote-only telemetry_config is not drift" -trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' title "Deploy does not remove it" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script index dab44354e0e..03058a0cf1b 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite write-only secret and burst_scaling_enabled not r trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The write-only fields are skipped as input_only (confirms the matched rules)" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy is a no-op (no config/ai-gateway/tags update calls)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json index a9353ddba55..79a8a77c640 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json index 86932f72ce6..2143567be39 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script index 919470c4ca4..a37c37a2df5 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.catalog_name' trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other_catalog" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json index c8e90eb2aad..b48b79aecf3 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json index 3e424ed3fc8..e44a5467be8 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script index d48a60c0ace..c27e1968877 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.name' trace update_file.py databricks.yml "name: test-endpoint-$UNIQUE_NAME" "name: test-endpoint-2-$UNIQUE_NAME" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json index 32dd192daab..3e58aa755c8 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json index f8dcb1c5510..19ed3fca2b1 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script index 0b68be7c353..db80789d106 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') @@ -16,7 +16,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '{route_optimized, id}' > print_requests.py //serving-endpoints > out.first-requests.$DATABRICKS_BUNDLE_ENGINE.json trace update_file.py databricks.yml "route_optimized: false" "route_optimized: true" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json index a9353ddba55..79a8a77c640 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json index eeacb3be087..f7b210772cd 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script index 0e31f7a4848..cc76c4428c6 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.schema_name' trace update_file.py databricks.yml "schema_name: default" "schema_name: other_schema" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json index a9353ddba55..79a8a77c640 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json index e29d17e4dde..058747f3d60 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script index 7f915920ba1..079bc9a33c9 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.table_name_prefix' trace update_file.py databricks.yml "table_name_prefix: my_table" "table_name_prefix: other_table" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt index 6c3233c9ede..71522c2d87e 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.my_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script index 8b420559c3d..8f2e49038ed 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script @@ -8,7 +8,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json index f305ce97719..3f93160a2ab 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script index 6e556d2b1ab..3f942f4d306 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.ai_gateway' trace update_file.py databricks.yml 'catalog_name: "first-inference-catalog"' 'catalog_name: "second-inference-catalog"' -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json index 394377f07f8..32f88590a01 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script index 763379ad9a6..c9b81cd0d6d 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script @@ -17,7 +17,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config' trace update_file.py databricks.yml "value: my-team-one" "value: my-team-two" trace update_file.py databricks.yml 'catalog_name: "first-inference-catalog"' 'catalog_name: "second-inference-catalog"' -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json index 47e992c1cf9..b4d67db672b 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/script b/acceptance/bundle/resources/model_serving_endpoints/update/config/script index 3a28d1dcda2..c4656a110f8 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config' trace update_file.py databricks.yml "name: gpt-4o-mini" "name: gpt-5o-mini" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json index cf8c8b3f0ca..1ff5746e670 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script index 391d369d686..f049becec06 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script @@ -16,7 +16,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.email_notifications' # Update email_notifications - change the email address trace update_file.py databricks.yml "user1@example.com" "user2@example.com" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "There a bug in TF where it does not actually update the email notifications for a serving endpoint." diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json index 73202db539d..4e1999adb7e 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/script b/acceptance/bundle/resources/model_serving_endpoints/update/tags/script index 6f69dc2f063..01c0e26eedd 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.tags' trace update_file.py databricks.yml "value: my-team-one" "value: my-team-two" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/permissions/_script b/acceptance/bundle/resources/permissions/_script index 8f6a0d9c8cd..df20dfd2fa9 100644 --- a/acceptance/bundle/resources/permissions/_script +++ b/acceptance/bundle/resources/permissions/_script @@ -1,7 +1,7 @@ -trace $CLI bundle validate -o json | nostamp | jq .resources.$RESOURCE.foo.permissions +trace $CLI bundle validate -o json | nostamp.py | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json print_requests() { jq -c < out.requests.txt | jq 'select(.method != "GET" and (.path | contains("permissions")) and (.path | contains("/api/2.0/bundle") | not))' diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json index 6774f270231..718499a0f66 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.apps.foo": { diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json index 6774f270231..718499a0f66 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.apps.foo": { diff --git a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json index bd13e780112..84f6bcd2feb 100644 --- a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.clusters.foo": { diff --git a/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json b/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json index 423fa9d0eb8..5793e74dd90 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.dashboards.foo": { diff --git a/acceptance/bundle/resources/permissions/dashboards/create/script b/acceptance/bundle/resources/permissions/dashboards/create/script index 41f26f63604..2d578386e6a 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/script +++ b/acceptance/bundle/resources/permissions/dashboards/create/script @@ -7,7 +7,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -o json | jq .resources.dashboards.foo.permissions rm out.requests.txt -$CLI bundle plan --output json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json index cc7dfa8e463..35dfda54426 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.database_instances.foo": { diff --git a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json index 6ce406b3d28..de8c34ac0ca 100644 --- a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.experiments.foo": { diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json index 90736028eb4..fd4fc45a826 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.genie_spaces.foo": { diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script index e589a40bd5b..e681e725cde 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json index f048f3e6f48..7185b30f356 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/script b/acceptance/bundle/resources/permissions/jobs/added_remotely/script index e85bcde5379..3b872d6446b 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/script +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/script @@ -10,7 +10,7 @@ title "Add permissions out of band" trace $CLI permissions set jobs "$job_id" --json @remote_add.json trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json index b85fdd51a06..9b131bdc4af 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt index 4b289dbd158..d3c2e3afdca 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script index 3c8217c195c..529b94814fa 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script @@ -1,11 +1,11 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle validate -t green -o json | nostamp | jq .resources +trace $CLI bundle validate -t green -o json | nostamp.py | jq .resources trace errcode $CLI bundle deploy -t green print_requests.py //jobs --nostamp &> out.deploy.requests.json # check plan to ensure there is not drift -trace $CLI bundle plan -o json -t green | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json -t green | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI bundle destroy -t green --auto-approve print_requests.py //jobs --nostamp &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json index 88e1e115a9c..16944e5b4dd 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json index a72e11dc488..74cc6c46453 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/script b/acceptance/bundle/resources/permissions/jobs/delete_one/script index 1f0aa1c80a7..82cd9d0aa94 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/script +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/script @@ -8,7 +8,7 @@ if [ -n "$CLOUD_ENV" ]; then fi rm -f out.requests.txt -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json cleanup() { trace errcode $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json index bd43de04ba7..ceb3d6d81cf 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json index cda86e4ceb4..82626507c1a 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script index cce4d71d0ce..f91513db46e 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script @@ -1,4 +1,4 @@ -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_create.json @@ -11,7 +11,7 @@ title "Delete permissions remotely" trace $CLI permissions set jobs "$job_id" --json @remote_delete.json trace print_requests.py --nostamp //jobs/ -trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_restore.json diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml index 921045dfd09..4ca61be3f36 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml @@ -1,4 +1,3 @@ -# Recording needs a bundle it has seen from the start. This test seeds a state file, -# so recording refuses it. TODO(DMS): drop this once existing state can be -# handed over to the service (see the TODO in dstate.Open). +# Deploy runs as the service principal, later commands as the user - who resolves no deployment of +# its own, so bundle summary reads the job id back null. Recording needs state staged for the reader. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json index 0ec914589b1..d8a4d90a9b0 100644 --- a/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json index b85fdd51a06..9b131bdc4af 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json index 2976bb3ebe6..5d47f0cc8ec 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json index 42bd247c77b..993098a0ee6 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json index e7bfda922b8..f217c1f7185 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json index a2e93e298eb..5c1c6301698 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json index 02ffe2db63a..0402134e14c 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json index 1b25b762637..49058decfa6 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 4, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json index 57ee99c53f4..2c6d989004f 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json index 89b3cf0f302..8cce904f66b 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/update/script b/acceptance/bundle/resources/permissions/jobs/update/script index 28f2c121f2b..fb88499c271 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/script +++ b/acceptance/bundle/resources/permissions/jobs/update/script @@ -1,10 +1,10 @@ cp databricks.yml databricks.yml.saved trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_create.json trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan_post_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_post_create.$DATABRICKS_BUNDLE_ENGINE.json job_id="$(read_id.py job_with_permissions)" @@ -13,21 +13,21 @@ rm -f out.requests.txt title "Update one permission and deploy again\n" update_file.py databricks.yml CAN_VIEW CAN_MANAGE -trace $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_update.json trace $CLI bundle plan title "Delete one permission and deploy again\n" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_delete_one.json trace $CLI bundle plan title "Delete the whole block and deploy again\n" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan @@ -36,7 +36,7 @@ title "Restore original config\n" trace diff.py databricks.yml.saved databricks.yml mv databricks.yml.saved databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_restore_original.json trace $CLI bundle plan @@ -44,7 +44,7 @@ trace $CLI bundle plan title "Set permissions: []\n" grep -vE '(PERMISSIONS|DELETE_ONE)' databricks.yml > tmp.yml && mv tmp.yml databricks.yml update_file.py databricks.yml '# permissions: [] # EXPLICIT_EMPTY' 'permissions: [] # EXPLICIT_EMPTY' -trace $CLI bundle plan -o json | nostamp > out.plan_set_empty.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_set_empty.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_set_empty.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json index 151138ddc8b..a783dd0aa59 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json index 268172af1af..ad2ad5b9245 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.models.foo": { diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/script b/acceptance/bundle/resources/permissions/models/current_can_manage/script index 0a6935c20b8..8159841e4e3 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json index 41c0fc3ba64..38cad367876 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json index 029250bd21a..56c72cff931 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json index 97493239403..8daa49692ad 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json index 41c0fc3ba64..38cad367876 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json index 240341258e8..e696913fa49 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json index 2ee78e63631..ba600d05faa 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json index db74caabfca..ac3785d797b 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json index 4586f292b32..c3518ec2310 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json index 7e20625e706..6ee152943b7 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 4, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json index f4419563280..a43fcd03904 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/script b/acceptance/bundle/resources/permissions/pipelines/update/script index 9ace44addb4..33224b30de3 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/script +++ b/acceptance/bundle/resources/permissions/pipelines/update/script @@ -1,5 +1,5 @@ cp databricks.yml databricks.yml.saved -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_create.json trace $CLI bundle plan @@ -11,14 +11,14 @@ rm -f out.requests.txt title "Update one permission and deploy again\n" update_file.py databricks.yml CAN_VIEW CAN_MANAGE -trace $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_update.json trace $CLI bundle plan title "Delete one permission and deploy again\n" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_delete_one.json trace $CLI bundle plan @@ -26,14 +26,14 @@ trace $CLI bundle plan title "Delete the whole block and deploy again\n" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' --sort > out.requests_delete_all.json trace $CLI bundle plan title "Restore original config\n" mv databricks.yml.saved databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_restore_original.json trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json index 302eeb3c913..0a0293bf156 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.postgres_projects.foo": { diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json index 6720a13fdaa..f1f44e53bc6 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.sql_warehouses.foo": { diff --git a/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json b/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json index 71d7ae5cbef..61eb85a3f0b 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/target_permissions/script b/acceptance/bundle/resources/permissions/target_permissions/script index 7a443d8cc2d..a2aa063789d 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/script +++ b/acceptance/bundle/resources/permissions/target_permissions/script @@ -1,5 +1,5 @@ trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py --nostamp //jobs/ > out.requests_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json index eea6e75cb51..d49433933ee 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.vector_search_endpoints.foo": { diff --git a/acceptance/bundle/resources/pipelines/auto-approve/script b/acceptance/bundle/resources/pipelines/auto-approve/script index f0b6c4f9044..c98e2f8e5b3 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/script +++ b/acceptance/bundle/resources/pipelines/auto-approve/script @@ -9,12 +9,12 @@ trap cleanup EXIT trace $CLI bundle deploy title "Assert the pipeline is created" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.bar.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.bar.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" title "Assert the job is created" -JOB_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.jobs.foo.id') -$CLI jobs get "${JOB_ID}" | nostamp| jq '{name: .settings.name}' +JOB_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.jobs.foo.id') +$CLI jobs get "${JOB_ID}" | nostamp.py| jq '{name: .settings.name}' title "Remove resources from configuration." trace rm resources.yml diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script index a319f62fb18..79927ff6447 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script @@ -8,5 +8,5 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method == "POST") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method == "POST") | .body' out.requests.txt | nostamp.py rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/script b/acceptance/bundle/resources/pipelines/num-workers-zero/script index d23c1e063de..58d7db9fa5b 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/script +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/script @@ -1,3 +1,3 @@ trace $CLI bundle deploy > out.deploy.txt 2>&1 -trace errcode jq -s '.[] | select(.method == "POST" and (.path | contains("/pipelines")))' out.requests.txt | nostamp > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +trace errcode jq -s '.[] | select(.method == "POST" and (.path | contains("/pipelines")))' out.requests.txt | nostamp.py > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/_script b/acceptance/bundle/resources/pipelines/recreate-keys/_script index 45c53d86743..1121836f497 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/_script +++ b/acceptance/bundle/resources/pipelines/recreate-keys/_script @@ -2,7 +2,7 @@ trace cat databricks.yml touch foo.py touch bar.py trace $CLI bundle plan # should show 'create' -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy ppid1=`read_id.py my` @@ -10,7 +10,7 @@ ppid1=`read_id.py my` print_requests() { # Excludes /api/2.0/bundle: a DMS operation path embeds the resource key, so it also # contains "/pipelines". - jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp.py rm -f out.requests.txt } @@ -18,14 +18,14 @@ trace print_requests trace update_file.py databricks.yml $CONFIG_UPDATE trace $CLI bundle plan # should show 'recreate' -$CLI bundle plan -o json | nostamp > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --auto-approve trace print_requests title "Fetch pipeline ID and verify remote state" ppid2=`read_id.py my` -trace $CLI pipelines get $ppid2 | nostamp +trace $CLI pipelines get $ppid2 | nostamp.py title "Verify that original pipeline is gone" trace musterr $CLI pipelines get $ppid1 diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json index 1a34eb33dd4..03f9b803d55 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.my": { diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json index a2bf3b724c7..c15ac5eae0d 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json index d5bd316376b..7c768907755 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.my": { diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json index 6987c3ad761..da70dac7a17 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/pipelines/recreate/script b/acceptance/bundle/resources/pipelines/recreate/script index 0a814d08985..247959cb281 100644 --- a/acceptance/bundle/resources/pipelines/recreate/script +++ b/acceptance/bundle/resources/pipelines/recreate/script @@ -8,8 +8,8 @@ trap cleanup EXIT trace $CLI bundle deploy title "Assert the pipeline is created with catalog" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.foo.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" # Note: In Terraform provider v1.98.0+, changing catalog no longer triggers recreation. # We switch to using storage location instead, which still triggers recreation. diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json b/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json index c07de448831..cb852987772 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/script b/acceptance/bundle/resources/pipelines/remote_matches_config/script index 6a6a334022e..267aa0cc3b7 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/script +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/script @@ -15,6 +15,6 @@ r["run_as"] = {"user_name": "changed@example.test"} EOF trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/update/script b/acceptance/bundle/resources/pipelines/update/script index d9eb259cbb9..f23ae69f62a 100644 --- a/acceptance/bundle/resources/pipelines/update/script +++ b/acceptance/bundle/resources/pipelines/update/script @@ -21,7 +21,7 @@ rm out.requests.txt title "Fetch pipeline ID and verify remote state" ppid=`read_id.py my` -trace $CLI pipelines get $ppid | nostamp +trace $CLI pipelines get $ppid | nostamp.py rm out.requests.txt title "Destroy the pipeline and verify that it's removed from the state and from remote" diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script index 919fe6e4f4f..f743dc45177 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script @@ -16,7 +16,7 @@ trap cleanup EXIT # resources.json. The field is omitted from JSON when unset (omitempty + # ForceSendFields tracking), so an absent value falls back to "(unset)". get_purge() { - gron.py < .databricks/bundle/default/resources.json | grep 'postgres_branches.branch.state.purge_on_delete' || echo '(unset)' + print_state.py | gron.py | grep 'postgres_branches.branch.state.purge_on_delete' || echo '(unset)' } title "Step 1: deploy with purge_on_delete unset" diff --git a/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script b/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script index a65cb5ee98b..f86eb93b434 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script +++ b/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script @@ -13,7 +13,7 @@ rm -f out.requests.txt title "Change the fork point: source_branch_time is immutable, so this recreates" trace update_file.py databricks.yml "2026-01-01T00:00:00Z" "2026-02-01T00:00:00Z" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy --auto-approve trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/update_expire_time/script b/acceptance/bundle/resources/postgres_branches/update_expire_time/script index 6c393dc60f0..c35cbbe34f0 100644 --- a/acceptance/bundle/resources/postgres_branches/update_expire_time/script +++ b/acceptance/bundle/resources/postgres_branches/update_expire_time/script @@ -19,7 +19,7 @@ trace update_file.py databricks.yml "2030-01-01T00:00:00Z" "2031-06-15T12:00:00Z # Only terraform applies it: expire_time is a member of the # expiration oneof, and the direct engine masks it under its own name. See test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/script b/acceptance/bundle/resources/postgres_branches/update_protected/script index 1566c885831..7235a56c82c 100755 --- a/acceptance/bundle/resources/postgres_branches/update_protected/script +++ b/acceptance/bundle/resources/postgres_branches/update_protected/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_branches.dev_branch" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -28,7 +28,7 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields title "Verify no_change (no changes)" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -39,7 +39,7 @@ trace update_file.py databricks.yml "is_protected: false" "is_protected: true" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -51,7 +51,7 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields title "Restore is_protected to false" trace update_file.py databricks.yml "is_protected: true" "is_protected: false" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_branches/update_ttl/script b/acceptance/bundle/resources/postgres_branches/update_ttl/script index a450a54d745..ee3e60d1597 100644 --- a/acceptance/bundle/resources/postgres_branches/update_ttl/script +++ b/acceptance/bundle/resources/postgres_branches/update_ttl/script @@ -24,7 +24,7 @@ trace update_file.py databricks.yml "ttl: 604800s" "ttl: 259200s" # creation_time + ttl and never echoes ttl, which the testserver does not model. # The update_mask each engine sends is what this test is about. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_databases/update/script b/acceptance/bundle/resources/postgres_databases/update/script index bdbad2b67c6..22677273763 100644 --- a/acceptance/bundle/resources/postgres_databases/update/script +++ b/acceptance/bundle/resources/postgres_databases/update/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -29,7 +29,7 @@ trace $CLI postgres get-database "${database_name}" | database_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -40,7 +40,7 @@ trace update_file.py databricks.yml "postgres_database: initial_db_name" "postgr trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -52,7 +52,7 @@ trace $CLI postgres get-database "${database_name}" | database_fields title "Restore postgres_database to original value" trace update_file.py databricks.yml "postgres_database: renamed_db_name" "postgres_database: initial_db_name" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script b/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script index 65696066385..43c5a6172d4 100644 --- a/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script +++ b/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script @@ -22,7 +22,7 @@ trace update_file.py databricks.yml " suspend_timeout_duration: 300s" "" # Terraform provider masks the whole spec, which the API accepts and then ignores. See # test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -32,4 +32,4 @@ trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//po # fail again on every deploy, terraform applied it and is clean. trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields trace $CLI bundle plan &> out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script index f2506c21a01..74bd7ed7077 100755 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -29,7 +29,7 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -40,7 +40,7 @@ trace update_file.py databricks.yml "autoscaling_limit_max_cu: 8" "autoscaling_l trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -52,7 +52,7 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields title "Restore endpoint autoscaling to original value" trace update_file.py databricks.yml "autoscaling_limit_max_cu: 4" "autoscaling_limit_max_cu: 8" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script index ca616cee3f1..e179f96bc5f 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script @@ -22,7 +22,7 @@ trace update_file.py databricks.yml "statement_timeout: 8000" "statement_timeout # spec.settings.pg_settings are both accepted, while an indexed path is answered with # 400 "Unknown field path in update_mask". trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script b/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script index 2f383859007..5b57e845759 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script @@ -26,7 +26,7 @@ title "Plan and deploy" # Both engines apply the change; only the mask they send differs, so out.requests # is the one per-engine file left. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script index 1bdc077e6e1..958d4e2d980 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script @@ -12,7 +12,7 @@ trap cleanup EXIT # resources.json. The field is omitted from JSON when unset (omitempty + # ForceSendFields tracking), so an absent value falls back to "(unset)". get_purge() { - gron.py < .databricks/bundle/default/resources.json | grep purge_on_delete || echo '(unset)' + print_state.py | gron.py | grep purge_on_delete || echo '(unset)' } title "Step 1: deploy with purge_on_delete unset" diff --git a/acceptance/bundle/resources/postgres_projects/remove_history_retention/script b/acceptance/bundle/resources/postgres_projects/remove_history_retention/script index 4d426ae616f..a4f41995e28 100644 --- a/acceptance/bundle/resources/postgres_projects/remove_history_retention/script +++ b/acceptance/bundle/resources/postgres_projects/remove_history_retention/script @@ -16,7 +16,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Remove history_retention_duration and re-deploy" trace update_file.py databricks.yml " history_retention_duration: 604800s" "" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -25,4 +25,4 @@ trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace- # record the drift it leaves behind. trace $CLI postgres get-project "${project_name}" | project_fields > out.project.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan &> out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script index 574a6cf5301..d3aeb95ebfa 100644 --- a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script +++ b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script @@ -25,7 +25,7 @@ trace update_file.py databricks.yml "suspend_timeout_duration: 300s" "suspend_ti # member of the suspension oneof, nested under default_endpoint_settings, and the # direct engine masks it under its own name. See test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/script b/acceptance/bundle/resources/postgres_projects/update_display_name/script index 657eca21869..5601e360c32 100755 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/script +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -27,7 +27,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -38,7 +38,7 @@ trace update_file.py databricks.yml "Original Name" "Updated Name" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -50,7 +50,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Restore display_name to original value" trace update_file.py databricks.yml "Updated Name" "Original Name" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_projects/update_history_retention/script b/acceptance/bundle/resources/postgres_projects/update_history_retention/script index 7a7ae267a73..d67c658c710 100644 --- a/acceptance/bundle/resources/postgres_projects/update_history_retention/script +++ b/acceptance/bundle/resources/postgres_projects/update_history_retention/script @@ -16,7 +16,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Change history_retention_duration to 259200s and re-deploy" trace update_file.py databricks.yml "history_retention_duration: 604800s" "history_retention_duration: 259200s" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml index ce8be6534d3..26f5e3debde 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml @@ -2,4 +2,4 @@ Cloud = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml index 6de86c5f2b6..01b05580cec 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml @@ -7,3 +7,7 @@ Cloud = false # bind/deploy are engine-agnostic here; run on direct for a single stable output. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# bind and unbind refuse to run when deployment history recording is on, so the +# escape hatch this test covers is not available there yet. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/update/script b/acceptance/bundle/resources/postgres_roles/update/script index 5eea84136a6..d98081efd6f 100644 --- a/acceptance/bundle/resources/postgres_roles/update/script +++ b/acceptance/bundle/resources/postgres_roles/update/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -28,7 +28,7 @@ trace $CLI postgres get-role "${role_name}" | role_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -39,7 +39,7 @@ trace update_file.py databricks.yml "createdb: false" "createdb: true" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -50,7 +50,7 @@ trace $CLI postgres get-role "${role_name}" | role_fields title "Restore attributes.createdb to original value" trace update_file.py databricks.yml "createdb: true" "createdb: false" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json index e840a5f3161..ce08323b631 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json @@ -1,7 +1,7 @@ >>> errcode [CLI] bundle plan -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml index cef45fe553a..9b8ddfcd2bd 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml @@ -1,3 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index 15c9c619892..796ce31264a 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -17,7 +17,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... Created quality_monitors.monitor1 -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml new file mode 100644 index 00000000000..2acd446f3cd --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml @@ -0,0 +1,3 @@ +# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py +# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json index cfaff9efccb..ae5c05d00ac 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index a14879d79f1..a51f2233f7d 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "output_schema_name: main.qm_test_${UNIQUE_NAME}" "output_schema_name: main.qm_test_${UNIQUE_NAME}_2" trace $CLI bundle plan | contains.py "1 to change" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json index 9b233a1fdbb..eb98b10555c 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json @@ -1,7 +1,7 @@ >>> errcode [CLI] bundle plan -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml index ae5c7bd798f..d73c45e3119 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml @@ -1,3 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml index 7e03ffedd9a..808576f4791 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml @@ -1,2 +1,6 @@ # this does not work on terraform EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py +# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json index b4a49e6fcd7..133e967a99c 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.quality_monitors.monitor1": { diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json index bb26c592f1d..2cb6b0913a4 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index 4f3596f8375..36e85d2164c 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -17,7 +17,7 @@ trap cleanup EXIT envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy &> out.deploy.txt @@ -26,4 +26,4 @@ trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^ # store state to ensure we have table_name there print_state.py | grep name > out.state.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/script b/acceptance/bundle/resources/registered_models/aliases_converge/script index ca9dc8f700a..7271aea0090 100644 --- a/acceptance/bundle/resources/registered_models/aliases_converge/script +++ b/acceptance/bundle/resources/registered_models/aliases_converge/script @@ -15,7 +15,7 @@ title "Plan is a no-op: GET never echoes aliases, so they must not drift" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The config-set aliases are skipped as input_only (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes.aliases' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes.aliases' title "Edit the comment: an in-place update, not a recreate" export COMMENT="updated comment" diff --git a/acceptance/bundle/resources/registered_models/basic/script b/acceptance/bundle/resources/registered_models/basic/script index 61dff74182c..c0d1d5e648d 100644 --- a/acceptance/bundle/resources/registered_models/basic/script +++ b/acceptance/bundle/resources/registered_models/basic/script @@ -20,7 +20,7 @@ trap cleanup EXIT deploy_registered_model() { trace $CLI bundle plan trace $CLI bundle deploy - registered_model_id=$($CLI bundle summary --output json | nostamp | jq -r '.resources.registered_models.my_registered_model.id') + registered_model_id=$($CLI bundle summary --output json | nostamp.py | jq -r '.resources.registered_models.my_registered_model.id') trace $CLI registered-models get "${registered_model_id}" | jq '{name, comment, catalog_name, schema_name}' } diff --git a/acceptance/bundle/resources/registered_models/drift/browse_only/script b/acceptance/bundle/resources/registered_models/drift/browse_only/script index a3d66f24b0b..84986d5df48 100644 --- a/acceptance/bundle/resources/registered_models/drift/browse_only/script +++ b/acceptance/bundle/resources/registered_models/drift/browse_only/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite the backend computing browse_only" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only browse_only is skipped as output_only (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes.browse_only' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes.browse_only' title "Redeploy is a no-op (no UpdateRegisteredModel call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/auto-approve/script b/acceptance/bundle/resources/schemas/auto-approve/script index f2f2b6cbade..098ef3327b8 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/script +++ b/acceptance/bundle/resources/schemas/auto-approve/script @@ -20,8 +20,8 @@ title "Assert the schema is created" trace $CLI schemas get "${CATALOG_NAME}.${SCHEMA_NAME}" | jq "{full_name, comment}" title "Assert the pipeline is created and uses the schema" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.foo.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" title "Create a volume in the schema, and add a file to it. This ensures that the schema has some data in it and deletion will fail unless the generated diff --git a/acceptance/bundle/resources/schemas/comment_out_of_band/script b/acceptance/bundle/resources/schemas/comment_out_of_band/script index 70871c43a9b..4ac6a877cce 100644 --- a/acceptance/bundle/resources/schemas/comment_out_of_band/script +++ b/acceptance/bundle/resources/schemas/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/schemas/$SCHEMA" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the schema" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/script b/acceptance/bundle/resources/schemas/drift/managed_properties/script index fadd886cd48..f4aa3e83214 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/script +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite UC auto-populating managed properties" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only properties map is skipped as a backend default (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy is a no-op (no UpdateSchema call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt index 2c6ded2a841..668da79bfb5 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.plan.resources.secret_scopes.my_scope.action = "create"; json.plan.resources.secret_scopes.my_scope.new_state.value.scope = "test-scope-[UNIQUE_NAME]-1"; diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt index 44b7e2831ca..f26d6cba67b 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.lineage = "[UUID]"; json.serial = 1; diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt index 8745d467735..2782639ba91 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.lineage = "[UUID]"; json.serial = 2; diff --git a/acceptance/bundle/resources/secret_scopes/basic/script b/acceptance/bundle/resources/secret_scopes/basic/script index e8aebc1853f..ebaaf04d898 100755 --- a/acceptance/bundle/resources/secret_scopes/basic/script +++ b/acceptance/bundle/resources/secret_scopes/basic/script @@ -9,7 +9,7 @@ cleanup() { trap cleanup EXIT title "create the secret scope" -trace $CLI bundle plan -o json | gron.py --sort-arrays acls > out.plan1.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan1.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq scope_name=$($CLI bundle summary --output json | jq -r '.resources.secret_scopes.my_scope.name') @@ -26,7 +26,7 @@ title "update the name of the scope (should recreate)" export SECRET_SCOPE_NAME="test-scope-$UNIQUE_NAME-2" envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json | gron.py --sort-arrays acls > out.plan2.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan2.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq # Capture API requests for verification. Terraform cleans up ACLs before deleting the scope, but direct does not, hence the difference in requests. @@ -43,4 +43,4 @@ trace $CLI secrets list-acls $scope_name | jq -c '.[]' | sort trace print_requests.py //secrets title "verify there's no persistent drift" -trace $CLI bundle plan -o json | gron.py --sort-arrays acls > out.plan_verify_no_drift.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan_verify_no_drift.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/secrets/remove-expire-time/script b/acceptance/bundle/resources/secrets/remove-expire-time/script index 5f507f1f39a..2f461ba392b 100644 --- a/acceptance/bundle/resources/secrets/remove-expire-time/script +++ b/acceptance/bundle/resources/secrets/remove-expire-time/script @@ -7,13 +7,13 @@ read_state.py secrets secret1 name expire_time title "Remove expire_time and re-deploy" trace update_file.py databricks.yml ' expire_time: "2030-01-01T00:00:00Z"' "" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --var secret_value=secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 name expire_time title "Plan again to record any drift the removal leaves behind" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt diff --git a/acceptance/bundle/resources/secrets/update-expire-time/script b/acceptance/bundle/resources/secrets/update-expire-time/script index 9b915b11936..b9321fd8782 100644 --- a/acceptance/bundle/resources/secrets/update-expire-time/script +++ b/acceptance/bundle/resources/secrets/update-expire-time/script @@ -7,7 +7,7 @@ read_state.py secrets secret1 name expire_time title "Change expire_time and re-deploy" trace update_file.py databricks.yml "2030-01-01T00:00:00Z" "2031-06-15T12:00:00Z" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --var secret_value=secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 name expire_time diff --git a/acceptance/bundle/resources/secrets/update-value/output.txt b/acceptance/bundle/resources/secrets/update-value/output.txt index 9e94921e2f3..e6cd1b57674 100644 --- a/acceptance/bundle/resources/secrets/update-value/output.txt +++ b/acceptance/bundle/resources/secrets/update-value/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle plan --var secret_value=initial-secret-value -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.secrets.secret1": { @@ -42,7 +42,7 @@ secrets secret1 catalog_name='main' schema_name='default' name='test_secret' com === Update secret value by passing a different variable value >>> [CLI] bundle plan --var secret_value=updated-secret-value -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -117,20 +117,14 @@ secrets secret1 catalog_name='main' schema_name='default' name='test_secret' com === Verify state does not contain actual secret value >>> print_state.py { - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 2, - "state": { "resources.secrets.secret1": { - "__id__": "main.default.test_secret", - "state": { - "catalog_name": "main", - "comment": "Test secret", - "name": "test_secret", - "schema_name": "default", - "value": "" - } + "__id__": "main.default.test_secret", + "state": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "" + } } - } } diff --git a/acceptance/bundle/resources/secrets/update-value/script b/acceptance/bundle/resources/secrets/update-value/script index a5c1596e29c..224073934fc 100755 --- a/acceptance/bundle/resources/secrets/update-value/script +++ b/acceptance/bundle/resources/secrets/update-value/script @@ -1,16 +1,16 @@ -trace $CLI bundle plan --var secret_value=initial-secret-value -o json +trace $CLI bundle plan --var secret_value=initial-secret-value -o json | nostamp.py trace $CLI bundle deploy --var secret_value=initial-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment value title "Update secret value by passing a different variable value" -trace $CLI bundle plan --var secret_value=updated-secret-value -o json +trace $CLI bundle plan --var secret_value=updated-secret-value -o json | nostamp.py trace $CLI bundle deploy --var secret_value=updated-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment value title "Verify state does not contain actual secret value" -{ trace print_state.py | contains.py "!initial-secret-value" "!updated-secret-value"; } || true +{ trace print_state.py | jq .state | contains.py "!initial-secret-value" "!updated-secret-value"; } || true rm -f out.requests.txt diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script index 58ce3a2f5ad..eaf11e08529 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script @@ -16,7 +16,7 @@ trace $CLI vector-search-endpoints patch-endpoint "${endpoint_name}" --target-qp title "Plan detects drift and proposes update" trace $CLI bundle plan | contains.py "Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged" -$CLI bundle plan --output json | jq '{plan: .plan | map_values({action, changes})}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, changes})}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json title "Deploy restores target_qps to 1" rm -f out.requests.txt diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt index 9f3980b3e80..77b87027314 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -17,7 +17,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle plan --output json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -71,7 +71,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "name": "vector" } ], - "schema_json": "{\"id\":\"int\",\"vector\":\"array\u003cfloat\u003e\"}" + "schema_json": "{\"id\":\"int\",\"vector\":\"array\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "endpoint_uuid": "[UUID]", @@ -87,9 +87,9 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "direct_access_index_spec.schema_json": { "action": "skip", "reason": "normalized_by_backend", - "old": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}", - "new": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}", - "remote": "{\"id\":\"int\",\"vector\":\"array\u003cfloat\u003e\"}" + "old": "{\"id\":\"integer\",\"vector\":\"array\"}", + "new": "{\"id\":\"integer\",\"vector\":\"array\"}", + "remote": "{\"id\":\"int\",\"vector\":\"array\"}" }, "endpoint_uuid": { "action": "skip", diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script index b6170f5b613..5761eaae689 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script @@ -14,7 +14,7 @@ title "Change endpoint_type (recreates endpoint; index recreate cascade is not y trace update_file.py databricks.yml "endpoint_type: STANDARD" "endpoint_type: STORAGE_OPTIMIZED" trace $CLI bundle plan -trace $CLI bundle plan --output json +trace $CLI bundle plan --output json | nostamp.py rm -f out.requests.txt trace $CLI bundle deploy --auto-approve diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt b/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt index cccd121c387..4b9a961521b 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.catalogs.main_catalog": { diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/script b/acceptance/bundle/resources/volumes/catalog-var-ref/script index 7f1b6190766..5e6053edc06 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/script +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/script @@ -1 +1 @@ -$CLI bundle plan -o json +$CLI bundle plan -o json | nostamp.py diff --git a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json index 68e26c30d44..682a73bf1f1 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json +++ b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/volumes/change-name/script b/acceptance/bundle/resources/volumes/change-name/script index 396ffb15155..61d993f6516 100644 --- a/acceptance/bundle/resources/volumes/change-name/script +++ b/acceptance/bundle/resources/volumes/change-name/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml myvolume mynewvolume trace $CLI bundle plan # terraform marks this as "update", direct marks this as "update_with_id" -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //unity diff --git a/acceptance/bundle/resources/volumes/comment_out_of_band/script b/acceptance/bundle/resources/volumes/comment_out_of_band/script index 3c01424f369..97ce5529786 100644 --- a/acceptance/bundle/resources/volumes/comment_out_of_band/script +++ b/acceptance/bundle/resources/volumes/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/volumes/$VOLUME" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the volume" -trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan["resources.volumes.volume1"].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json index b5ef8e7d906..cd7a6dfd044 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json +++ b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.volumes.foo": { diff --git a/acceptance/bundle/resources/volumes/remote-change-name/script b/acceptance/bundle/resources/volumes/remote-change-name/script index 536ad6d50b8..3337fd0c6c5 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/script +++ b/acceptance/bundle/resources/volumes/remote-change-name/script @@ -1,4 +1,4 @@ -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI volumes update mycatalog.myschema.myname --json '{"new_name": "my_new_name"}' diff --git a/acceptance/bundle/resources/volumes/uppercase-name/script b/acceptance/bundle/resources/volumes/uppercase-name/script index b5403a291f8..d2931527477 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/script +++ b/acceptance/bundle/resources/volumes/uppercase-name/script @@ -21,7 +21,7 @@ if [ "$DATABRICKS_BUNDLE_ENGINE" = "direct" ]; then # seconds after create, so the schema's `properties` change is present # nondeterministically. Prune it with normalize_uc_payload.py before reshaping so # the golden is stable. See #6027 and #6040. - $CLI bundle plan -o json \ + $CLI bundle plan -o json | nostamp.py \ | normalize_uc_payload.py \ | jq '.plan | to_entries | map({resource: .key, changes: (.value.changes // {} | map_values({action, reason}))})' \ >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/run_as/pipelines/_script b/acceptance/bundle/run_as/pipelines/_script index 293600ded7d..d7cf3f76a2a 100644 --- a/acceptance/bundle/run_as/pipelines/_script +++ b/acceptance/bundle/run_as/pipelines/_script @@ -1,5 +1,5 @@ print_requests() { - jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp.py rm out.requests.txt } @@ -15,7 +15,7 @@ for target in "${targets[@]}"; do trace $CLI bundle plan -t $target # Debug plan - $CLI bundle plan -o json -t $target | nostamp > out.plan_$target.$DATABRICKS_BUNDLE_ENGINE.json + $CLI bundle plan -o json -t $target | nostamp.py > out.plan_$target.$DATABRICKS_BUNDLE_ENGINE.json # Deploy rm -f out.requests.txt diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json index 6bd7cce4936..b29f4242bc9 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json index 98e33a7a82b..fdcbed9bea6 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json index 168d9852fb1..85d6696a669 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json index a489222a9bd..02c785a484d 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json index 5f886cd55ab..f1e14de6938 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json index ef31749eae8..7dab8bf4cbb 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json index 1e780047697..48c2635f87a 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json index 4640930e261..dd0a3e931c2 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/select/basic/out.test.toml b/acceptance/bundle/select/basic/out.test.toml index a420f50c421..b5dac5e552c 100644 --- a/acceptance/bundle/select/basic/out.test.toml +++ b/acceptance/bundle/select/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/basic/script b/acceptance/bundle/select/basic/script index 12db6ca0ba0..e8b7741af95 100644 --- a/acceptance/bundle/select/basic/script +++ b/acceptance/bundle/select/basic/script @@ -22,7 +22,7 @@ trace $CLI bundle plan --select jobs.foo # JSON embeds remote state that differs between local and cloud), then deploy it: # inline, or via --plan (READPLAN=1). The deploy is not traced because readplanarg # varies the command line between READPLAN variants, which must produce identical output. -$CLI bundle plan --select jobs.foo -o json | nostamp > plan.json +$CLI bundle plan --select jobs.foo -o json > plan.json title "bundle deploy --select jobs.foo\n" $CLI bundle deploy --select jobs.foo $(readplanarg plan.json) # The deploy reports that --select was used via telemetry. @@ -37,7 +37,7 @@ trace $CLI bundle summary # foo and bar are already deployed, so only baz remains to create. title "Full plan after partial deploy" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > plan-full.json +$CLI bundle plan -o json > plan-full.json title "Full deploy\n" $CLI bundle deploy $(readplanarg plan-full.json) # Only baz is created this time. diff --git a/acceptance/bundle/select/basic/test.toml b/acceptance/bundle/select/basic/test.toml index fe41d5f871a..7166a9e7946 100644 --- a/acceptance/bundle/select/basic/test.toml +++ b/acceptance/bundle/select/basic/test.toml @@ -8,3 +8,7 @@ EnvMatrix.READPLAN = ["", "1"] # databricks.yml and the serialized plans are generated at runtime; the plan JSON # is kept out of the recorded output because it embeds local/cloud-specific state. Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json"] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/select/grants_permissions/out.test.toml b/acceptance/bundle/select/grants_permissions/out.test.toml index dd03baaabb6..bf0f576652e 100644 --- a/acceptance/bundle/select/grants_permissions/out.test.toml +++ b/acceptance/bundle/select/grants_permissions/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/grants_permissions/test.toml b/acceptance/bundle/select/grants_permissions/test.toml index 4b4735b927d..544f8f1e442 100644 --- a/acceptance/bundle/select/grants_permissions/test.toml +++ b/acceptance/bundle/select/grants_permissions/test.toml @@ -11,3 +11,7 @@ RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json"] + +# Dumps plans and replays them via deploy --plan/readplanarg; the replay needs the DMS fields, +# so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/feature_flags/output.txt b/acceptance/bundle/state/feature_flags/output.txt index df55e7c6cee..1ed17f9b23e 100644 --- a/acceptance/bundle/state/feature_flags/output.txt +++ b/acceptance/bundle/state/feature_flags/output.txt @@ -1,12 +1,12 @@ -=== a version-3 state recording a feature is rejected (this CLI records no features yet) +=== a state depending on a feature this CLI does not recognize is rejected >>> errcode [CLI] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information +Error: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information Exit code: 1 -=== a version-3 state with an empty features map is accepted, and a deploy keeps it at version 3 (no flip to 2) +=== a state with an empty features map is accepted, and a deploy keeps it at version 2 >>> [CLI] bundle plan create jobs.my_job @@ -19,4 +19,4 @@ Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> gron.py .databricks/bundle/default/resources.json -json.state_version = 3; +json.state_version = 2; diff --git a/acceptance/bundle/state/feature_flags/resources.empty_features.json b/acceptance/bundle/state/feature_flags/resources.empty_features.json index b20c97aa074..876fdaf19ce 100644 --- a/acceptance/bundle/state/feature_flags/resources.empty_features.json +++ b/acceptance/bundle/state/feature_flags/resources.empty_features.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "features": {}, "cli_version": "0.0.0-dev", "lineage": "test-lineage", diff --git a/acceptance/bundle/state/feature_flags/resources.with_feature.json b/acceptance/bundle/state/feature_flags/resources.with_feature.json index b844b098a70..acc2ec9823d 100644 --- a/acceptance/bundle/state/feature_flags/resources.with_feature.json +++ b/acceptance/bundle/state/feature_flags/resources.with_feature.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "features": { "future_feature": {} }, diff --git a/acceptance/bundle/state/feature_flags/script b/acceptance/bundle/state/feature_flags/script index e70b562f2ff..ef9570e318c 100644 --- a/acceptance/bundle/state/feature_flags/script +++ b/acceptance/bundle/state/feature_flags/script @@ -1,10 +1,10 @@ mkdir -p .databricks/bundle/default -title "a version-3 state recording a feature is rejected (this CLI records no features yet)" +title "a state depending on a feature this CLI does not recognize is rejected" cp resources.with_feature.json .databricks/bundle/default/resources.json trace errcode $CLI bundle plan 2>&1 | contains.py "requires features this CLI does not support: future_feature" "upgrade to the latest CLI version" "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" -title "a version-3 state with an empty features map is accepted, and a deploy keeps it at version 3 (no flip to 2)" +title "a state with an empty features map is accepted, and a deploy keeps it at version 2" cp resources.empty_features.json .databricks/bundle/default/resources.json trace $CLI bundle plan | contains.py "Plan:" trace $CLI bundle deploy diff --git a/acceptance/bundle/state/feature_flags/test.toml b/acceptance/bundle/state/feature_flags/test.toml index 5640bffa4dc..dfe3377a2b7 100644 --- a/acceptance/bundle/state/feature_flags/test.toml +++ b/acceptance/bundle/state/feature_flags/test.toml @@ -1,3 +1,6 @@ Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +# State feature-flag handling, not DMS recording: opt out of the DMS variant so the deploy in the +# empty-features case does not record (which would diverge from the non-recording variant). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 0a16971f472..7cf98129ee9 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,3 @@ -state version 999 is newer than supported version 3; upgrade the CLI +state version 999 is newer than supported version 2; upgrade the CLI Exit code: 1 diff --git a/acceptance/bundle/state/permission_level_migration/out.plan.direct.json b/acceptance/bundle/state/permission_level_migration/out.plan.direct.json index 33d6194c91e..1492e489619 100644 --- a/acceptance/bundle/state/permission_level_migration/out.plan.direct.json +++ b/acceptance/bundle/state/permission_level_migration/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "test-lineage", "serial": 1, diff --git a/acceptance/bundle/state/permission_level_migration/script b/acceptance/bundle/state/permission_level_migration/script index 32211ebeb49..55fb4aa769f 100644 --- a/acceptance/bundle/state/permission_level_migration/script +++ b/acceptance/bundle/state/permission_level_migration/script @@ -9,10 +9,10 @@ mkdir -p .databricks/bundle/default cp resources.v1.json .databricks/bundle/default/resources.json title "Plan with old v1 state" -trace $CLI bundle plan -o json > out.plan.direct.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.direct.json title "Deploy (migrates state)" trace $CLI bundle deploy title "Print state after deploy" -trace print_state.py | nostamp --indent 1 +trace print_state.py | nostamp.py --indent 1 diff --git a/acceptance/bundle/summary/modified_status/script b/acceptance/bundle/summary/modified_status/script index 22149b2c76b..fd509c95af0 100644 --- a/acceptance/bundle/summary/modified_status/script +++ b/acceptance/bundle/summary/modified_status/script @@ -1,14 +1,14 @@ title "Initial view of resources without id and modified_status=created" -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources trace $CLI bundle deploy title "Post-deployment view of resources with id and without modified_status" -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources mv $VARIANT databricks.yml title "Expecting all resources to have modified_status=deleted" -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources trace $CLI bundle destroy --auto-approve -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json index 49c706fc239..5607a8fc757 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json index 30918164e2c..4ec8e61237d 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json index b150391beeb..3fb9ec72a4f 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json index a153855fe26..3b20349f64d 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 6b79c6f2251..883a405fa3e 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -7,20 +7,20 @@ trace $CLI bundle validate -t prod trace $CLI bundle plan -t dev trace $CLI bundle plan -t prod -$CLI bundle plan -o json -t dev > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp.py > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json rm ../../out.requests.txt # With --plan variant we don't build artifacts, so we can filter out relevant log lines $CLI bundle deploy -t dev $(readplanarg ../../out.plan_dev.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading .databricks' print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift -trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t dev -o json | nostamp.py > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy -t prod $(readplanarg ../../out.plan_prod.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading dist' print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift -trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t prod -o json | nostamp.py > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json index 0c6800b20b3..b69ca087696 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json index e637e98dde5..1bb26918106 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/integration_classic/script b/acceptance/bundle/templates/default-python/integration_classic/script index 50b39d6ffce..98c5b97123c 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/script +++ b/acceptance/bundle/templates/default-python/integration_classic/script @@ -17,7 +17,7 @@ trap "trace $CLI bundle destroy -t dev --auto-approve" EXIT trace $CLI bundle validate -t dev trace $CLI bundle validate -t dev -o json | jq -S "del($JQ_DELETE)" > ../out.validate.dev.json -$CLI bundle plan -o json -t dev > ../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > ../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t dev trace $CLI bundle summary -t dev trace $CLI bundle summary -t dev -o json | jq -S "del($JQ_DELETE)" > ../out.summary.dev.json @@ -30,7 +30,7 @@ trace $CLI bundle validate -t prod trace $CLI bundle validate -t prod -o json | jq -S "del($JQ_DELETE)" > ../out.validate.prod.json trace diff.py ../out.validate.dev.json ../out.validate.prod.json rm ../out.validate.prod.json -$CLI bundle plan -o json -t prod > ../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp.py > ../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t prod trace $CLI bundle summary -t prod trace $CLI bundle summary -t prod -o json | jq -S "del($JQ_DELETE)" > ../out.summary.prod.json diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json index 9967e015883..eac2e21abdf 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json index 57a12ae5bb3..c9b06ebd2c5 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json index dd0e402c7df..f68705b05d9 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json index a9db4ff38f4..42a1c6a6351 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index 86272b4a266..c0f2e58f4f4 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -7,19 +7,19 @@ trace $CLI bundle validate -t prod trace $CLI bundle plan -t dev trace $CLI bundle plan -t prod -$CLI bundle plan -o json -t dev > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp.py > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json rm ../../out.requests.txt trace $CLI bundle deploy -t dev print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift -trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t dev -o json | nostamp.py > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t prod print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift -trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t prod -o json | nostamp.py > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/record-deployment-history/out.test.toml b/acceptance/bundle/templates/record-deployment-history/out.test.toml index 27ec2a7fcd6..9921e91a794 100644 --- a/acceptance/bundle/templates/record-deployment-history/out.test.toml +++ b/acceptance/bundle/templates/record-deployment-history/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/templates/record-deployment-history/test.toml b/acceptance/bundle/templates/record-deployment-history/test.toml index a1d3232167a..89b85620c2a 100644 --- a/acceptance/bundle/templates/record-deployment-history/test.toml +++ b/acceptance/bundle/templates/record-deployment-history/test.toml @@ -2,3 +2,7 @@ Env.DATABRICKS_BUNDLE_INIT_RECORD_DEPLOYMENT_HISTORY = "true" # What is under test is what `bundle init` writes, so one engine is enough. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The generated project sets experimental.record_deployment_history; DMS=true permits it so +# the validate step below loads instead of rejecting it. +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index 6a259762f65..2db98f4b429 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -10,14 +10,6 @@ EnvMatrix.DMS = ["", "true"] # DMS recording is only supported by the direct engine; it is a no-op on terraform. EnvMatrixExclude.dms_needs_direct = ["DMS=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] -# Saved plans don't carry the deployment stamp. A first plan writes the deployment record, -# so `deploy --plan` creates resources without it and reports drift on the next plan. -EnvMatrixExclude.dms_no_readplan = ["DMS=true", "READPLAN=1"] - -# Recording is gated off for users (see validate.ValidateRecordDeploymentHistory), so -# force it on: the point of this run is to exercise it. -Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" - # The DMS run asserts the same golden files as the engine runs. EnvRepl.DMS = false diff --git a/bundle/bundle.go b/bundle/bundle.go index 1eb9ce15654..1832be861c3 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -317,6 +317,14 @@ func (b *Bundle) WorkspaceClient(ctx context.Context) *databricks.WorkspaceClien return client } +// ConfiguresDeploymentHistory reports whether this bundle is configured to record deployment history with the +// deployment metadata service, from experimental.record_deployment_history or +// DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY. +func (b *Bundle) ConfiguresDeploymentHistory(ctx context.Context) bool { + configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory + return env.RecordsDeploymentHistory(ctx, configured) +} + // SetWorkpaceClient sets the workspace client for this bundle. // This is used to inject a mock client for testing. func (b *Bundle) SetWorkpaceClient(w *databricks.WorkspaceClient) { diff --git a/bundle/config/deployment.go b/bundle/config/deployment.go index b7efb4456f9..b59d1b1da02 100644 --- a/bundle/config/deployment.go +++ b/bundle/config/deployment.go @@ -7,4 +7,20 @@ type Deployment struct { // Lock configures locking behavior on deployment. Lock Lock `json:"lock,omitempty"` + + // History reports what the deployment metadata service has recorded for this + // bundle. Output only: it is read from the service for 'bundle summary' and is + // unset when the bundle does not record deployment history. + History *DeploymentHistory `json:"history,omitempty" bundle:"readonly"` +} + +// DeploymentHistory identifies the bundle's deployment in the deployment +// metadata service. +type DeploymentHistory struct { + // DeploymentID is the ID the service assigned to this bundle's deployment. + DeploymentID string `json:"deployment_id,omitempty"` + + // LatestVersionID is the most recent version recorded for the deployment. It is + // unset when the deployment exists but has no version yet. + LatestVersionID string `json:"latest_version_id,omitempty"` } diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 658f1cea819..005c4dfcade 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -53,6 +53,9 @@ type Experimental struct { // RecordDeploymentHistory opts the bundle into the deployment metadata // service (DMS), which records deployment history and tracks what changed // across deployments. + // + // Only for bundles with no deployed resources yet: DMS becomes the source of + // truth for their state. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index ea45903508b..40511ae1dcd 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 65cddeaff19..c16655a3518 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -156,7 +156,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false)); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/deploy/metadata/annotate_deployment_version.go b/bundle/deploy/metadata/annotate_deployment_version.go new file mode 100644 index 00000000000..c08f7b52345 --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version.go @@ -0,0 +1,65 @@ +package metadata + +import ( + "context" + "strconv" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type annotateDeployment struct { + deploymentID string +} + +// AnnotateDeployment stamps the DMS deployment onto every job and pipeline, so a workspace +// resource points back at the deployment that produced it - how lineage resolves a job to its +// bundle. It runs before the plan, or the stamp would show as drift against local config. +func AnnotateDeployment(deploymentID string) bundle.Mutator { + return &annotateDeployment{deploymentID: deploymentID} +} + +func (m *annotateDeployment) Name() string { + return "metadata.AnnotateDeployment" +} + +func (m *annotateDeployment) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, job := range b.Config.Resources.Jobs { + // Deployment is set by AnnotateJobs, which runs during initialize. + job.Deployment.DeploymentId = m.deploymentID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.DeploymentId = m.deploymentID + } + + return nil +} + +type annotateDeploymentVersion struct { + version int64 +} + +// AnnotateDeploymentVersion stamps the DMS version onto every job and pipeline. +// Separate from AnnotateDeployment because version only exists after CreateVersion runs. +func AnnotateDeploymentVersion(version int64) bundle.Mutator { + return &annotateDeploymentVersion{version: version} +} + +func (m *annotateDeploymentVersion) Name() string { + return "metadata.AnnotateDeploymentVersion" +} + +func (m *annotateDeploymentVersion) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + versionID := strconv.FormatInt(m.version, 10) + + for _, job := range b.Config.Resources.Jobs { + job.Deployment.VersionId = versionID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.VersionId = versionID + } + + return nil +} diff --git a/bundle/deploy/metadata/annotate_deployment_version_test.go b/bundle/deploy/metadata/annotate_deployment_version_test.go new file mode 100644 index 00000000000..297ba95f8ee --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version_test.go @@ -0,0 +1,50 @@ +package metadata + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAnnotateDeploymentVersion(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my-job": { + JobSettings: jobs.JobSettings{ + Deployment: &jobs.JobDeployment{Kind: jobs.JobDeploymentKindBundle}, + }, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "my-pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Deployment: &pipelines.PipelineDeployment{Kind: pipelines.DeploymentKindBundle}, + }, + }, + }, + }, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, AnnotateDeployment("dep-123"), AnnotateDeploymentVersion(7)) + require.NoError(t, diags.Error()) + + job := b.Config.Resources.Jobs["my-job"].Deployment + assert.Equal(t, "dep-123", job.DeploymentId) + assert.Equal(t, "7", job.VersionId) + // The kind set by AnnotateJobs is preserved. + assert.Equal(t, jobs.JobDeploymentKindBundle, job.Kind) + + pipeline := b.Config.Resources.Pipelines["my-pipeline"].Deployment + assert.Equal(t, "dep-123", pipeline.DeploymentId) + assert.Equal(t, "7", pipeline.VersionId) + assert.Equal(t, pipelines.DeploymentKindBundle, pipeline.Kind) +} diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index c9f5bec3516..3f24c82ee43 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -13,14 +13,23 @@ import ( "github.com/databricks/cli/libs/structs/structvar" ) -const currentPlanVersion = 2 +const currentPlanVersion = 3 type Plan struct { - PlanVersion int `json:"plan_version,omitempty"` - CLIVersion string `json:"cli_version,omitempty"` - Lineage string `json:"lineage,omitempty"` - Serial int `json:"serial,omitempty"` - Plan map[string]*PlanEntry `json:"plan,omitzero"` + PlanVersion int `json:"plan_version,omitempty"` + CLIVersion string `json:"cli_version,omitempty"` + Lineage string `json:"lineage,omitempty"` + Serial int `json:"serial,omitempty"` + + // DMS fields, set only when the bundle records deployment history. The plan targets DeploymentId + // and will create NextVersionId; LastVersionId is the deployment's most recent version at plan + // time. deploy --plan rejects the plan if the deployment moved on from these (see process.go), + // and passes LastVersionId as previous_version_id so the service rejects a stale version too. + DeploymentId string `json:"deployment_id,omitempty"` + NextVersionId string `json:"next_version_id,omitempty"` + LastVersionId string `json:"last_version_id,omitempty"` + + Plan map[string]*PlanEntry `json:"plan,omitzero"` // NotSelected is the number of resources removed by FilterToSelected via the // --select flag. Serialized so the summary survives a deploy from a plan file diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 6b2fffe9a16..af2a7277c77 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -58,6 +58,7 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, } } +// Create creates the resource and records its state. func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, newState any) error { var newID string var remoteState any @@ -83,7 +84,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -126,7 +127,10 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // Drop the state entry so a subsequent failure of Create or WaitAfterDelete // leaves no malformed (empty-ID) entry behind. The next plan will see "no // state" and retry as Create. - err = db.DeleteState(d.ResourceKey) + // + // Recorded as a recreate rather than a delete: if the create below fails, this is the + // operation DMS is left with, and it says the resource is mid-recreate. + err = db.DeleteState(ctx, d.ResourceKey, true) if err != nil { return fmt.Errorf("deleting state: %w", err) } @@ -168,12 +172,12 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, // The update emptied the resource out (e.g. all grants revoked). Keeping an entry // would report the node as tracked-and-unchanged forever, while a fresh deploy of // the same config plans no node at all; drop it so the two agree. - err = db.DeleteState(d.ResourceKey) + err = db.DeleteState(ctx, d.ResourceKey, false) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -218,7 +222,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -256,11 +260,15 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, } else if d.deleteConfirmedGone(ctx, oldID) { log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) } else { - return fmt.Errorf("deleting id=%s: %w", oldID, err) + // The resource is still there, so the history says why rather than leaving its + // operation pending. The state write below is what records a delete that worked. + err = fmt.Errorf("deleting id=%s: %w", oldID, err) + db.RecordFailure(d.ResourceKey, oldID, err) + return err } } - err = db.DeleteState(d.ResourceKey) + err = db.DeleteState(ctx, d.ResourceKey, false) if err != nil { return fmt.Errorf("deleting state id=%s: %w", oldID, err) } @@ -305,7 +313,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -315,11 +323,11 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // saveState saves a state with sensitive fields replaced by a placeholder value so secrets are never written // to disk in plaintext. -func (d *DeploymentUnit) saveState(db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { if err := zeroSensitiveFields(d.Adapter, state); err != nil { return fmt.Errorf("redacting state: %w", err) } - return db.SaveState(d.ResourceKey, newID, state, dependsOn) + return db.SaveState(ctx, d.ResourceKey, newID, state, dependsOn) } func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 9760ce95666..9c4ab260e03 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -61,8 +61,10 @@ type BindResult struct { // Call Finalize to commit the state or Cancel to discard. func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) + // The state is opened without a DMS client, so the writes below record nothing; + // phases.Bind and phases.Unbind refuse to run at all when recording is enabled. var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,14 +88,14 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err } // Save state with ID and empty state (like migrate does) - err = b.StateDB.SaveState(resourceKey, resourceID, struct{}{}, nil) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +111,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,13 +147,13 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err } - err = b.StateDB.SaveState(resourceKey, resourceID, sv.Value, dependsOn) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, sv.Value, dependsOn) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +167,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,13 +217,13 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") if err != nil { return err } // Delete the main resource - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, false) if err != nil { return err } @@ -235,7 +237,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st for key := range b.StateDB.Data.State { if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") { - err = b.StateDB.DeleteState(key) + err = b.StateDB.DeleteState(ctx, key, false) if err != nil { return err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 424ae2bdec9..72465fad346 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -43,6 +43,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } + // The state DB records every write with DMS from here on (via the buffer StartRecording opened + // in the deploy phase), so the service mirrors the WAL. Writes go out on one background + // goroutine, off the apply path, and are drained below once every worker has finished recording. g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { entry, err := plan.WriteLockEntry(resourceKey) if err != nil { @@ -73,6 +76,14 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } + // Stop resource CRUD once recording state with DMS has failed. + if buf := b.DmsAsyncOperationClient; buf != nil { + if err := buf.Err(); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + } + adapter, err := b.getAdapterForKey(resourceKey) if adapter == nil { logdiag.LogError(ctx, fmt.Errorf("%s: internal error: cannot get adapter: %w", errorPrefix, err)) @@ -103,7 +114,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa if entry.Gone { // Planning confirmed the resource is already deleted remotely; only // remove it from the state, without calling the delete API. - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, false) } else { err = d.Destroy(ctx, &b.StateDB) } @@ -134,8 +145,15 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } // TODO: redo calcDiff to downgrade planned action if possible (?) + // + // Success is recorded by the state writes inside Deploy, so a recreate reports + // each of its steps. err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { + // Empty for a create that never got an ID, and for a recreate whose delete + // step already dropped it. + failedID := b.StateDB.GetResourceID(resourceKey) + b.StateDB.RecordFailure(resourceKey, failedID, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -162,6 +180,14 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) + + // Wait for the uploads and report a failure here, with the deploy's other errors. The phase + // completes the version afterwards and drains again, quietly, so this is not printed twice. + if buf := b.DmsAsyncOperationClient; buf != nil { + if err := buf.Drain(); err != nil { + logdiag.LogError(ctx, err) + } + } } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 739ca3bd8e7..45d5b2d3663 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -8,6 +8,7 @@ import ( "maps" "reflect" "slices" + "strconv" "strings" "github.com/databricks/cli/bundle/config" @@ -15,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/terraform_dabs_map" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" @@ -25,6 +27,7 @@ import ( "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/jobs" ) var errDelayed = errors.New("must be resolved after apply") @@ -58,8 +61,10 @@ func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan. return nil } -// InitForApply initializes the DeploymentBundle for applying a pre-computed plan. -// StateDB must already be open for write before calling this function. +// InitForApply initializes the DeploymentBundle for applying a pre-computed plan: it parses the +// plan's entries (loaded as JSON when the plan came from a file) into the typed state cache the +// apply reads. A first deployment's id does not exist until the deploy creates it after approval; +// StampDeploymentID fills it in then. StateDB must already be open for write before calling this. func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { b.StateDB.AssertOpenedForWrite() @@ -113,6 +118,41 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. return nil } +// StampDeploymentID fills deploymentID into the plan's jobs and pipelines that still lack one, in +// both the state cache the apply reads and the plan JSON. It is called after approval creates the +// deployment, for a first deploy whose id did not exist at plan time; the version and any +// pre-existing id were stamped then. +func (b *DeploymentBundle) StampDeploymentID(deploymentID string) error { + for resourceKey, entry := range b.Plan.Plan { + if entry.NewState == nil || len(entry.NewState.Value) == 0 { + continue + } + sv, ok := b.StateCache.Load(resourceKey) + if !ok { + continue + } + var stamped bool + switch v := sv.Value.(type) { + case *jobs.JobSettings: + if v.Deployment.DeploymentId == "" { + v.Deployment.DeploymentId = deploymentID + stamped = true + } + case *dresources.PipelineState: + if v.Deployment.DeploymentId == "" { + v.Deployment.DeploymentId = deploymentID + stamped = true + } + } + if stamped { + if err := sv.SyncToJSON(entry.NewState); err != nil { + return fmt.Errorf("%s: stamping deployment into plan: %w", resourceKey, err) + } + } + } + return nil +} + // CalculatePlan computes the deployment plan by comparing local config against remote state. // StateDB must already be open for read before calling this function. func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root) (*deployplan.Plan, error) { @@ -128,6 +168,21 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, fmt.Errorf("reading config: %w", err) } + // Record the DMS deployment and version this plan targets. A saved plan carries them so + // deploy --plan can reject a plan the deployment has moved on from, and pass last_version_id + // as previous_version_id. History is set only while recording, so other plans leave these empty. + // configRoot is nil for destroy, which records its version separately. + if configRoot != nil && configRoot.Bundle.Deployment.History != nil { + h := configRoot.Bundle.Deployment.History + next, err := dms.NextVersion(h.LatestVersionID) + if err != nil { + return nil, fmt.Errorf("computing next deployment version: %w", err) + } + plan.DeploymentId = h.DeploymentID + plan.LastVersionId = h.LatestVersionID + plan.NextVersionId = strconv.FormatInt(next, 10) + } + b.Plan = plan g, err := makeGraph(plan) diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go new file mode 100644 index 00000000000..c0d4ffb2a7f --- /dev/null +++ b/bundle/direct/dstate/dms.go @@ -0,0 +1,58 @@ +package dstate + +import ( + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dms" +) + +// RecordedState is what the CLI serializes into the DMS operation's state field. It wraps the +// config so depends_on survives the round trip: DMS has no field for dependency edges, and +// nesting them in the config would collide with resource fields of the same name. +type RecordedState struct { + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` +} + +// applyDMSState replaces the file-derived resource state with what DMS recorded. DMS owns the +// resource set outright, so this runs on every recorded open and the file's copy is never read +// back as the truth: an empty set means the service tracks nothing, whether because the deploy +// created nothing or because the deployment is gone. The caller holds db.mu. +func (db *DeploymentState) applyDMSState(recorded []dms.Resource) error { + // Built first and assigned together, so a malformed envelope leaves the state as it was + // rather than half replaced. + resources := make(map[string]ResourceEntry, len(recorded)) + stateIDs := make(map[string]string, len(recorded)) + for _, res := range recorded { + entry, err := stateEntry(res) + if err != nil { + return err + } + resources[res.Key] = entry + stateIDs[res.Key] = entry.ID + } + + db.Data.State = resources + db.stateIDs = stateIDs + return nil +} + +// stateEntry unwraps the envelope the write path recorded for a resource. +func stateEntry(res dms.Resource) (ResourceEntry, error) { + var recorded RecordedState + if res.State != "" { + // The service stores state as an opaque string, so it arrives as the serialized + // envelope the write side sent. + if err := json.Unmarshal([]byte(res.State), &recorded); err != nil { + return ResourceEntry{}, fmt.Errorf("interpreting state recorded for %s: %w", res.Key, err) + } + } + + return ResourceEntry{ + ID: res.ID, + State: recorded.State, + DependsOn: recorded.DependsOn, + }, nil +} diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go new file mode 100644 index 00000000000..8edcf563ee8 --- /dev/null +++ b/bundle/direct/dstate/dms_test.go @@ -0,0 +1,100 @@ +package dstate + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dms" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestApplyDMSState(t *testing.T) { + // The service stores state as an opaque string, so the envelope the write path sent + // arrives verbatim. + envelope := `{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}` + + // What the file loaded, for the cases that check it is replaced or survives. + fileState := map[string]ResourceEntry{ + "resources.jobs.bar": {ID: "file-id", State: json.RawMessage(`{"name":"from-file"}`)}, + } + + tests := []struct { + name string + existing map[string]ResourceEntry + recorded []dms.Resource + want map[string]ResourceEntry + wantErr string + }{ + { + name: "the envelope is unwrapped, depends_on and all", + recorded: []dms.Resource{ + {Key: "resources.jobs.foo", ID: "123", State: envelope}, + {Key: "resources.pipelines.bar", ID: "456"}, + }, + // depends_on comes back from the envelope, so a bundle whose local state was + // wiped still has the edges needed for delete ordering. + want: map[string]ResourceEntry{ + "resources.jobs.foo": { + ID: "123", + State: json.RawMessage(`{"name":"foo"}`), + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "${resources.pipelines.bar.id}"}}, + }, + "resources.pipelines.bar": {ID: "456"}, + }, + }, + { + name: "what DMS holds replaces what the file loaded", + existing: fileState, + recorded: []dms.Resource{{Key: "resources.jobs.foo", ID: "dms-id", State: `{"state":{"name":"from-dms"}}`}}, + want: map[string]ResourceEntry{"resources.jobs.foo": {ID: "dms-id", State: json.RawMessage(`{"name":"from-dms"}`)}}, + }, + { + // Nil is what a list holding nothing returns, and it means a successful deploy of + // nothing rather than missing data. + name: "nothing recorded empties the state", + existing: fileState, + want: map[string]ResourceEntry{}, + }, + { + // The good resource comes first, so the error lands mid-way: what the file loaded + // has to survive whole rather than end up half replaced. + name: "a malformed envelope leaves the state as it was", + existing: fileState, + recorded: []dms.Resource{ + {Key: "resources.jobs.ok", ID: "999", State: `{"state":{"name":"ok"}}`}, + {Key: "resources.jobs.foo", ID: "123", State: "not json"}, + }, + wantErr: "interpreting state recorded for resources.jobs.foo", + want: fileState, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var db DeploymentState + db.Data.State = tt.existing + db.stateIDs = stateIDsOf(tt.existing) + + err := db.applyDMSState(tt.recorded) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + + assert.Equal(t, tt.want, db.Data.State) + assert.Equal(t, stateIDsOf(tt.want), db.stateIDs) + }) + } +} + +// stateIDsOf is the id index the state keeps alongside its entries. +func stateIDsOf(entries map[string]ResourceEntry) map[string]string { + ids := make(map[string]string, len(entries)) + for key, entry := range entries { + ids[key] = entry.ID + } + return ids +} diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index e4d21a7054a..dccb06686a5 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -3,8 +3,6 @@ package dstate import ( "encoding/json" "fmt" - "slices" - "strings" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/structs/structpath" @@ -15,24 +13,6 @@ import ( // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // featureStateVersion states carry a feature list this CLI does not yet write or - // understand (see the featureStateVersion doc comment). A featureStateVersion - // state with no features is equivalent to currentStateVersion, so accept it and - // return without running the migrations below, leaving the on-disk version at - // featureStateVersion rather than flipping it down. One that records any feature - // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. - if db.StateVersion == featureStateVersion { - if len(db.Features) == 0 { - return nil - } - features := make([]string, 0, len(db.Features)) - for name := range db.Features { - features = append(features, name) - } - slices.Sort(features) - return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) - } - if db.StateVersion == currentStateVersion { return nil } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index af2e2638eb3..4e2f8c03344 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -10,13 +10,17 @@ import ( "io/fs" "os" "path/filepath" + "slices" + "strings" "sync" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" "github.com/google/uuid" ) @@ -28,36 +32,49 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // featureStateVersion is the schema version a future CLI will write once it - // records deployment state "feature flags" (see Header.Features). This CLI does - // not write it and records no features; it exists now only so this CLI reads - // such states correctly (see migrateState): - // - featureStateVersion with no features -> accept and leave the version as-is - // - featureStateVersion with any feature -> refuse, tell the user to upgrade - // - // A featureStateVersion state with no features is equivalent to - // currentStateVersion, but we deliberately do not flip the on-disk version down - // to currentStateVersion: a state written at featureStateVersion stays at - // featureStateVersion. This is forward-compat scaffolding so that a later release - // can start writing featureStateVersion + features without older CLIs (with this - // change) either mishandling a feature they lack or rejecting a featureless state - // outright. featureStateVersion is always 3. - featureStateVersion = 3 - - // supportedStateVersion is the highest schema version this CLI can read. It is - // normally equal to currentStateVersion — the version this CLI reads is the - // version it writes — and exceeds it only during a two-phase version bump like - // the current feature-flag scaffolding, where this CLI reads (but does not - // write) featureStateVersion. A state newer than this is rejected as too new. - supportedStateVersion = featureStateVersion + // supportedStateVersion is the highest schema version this CLI can read: the + // version it writes. A state newer than this is rejected as too new. + supportedStateVersion = currentStateVersion ) -// featuresDocURL is the single documentation page describing deployment state -// feature flags. It is shown when a state records a feature this CLI does not -// support; it is a fixed link for all features. The #state-features anchor points -// at the feature table; if it ever breaks, the user still lands on the page. +// featureRecordDeploymentHistory marks a state whose resources are also recorded with the +// deployment metadata service. Both stores are kept in step, so the marker is what tells a +// reader the two already agree. A CLI that does not know the name refuses the state rather +// than deploying over a deployment it would leave the service out of step with. +// +// The marker is sticky: once a deployment is recorded, the service holds resources that a +// CLI which is not recording must not touch. So turning recording off does not clear it, and +// deploying such a state without recording is refused (see RequiresDeploymentHistory). +const featureRecordDeploymentHistory = "record_deployment_history" + +// featuresDocURL explains the state-features mechanism in the error shown when this CLI refuses a +// state that depends on a feature it does not recognize. const featuresDocURL = "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" +// recognizedFeatures is the set of state feature flags this CLI understands. A state depending on +// any feature not listed here is refused (see checkStateFeatures), so a newer CLI's feature is not +// silently clobbered by this one. +var recognizedFeatures = map[string]struct{}{ + featureRecordDeploymentHistory: {}, +} + +// checkStateFeatures refuses a state that depends on a feature this CLI does not recognize: keeping +// the state and the service in step needs every reader to understand the features in play, so an +// unknown one belongs to a newer CLI and this one refuses rather than clobber it. +func checkStateFeatures(features map[string]struct{}) error { + var unsupported []string + for feature := range features { + if _, ok := recognizedFeatures[feature]; !ok { + unsupported = append(unsupported, feature) + } + } + if len(unsupported) == 0 { + return nil + } + slices.Sort(unsupported) + return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(unsupported, ", "), featuresDocURL) +} + // errStaleWAL is returned when the WAL serial is behind the expected serial. // The caller should delete the stale WAL and proceed normally. var errStaleWAL = errors.New("stale WAL") @@ -70,6 +87,14 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string + + // operationBuffer records each state write with DMS. Nil unless the bundle records deployment + // history, in which case StartRecording installs it once the version exists. + operationBuffer *dms.OperationBuffer + + // dmsClient talks to the deployment metadata service. Open builds it from the workspace + // client when the deployment records history; nil otherwise. + dmsClient *dms.Client } type Header struct { @@ -84,10 +109,9 @@ type Header struct { Serial int `json:"serial"` // Features maps each feature flag this state depends on to a (currently empty) - // value. This CLI writes no features; it only reads the field to detect a state - // that depends on features it lacks and refuse it (see migrateState). It is a - // map so a future CLI can attach per-feature data without reshaping the state. - // Empty/omitted for states that use no features. + // value. It is read to detect a state that depends on features this CLI lacks and + // refuse it (see migrateState). It is a map so a future CLI can attach per-feature + // data without reshaping the state. Empty/omitted for states that use no features. Features map[string]struct{} `json:"features,omitempty"` } @@ -111,6 +135,52 @@ type WALEntry struct { Value *ResourceEntry `json:"v,omitempty"` // nil means delete } +// StartRecording opens the operation buffer that records each subsequent state write with DMS +// under deploymentID and versionID, so what the service holds mirrors the WAL. It uses the client +// Open built, so it is called only when the deployment records history, and once the version +// exists (after approval) - which is why it is not an Open option. Returns the buffer so the +// deploy can drain it. +func (db *DeploymentState) StartRecording(ctx context.Context, deploymentID string, versionID int64) *dms.OperationBuffer { + buf := dms.StartOperationBuffer(ctx, db.dmsClient, deploymentID, versionID) + + db.mu.Lock() + defer db.mu.Unlock() + db.operationBuffer = buf + return buf +} + +// RecordFailure records that a resource did not apply, so the history says why rather than +// leaving the resource out. resourceID is the id it had before the failure. +func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause error) { + r := db.recorder() + if r == nil { + return + } + + // The service refuses a failure that leaves a live resource described by nothing, so re-state + // what it still has. An empty resourceID means there is nothing left: a create that never + // landed, or a recreate whose delete already went through. + var recorded json.RawMessage + if entry, ok := db.GetResourceEntry(resourceKey); resourceID != "" && ok && len(entry.State) > 0 { + var err error + recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + if err != nil { + // Nothing the caller can act on, so record the failure without the state. + recorded = nil + } + } + + r.RecordFailure(resourceKey, resourceID, recorded, cause) +} + +// recorder reads the buffer under db.mu, which guards it, and returns nil when the bundle does +// not record deployment history. +func (db *DeploymentState) recorder() *dms.OperationBuffer { + db.mu.Lock() + defer db.mu.Unlock() + return db.operationBuffer +} + func NewDatabase(lineage string, serial int) Database { return Database{ Header: Header{ @@ -123,47 +193,81 @@ func NewDatabase(lineage string, serial int) Database { } } -func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +// SaveState records the resource's state after an operation was applied to it. +func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { db.AssertOpenedForWrite() - db.mu.Lock() - defer db.mu.Unlock() - - if db.Data.State == nil { - db.Data.State = make(map[string]ResourceEntry) - } jsonMessage, err := json.Marshal(state) if err != nil { return err } - entry := ResourceEntry{ ID: newID, State: json.RawMessage(jsonMessage), DependsOn: dependsOn, } + db.mu.Lock() + if db.Data.State == nil { + db.Data.State = make(map[string]ResourceEntry) + } err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) if err == nil { db.stateIDs[key] = newID } - return err + buf := db.operationBuffer + db.mu.Unlock() + + if err != nil { + return err + } + if buf == nil { + return nil + } + + // Recorded after the WAL write, so DMS never reports state the deploy failed to persist, + // and outside db.mu because recording waits when the service is behind - waiting under the + // lock would hold up every other resource's write. + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + if err != nil { + return err + } + buf.RecordOperation(ctx, key, false, newID, recorded) + + return nil } -func (db *DeploymentState) DeleteState(key string) error { +// DeleteState drops the resource's state entry: the resource is gone. inProgress records the +// operation as unfinished, which is what the first half of a recreate wants - an interrupted +// deploy must not leave the resource described as finished. +func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgress bool) error { db.AssertOpenedForWrite() - db.mu.Lock() - defer db.mu.Unlock() + db.mu.Lock() if db.Data.State == nil { + db.mu.Unlock() return nil } - + // Read before the delete: DMS needs the id to say which resource went away. + deletedID := db.stateIDs[key] err := appendJSONLine(db.walFile, WALEntry{Key: key}) if err == nil { delete(db.stateIDs, key) } - return err + buf := db.operationBuffer + db.mu.Unlock() + + if err != nil { + return err + } + + // State is nil: the resource no longer exists. Recorded outside the lock for the + // same reason as SaveState. + if buf != nil { + buf.RecordOperation(ctx, key, inProgress, deletedID, nil) + } + + return nil } func (db *DeploymentState) GetResourceEntry(key string) (ResourceEntry, bool) { @@ -201,6 +305,24 @@ func (db *DeploymentState) StateCLIVersion() string { return db.Data.CLIVersion } +// RequiresDeploymentHistory reports whether the state depends on the deployment metadata +// service recording it. Deploying such a state without recording would leave the service +// holding a deployment that no longer matches, so the caller refuses instead. +func (db *DeploymentState) RequiresDeploymentHistory() bool { + db.AssertOpenedForReadOrWrite() + db.mu.Lock() + defer db.mu.Unlock() + + _, ok := db.Data.Features[featureRecordDeploymentHistory] + return ok +} + +// DmsClient returns the deployment metadata service client Open built from the workspace client, +// or nil when the deployment does not record history. +func (db *DeploymentState) DmsClient() *dms.Client { + return db.dmsClient +} + // GetOrInitLineage returns the deployment lineage, generating and storing a new // one if the state does not have one yet. It is the single place the lineage is // initialized, shared so the direct deployment engine (when it writes state, via @@ -228,9 +350,21 @@ type ( // If true, the state is opened in Write mode, which enables methods such as SaveState // but disables GetResourceEntry (since writes go strictly into WAL and not in memory). WithWrite bool + + // If true, the deployment records history with the metadata service: Open builds a DMS + // client from the workspace client, reads resources from the service, and refuses a state + // that tracks resources without the recording marker. It forces WithRecovery off, since the + // service is the source of truth and a leftover WAL is discarded rather than replayed. + WithDeploymentHistory bool ) -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite) error { +// Open reads the deployment state from disk, recovering the WAL when withRecovery is set. +// When withDeploymentHistory is set it builds a DMS client from wsClient and reads resources from +// the service instead, with dmsDeploymentID the id the service holds (empty before the first +// recorded deploy); lineage and serial still come from the file, since that is what the write path +// increments. Open only reads through the client - StartRecording installs the write path once a +// version exists. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, wsClient *databricks.WorkspaceClient, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { db.mu.Lock() defer db.mu.Unlock() @@ -238,7 +372,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W panic(fmt.Sprintf("state already opened: %v, cannot open %v", db.Path, path)) } - err := db.unlockedOpen(ctx, path, withRecovery, withWrite) + err := db.unlockedOpen(ctx, path, withRecovery, withWrite, wsClient, withDeploymentHistory, dmsDeploymentID) if err != nil { // A failed open must leave the receiver closed. unlockedOpen assigns // db.Path before every fallible step, so without this the receiver stays @@ -260,8 +394,20 @@ func (db *DeploymentState) reset() { db.stateIDs = nil } -func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite) error { +func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, wsClient *databricks.WorkspaceClient, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { db.Path = path + + if withDeploymentHistory { + // The service is the source of truth for a recorded deployment, so a leftover WAL is + // discarded below rather than replayed: force recovery off regardless of what was asked. + withRecovery = false + client, err := dms.NewClient(wsClient) + if err != nil { + return err + } + db.dmsClient = client + } + data, err := os.ReadFile(db.Path) if err != nil { if errors.Is(err, fs.ErrNotExist) { @@ -281,14 +427,22 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe } walPath := db.Path + walSuffix - _, err = os.Stat(walPath) - switch { - case errors.Is(err, fs.ErrNotExist): - // no WAL, nothing to do - case err != nil: - return fmt.Errorf("failed to stat WAL file %s: %w", walPath, err) - default: // WAL exists - if withRecovery { + _, statErr := os.Stat(walPath) + walExists := statErr == nil + if statErr != nil && !errors.Is(statErr, fs.ErrNotExist) { + return fmt.Errorf("failed to stat WAL file %s: %w", walPath, statErr) + } + + // A recorded deployment gets its resources from the service (applyDMSState below), so the WAL - + // a local log of an uncommitted deploy - is not merged. A leftover from a crash or a declined + // deploy is removed so the next write's exclusive create of the WAL does not fail. Without + // recording, replay the WAL as usual. + if walExists { + if withDeploymentHistory { + if err := os.Remove(walPath); err != nil { + return fmt.Errorf("removing WAL file %s: %w", walPath, err) + } + } else if withRecovery { if err := db.replayWAL(ctx); err != nil { return fmt.Errorf("reading state from %s: %w", path, err) } @@ -301,6 +455,46 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe return fmt.Errorf("migrating state %s: %w", path, err) } + if err := checkStateFeatures(db.Data.Features); err != nil { + return err + } + + if withDeploymentHistory { + // The service is the source of truth for a recorded deployment; local state is irrelevant. + // This state was pulled from the workspace, so it reflects the remote and not just a local + // cache, and a leftover WAL was discarded above rather than replayed. It records the feature + // once recording owns it, so a recorded deployment carries no resources here (a tombstone) + // and applyDMSState below loads the service's. A state that still tracks resources without + // the feature is a deployment made before recording: treating the service as authoritative + // would create those resources a second time, so refuse it (adopting one is a followup). + if _, recorded := db.Data.Features[featureRecordDeploymentHistory]; !recorded && len(db.Data.State) > 0 { + return errors.New(`this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again`) + } + + // Mark the state as depending on the service so this CLI keeps recording it (see + // RequiresDeploymentHistory). unlockedSave then writes the header alone. Bumping the state + // version so older CLIs refuse a recorded state is a followup. + if db.Data.Features == nil { + db.Data.Features = make(map[string]struct{}, 1) + } + db.Data.Features[featureRecordDeploymentHistory] = struct{}{} + + if dmsDeploymentID != "" { + resources, err := db.dmsClient.ListResources(ctx, dmsDeploymentID) + if err != nil { + return err + } + if err := db.applyDMSState(resources); err != nil { + return err + } + } + } + if withWrite { if err := os.MkdirAll(filepath.Dir(walPath), 0o755); err != nil { return fmt.Errorf("failed to create state directory: %w", err) @@ -405,6 +599,7 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) if header.Serial > expectedSerial { return false, fmt.Errorf("WAL serial (%d) is ahead of expected (%d), state may be corrupted", header.Serial, expectedSerial) } + newSerial = header.Serial newCLIVersion = header.CLIVersion } else { @@ -604,7 +799,7 @@ func (db *DeploymentState) ExportState(ctx context.Context) resourcestate.Export // the WAL. A torn write would therefore leave a state file that Open rejects // next to an intact WAL it never reads. func (db *DeploymentState) unlockedSave() error { - data, err := json.MarshalIndent(db.Data, "", " ") + data, err := json.MarshalIndent(db.dataToPersist(), "", " ") if err != nil { return err } @@ -640,6 +835,20 @@ func (db *DeploymentState) unlockedSave() error { return nil } +// dataToPersist returns what the state file should hold: db.Data itself, unless the deployment +// records its history, in which case it is the header alone. The service holds those resources and +// is where they are read back from, so writing them here too would be a second, never-read copy. +// Returns a copy, leaving the in-memory state - what the rest of the deploy works from - untouched. +func (db *DeploymentState) dataToPersist() Database { + data := db.Data + if _, recorded := data.Features[featureRecordDeploymentHistory]; !recorded { + return data + } + + data.State = map[string]ResourceEntry{} + return data +} + func appendJSONLine(file *os.File, obj any) error { data, err := json.Marshal(obj) if err != nil { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 34ed9ff1c5a..8ccd6277ecb 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -21,14 +21,14 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) @@ -38,7 +38,7 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) mustFinalize(t, &db) _, err := os.Stat(path) @@ -94,10 +94,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true)) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "") }) mustFinalize(t, &db) } @@ -115,12 +115,12 @@ func TestCLIVersionRecordsLastWriter(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db.SaveState("resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) mustFinalize(t, &db) var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, build.GetInfo().Version, reopened.Data.CLIVersion) assert.Equal(t, 2, reopened.Data.Serial) mustFinalize(t, &reopened) @@ -142,7 +142,7 @@ func TestHeaderOnlyWALDoesNotUpdateCLIVersion(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, "0.1.2", recovered.Data.CLIVersion, "a header-only WAL wrote no state, so the version must not move") assert.Equal(t, 1, recovered.Data.Serial) mustFinalize(t, &recovered) @@ -154,12 +154,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -175,60 +175,28 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) mustFinalize(t, &recovered) } -// TestEmptyFeatureStateAcceptedWithoutFlippingVersion pins the special case that a -// featureStateVersion state with no features is accepted as-is — the on-disk version -// is left at featureStateVersion, not flipped down to currentStateVersion — and that -// a featureStateVersion state recording any feature is refused. This is scaffolding -// for the deferred version bump, special-cased to featureStateVersion only (see the -// featureStateVersion doc comment). -// -// When the baseline is actually bumped to featureStateVersion, this special case must -// go away. This test is the forcing function: it fails once featureStateVersion is -// removed, making the author decide what the post-bump behavior should be. -func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { - // The special case applies to featureStateVersion (3) only. - require.Equal(t, 2, currentStateVersion, "when currentStateVersion is bumped, remove featureStateVersion and this special case") - require.Equal(t, 3, featureStateVersion) - - empty := &Database{Header: Header{StateVersion: featureStateVersion}} - require.NoError(t, migrateState(empty)) - assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") - - // v3 that records a feature is refused: this CLI does not understand features. - withFeature := &Database{Header: Header{ - StateVersion: featureStateVersion, - Features: map[string]struct{}{"future_feature": {}}, - }} - err := migrateState(withFeature) - require.Error(t, err) - assert.Contains(t, err.Error(), "requires features this CLI does not support") - assert.Contains(t, err.Error(), "future_feature") - assert.Contains(t, err.Error(), "upgrade to the latest CLI version") - assert.Contains(t, err.Error(), featuresDocURL) -} - func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db2.DeleteState("jobs.my_job")) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", false)) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -240,7 +208,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -252,12 +220,12 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Upgrading to write reuses the same lineage (it goes into the WAL header), // and a write makes it durable. require.NoError(t, db.UpgradeToWrite()) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } @@ -271,13 +239,13 @@ func TestOpenFailureLeavesStateClosed(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte("{not json"), 0o600)) var db DeploymentState - require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) assert.Empty(t, db.Path) // Once the state file is readable, the same receiver opens without panicking. seed := `{"state_version":2,"cli_version":"0.1.2","lineage":"test-lineage","serial":1,"state":{}}` require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) assert.Equal(t, "test-lineage", db.Data.Lineage) mustFinalize(t, &db) } diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index ea7ae1e51b6..764397d5fc3 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/statemgmt/resourcestate" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/structs/structvar" ) @@ -50,6 +51,13 @@ type DeploymentBundle struct { Plan *deployplan.Plan RemoteStateCache sync.Map StateCache structvar.Cache + + // DMS carries deployment-history recording across the phases; nil unless the bundle records + // history. DmsApiClient is set where the state is opened. The deployment id and version live + // in bundle.deployment.history, not here. DmsAsyncOperationClient is opened once the deploy + // phase creates the version, and Apply drains it before returning. + DmsApiClient *dms.Client + DmsAsyncOperationClient *dms.OperationBuffer } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/env/dms.go b/bundle/env/dms.go new file mode 100644 index 00000000000..d7258a76867 --- /dev/null +++ b/bundle/env/dms.go @@ -0,0 +1,22 @@ +package env + +import "context" + +// RecordDeploymentHistoryVariable enables recording without setting the config field. +// Exists for CLI acceptance tests so the whole bundle suite runs with recording on. +// Deliberately undocumented. +const RecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY" + +// RecordDeploymentHistoryEnv reports whether the environment turns on deployment history +// recording. Only "true" turns it on: anything else - including a typo like "TRUE" or "yes" - +// leaves the feature off rather than silently enabling it. +func RecordDeploymentHistoryEnv(ctx context.Context) bool { + value, _ := get(ctx, []string{RecordDeploymentHistoryVariable}) + return value == "true" +} + +// RecordsDeploymentHistory reports whether recording is on, from config or env var. +// Single predicate for all recording code paths; keeps them in sync. +func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { + return configured || RecordDeploymentHistoryEnv(ctx) +} diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go new file mode 100644 index 00000000000..15bfe0e63f0 --- /dev/null +++ b/bundle/env/dms_test.go @@ -0,0 +1,41 @@ +package env + +import ( + "testing" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" +) + +func TestRecordDeploymentHistoryEnv(t *testing.T) { + for _, tc := range []struct { + value string + want bool + }{ + {"true", true}, + {"", false}, + {"0", false}, + {"false", false}, + // Only "true" counts, so a near miss leaves recording off. + {"1", false}, + {"TRUE", false}, + {"yes", false}, + } { + ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, tc.value) + assert.Equal(t, tc.want, RecordDeploymentHistoryEnv(ctx), "value %q", tc.value) + } +} + +func TestRecordDeploymentHistoryEnvUnset(t *testing.T) { + assert.False(t, RecordDeploymentHistoryEnv(t.Context())) +} + +func TestRecordsDeploymentHistory(t *testing.T) { + // The bundle setting alone is enough, and so is the environment; the env var + // exists so the acceptance suite can record without touching every databricks.yml. + assert.True(t, RecordsDeploymentHistory(t.Context(), true)) + assert.False(t, RecordsDeploymentHistory(t.Context(), false)) + + ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, "true") + assert.True(t, RecordsDeploymentHistory(ctx, false)) +} diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index e8b382b370d..7afa577ba8c 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -205,7 +205,7 @@ func BuildStateFromTF( } } - if err := stateDB.SaveState(node, id, sv.Value, dependsOn); err != nil { + if err := stateDB.SaveState(ctx, node, id, sv.Value, dependsOn); err != nil { return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 81f110f5095..eb141ea59b7 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -33,6 +33,11 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, en }() if engine.IsDirect() { + if b.ConfiguresDeploymentHistory(ctx) { + logdiag.LogError(ctx, errors.New("bind is not supported for a bundle that records deployment history")) + return + } + // Direct engine: import into temp state, run plan, check for changes // This follows the same pattern as terraform import groupName, ok := terraform.TerraformToGroupName[opts.ResourceType] @@ -94,7 +99,8 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, en } } else { // Terraform engine: use terraform import - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Import(opts), @@ -129,6 +135,11 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r }() if engine.IsDirect() { + if b.ConfiguresDeploymentHistory(ctx) { + logdiag.LogError(ctx, errors.New("unbind is not supported for a bundle that records deployment history")) + return + } + groupName, ok := terraform.TerraformToGroupName[tfResourceType] if !ok { groupName = tfResourceType @@ -141,7 +152,8 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r return } } else { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Unbind(bundleType, tfResourceType, resourceKey), diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f16088e4518..5fe25dd2eee 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -26,9 +26,11 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) var deployApprovalGroups = []approvalGroup{ @@ -110,7 +112,8 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st return } - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, statemgmt.Load(state), metadata.Compute(), metadata.Upload(), @@ -168,7 +171,8 @@ func logDeploySummary(ctx context.Context, b *bundle.Bundle, plan *deployplan.Pl // It also cleans up the artifacts directory and transforms wheel tasks. // It is called by only "bundle deploy". func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]libraries.LocationToUpdate) { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, artifacts.CleanUp(), libraries.Upload(libs), ) @@ -179,12 +183,13 @@ func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]li // stateEngine is the engine the resolved state file uses; requestedEngine is // what bundle.engine / DATABRICKS_BUNDLE_ENGINE asked for and may differ (used // only by the post-deploy migration check). -func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler, stateEngine engine.EngineType, requestedEngine engine.EngineSetting, libs map[string][]libraries.LocationToUpdate, plan *deployplan.Plan) { +func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler, stateEngine engine.EngineType, requestedEngine engine.EngineSetting, libs map[string][]libraries.LocationToUpdate, plan *deployplan.Plan, dmsDeployment *bundledeployments.Deployment) { log.Info(ctx, "Phase: deploy") // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, scripts.Execute(config.ScriptPreDeploy), lock.Acquire(lock.GoalDeploy), ) @@ -195,7 +200,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } // lock is acquired here + // + // The version is created only after approval; CompleteVersion is deferred before + // lock.Release and no-ops until then. defer func() { + if _, err := drainOperationsAndCompleteVersion(ctx, b, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() @@ -209,7 +220,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Upload all source files and built artifacts as a single immutable snapshot. // snapshot.Upload() sets workspace.snapshot_path; the variable-resolution // pass expands ${workspace.snapshot_path} placeholders written by translate_paths. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, snapshot.Upload(), mutator.ResolveVariableReferencesOnlyResources("workspace"), ) @@ -246,7 +258,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } }() - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, deploy.StateUpdate(), deploy.StatePush(), permissions.ApplyWorkspaceRootPermissions(), @@ -259,6 +272,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } planFromFile := plan != nil + if plan == nil { // State is already open for read by process.go (for direct engine) plan = RunPlan(ctx, b, stateEngine) @@ -280,7 +294,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } if planFromFile { - // Initialize DeploymentBundle for applying the loaded plan + // Parse the loaded plan into the state cache the apply reads. A first deploy's deployment id + // is stamped later, after approval creates it (see below). err := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan) if err != nil { logdiag.LogError(ctx, err) @@ -295,17 +310,53 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - haveApproval, err := approvalForDeploy(ctx, b, plan) + haveApproval, approvalErr := approvalForDeploy(ctx, b, plan) + if !haveApproval { + // No version was created, so the deferred CompleteVersion is a no-op and the version + // number is left for the next deploy. Both the user declining and a console that + // cannot prompt land here. + if approvalErr != nil { + logdiag.LogError(ctx, approvalErr) + return + } + cmdio.LogString(ctx, "Deployment cancelled!") + return + } + + // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. + // A first deploy's id did not exist at plan time - the version and any existing id were stamped + // then - so stamp the one just created into the plan the apply reads. + if b.DeploymentBundle.DmsApiClient != nil { + existingID, _ := deploymentAndNextVersion(b) + createOrUpdateDeployment(ctx, b, dmsDeployment) + if logdiag.HasError(ctx) { + return + } + if existingID == "" { + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.StampDeploymentID(deploymentID); err != nil { + logdiag.LogError(ctx, err) + return + } + } + } + + // Create the version the plan was stamped with, staging an operation for every resource + // it touches. Doing it here rather than before the prompt means a declined deploy never + // claims a version number. + staged, err := stagedOperations(plan) if err != nil { logdiag.LogError(ctx, err) return } - if haveApproval { - deployCore(ctx, b, plan, stateEngine) - } else { - cmdio.LogString(ctx, "Deployment cancelled!") + if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { + logdiag.LogError(ctx, err) return } + deploymentID, versionID := deploymentAndNextVersion(b) + logDeploymentVersion(ctx, b, deploymentID, versionID) + + deployCore(ctx, b, plan, stateEngine) if logdiag.HasError(ctx) { return @@ -358,7 +409,8 @@ func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *d // b.Select is rejected for the terraform engine in ProcessBundleRet, so it is // never set here. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Plan(terraform.PlanGoal("deploy")), diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 201c3928e28..720f4ef7fa9 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/apierr" @@ -158,6 +159,21 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } + // Complete version before deleting remote files; the deployment node is under statePath. + completed, err := drainOperationsAndCompleteVersion(ctx, b, true) + if err != nil { + logdiag.LogError(ctx, err) + return + } + // A completed destroy's resources are gone, so its deployment record is deleted too. + if completed { + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.DmsApiClient.DeleteDeployment(ctx, deploymentID); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + return + } + } + bundle.ApplyContext(ctx, b, files.Delete()) if !logdiag.HasError(ctx) && b.Quiet < bundle.QuietAll { @@ -196,12 +212,26 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } + // DMS recording of this destroy: the version is created after approval, so a cancelled + // destroy records nothing. Deferred before lock.Release to hold the lock; a no-op once + // destroyCore has completed the version. defer func() { + completed, err := drainOperationsAndCompleteVersion(ctx, b, !logdiag.HasError(ctx)) + if err != nil { + logdiag.LogError(ctx, err) + } else if completed { + // A completed destroy's resources are gone, so its deployment record is deleted too. + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.DmsApiClient.DeleteDeployment(ctx, deploymentID); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + } + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) }() if !engine.IsDirect() { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, // We need to resolve artifact variable (how we do it in build phase) // because some of the to-be-destroyed resource might use this variable. // Not resolving might lead to terraform "Reference to undeclared resource" error @@ -253,6 +283,22 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } } + // Record a destroy version now that it is approved and the state WAL is open, but only when + // the state is recorded with DMS (its marker is set). This is the single gate on the + // destroy's recording: it opens the operation buffer that destroyCore drains and completes, + // so the drain and the deployment delete follow only when a version was started here. + // Destroy never creates or updates the deployment - it is about to be deleted. + if engine.IsDirect() && b.DeploymentBundle.StateDB.RequiresDeploymentHistory() { + staged, err := stagedOperations(plan) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if err := startVersion(ctx, b, dms.VersionTypeDestroy, staged); err != nil { + logdiag.LogError(ctx, err) + return + } + } destroyCore(ctx, b, plan, engine) } else { cmdio.LogString(ctx, "Destroy cancelled!") diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go new file mode 100644 index 00000000000..40257fe5f5c --- /dev/null +++ b/bundle/phases/dms.go @@ -0,0 +1,249 @@ +package phases + +import ( + "context" + "fmt" + "net/url" + "strconv" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// stagedOperations lists the resources the plan will touch, for CreateVersion to stage an +// operation each. Skipped and undefined actions are left out: nothing is applied for them, so +// their operations would stay pending and the service would hold no state for them. +func stagedOperations(plan *deployplan.Plan) ([]dms.StagedOperation, error) { + actions := plan.GetActions() + staged := make([]dms.StagedOperation, 0, len(actions)) + for _, action := range actions { + if action.ActionType == deployplan.Skip || action.ActionType == deployplan.Undefined { + continue + } + actionType, err := actionToSDK(action.ActionType) + if err != nil { + return nil, fmt.Errorf("%s: %w", action.ResourceKey, err) + } + staged = append(staged, dms.StagedOperation{ + ResourceKey: action.ResourceKey, + ActionType: actionType, + }) + } + return staged, nil +} + +// actionToSDK maps a deployplan action to the DMS action type a staged operation records. +// Only actions that mutate a resource are recordable; Skip and Undefined are rejected +// rather than silently coerced. +func actionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType, error) { + switch a { + case deployplan.Create: + return bundledeployments.OperationActionTypeOperationActionTypeCreate, nil + case deployplan.Update: + return bundledeployments.OperationActionTypeOperationActionTypeUpdate, nil + case deployplan.UpdateWithID: + return bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId, nil + case deployplan.Recreate: + return bundledeployments.OperationActionTypeOperationActionTypeRecreate, nil + case deployplan.Resize: + return bundledeployments.OperationActionTypeOperationActionTypeResize, nil + case deployplan.Delete: + return bundledeployments.OperationActionTypeOperationActionTypeDelete, nil + default: + return "", fmt.Errorf("cannot record operation: unsupported action %q", a) + } +} + +// deploymentAndNextVersion reads, from the history the recording set, the deployment id and the version +// this run will create (one past the deployment's most recent). Both zero when nothing is recorded +// - the bundle records no history, so version 0 marks "no version" for logDeploymentVersion. +func deploymentAndNextVersion(b *bundle.Bundle) (string, int64) { + h := b.Config.Bundle.Deployment.History + if h == nil { + return "", 0 + } + // LatestVersionID comes from the service as an integer, so this does not fail in practice. + version, err := dms.NextVersion(h.LatestVersionID) + if err != nil { + version = 1 + } + return h.DeploymentID, version +} + +// createOrUpdateDeployment creates the deployment on a first deploy, or updates the metadata this +// run changed. current is the record the service holds (nil before the first recorded deploy), +// diffed to mask the update down to what changed. Runs after approval, so a declined deploy leaves +// no deployment behind; a first deploy's new id is then stamped into the plan (StampDeploymentID). +func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bundledeployments.Deployment) { + db := &b.DeploymentBundle + metadata := deploymentMetadata(b) + deploymentID, _ := deploymentAndNextVersion(b) + switch mask := metadata.StaleFields(current); { + case deploymentID == "": + id, err := db.DmsApiClient.CreateDeployment(ctx, b.Config.Workspace.StatePath, metadata) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to create deployment: %w", err)) + return + } + deploymentID = id + case mask != "": + if err := db.DmsApiClient.UpdateDeployment(ctx, deploymentID, metadata, mask); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to update deployment: %w", err)) + return + } + } + + // A first deploy had no deployment to read at startup, so its id enters the history here. + bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { + if b.Config.Bundle.Deployment.History == nil { + b.Config.Bundle.Deployment.History = &config.DeploymentHistory{} + } + b.Config.Bundle.Deployment.History.DeploymentID = deploymentID + }) +} + +// startVersion claims the version the run settled on and opens the buffer that records +// each state write under it. Called after approval, so a declined deploy never claims a number. +// A no-op when the bundle does not record deployment history. +func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.VersionType, staged []dms.StagedOperation) error { + db := &b.DeploymentBundle + if db.DmsApiClient == nil { + return nil + } + deploymentID, versionID := deploymentAndNextVersion(b) + + // The version this run creates is one past the deployment's most recent, so the previous is + // one below it - empty for the first version. + previousVersionID := "" + if versionID > 1 { + previousVersionID = strconv.FormatInt(versionID-1, 10) + } + + // The server rejects this unless the version number exceeds last_version_id and + // previous_version_id matches it, which is what makes claiming the number up front + // safe: a deploy that took it in the meantime is reported, not overwritten. + var gitInfo *bundledeployments.GitInfo + if git := b.Config.Bundle.Git; git.Branch != "" || git.Commit != "" || git.OriginURL != "" { + gitInfo = &bundledeployments.GitInfo{ + Branch: git.Branch, + Commit: git.Commit, + OriginUrl: git.OriginURL, + } + } + version, err := db.DmsApiClient.CreateVersion(ctx, deploymentID, versionID, dms.CreateVersionRequest{ + CliVersion: build.GetInfo().Version, + VersionType: versionType, + PreviousVersionId: previousVersionID, + Operations: staged, + GitInfo: gitInfo, + }) + if err != nil { + return fmt.Errorf("failed to create deployment version: %w", err) + } + log.Infof(ctx, "Created deployment version: deployment=%s version=%s", deploymentID, version.VersionId) + + db.DmsAsyncOperationClient = db.StateDB.StartRecording(ctx, deploymentID, versionID) + return nil +} + +// drainOperationsAndCompleteVersion drains the buffered operations and completes the version. +// It returns whether the version completed successfully, which a destroy uses to decide whether +// to delete the deployment. Idempotent: the first call clears the buffer, so a deferred +// safety-net after an explicit completion is a no-op returning false. Also false when no version +// was created. +func drainOperationsAndCompleteVersion(ctx context.Context, b *bundle.Bundle, success bool) (bool, error) { + db := &b.DeploymentBundle + buf := db.DmsAsyncOperationClient + if buf == nil { + return false, nil + } + db.DmsAsyncOperationClient = nil + + // Drain first; a recording error fails the deploy even when the resources applied. Its error + // is left to whoever drained (apply), not returned twice. + if buf.Drain() != nil { + success = false + } + + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if !success { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + deploymentID, versionID := deploymentAndNextVersion(b) + if err := db.DmsApiClient.CompleteVersion(ctx, deploymentID, versionID, reason); err != nil { + return false, err + } + log.Infof(ctx, "Completed deployment version: deployment=%s version=%d reason=%s", deploymentID, versionID, reason) + + return success, nil +} + +// logDeploymentVersion logs the deployment version URL. Workspace ID is omitted +// so the page stays clickable in a terminal and redirects correctly without it. +func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, deploymentID string, version int64) { + if version == 0 { + return + } + + baseURL, err := url.Parse(b.WorkspaceClient(ctx).Config.CanonicalHostName()) + if err != nil { + // Only the link is lost, so report the version without it rather than failing + // a deploy over it. + log.Debugf(ctx, "Not linking to the recorded deployment: %s", err) + cmdio.LogString(ctx, fmt.Sprintf("Current Deployment Version: %s version %d", deploymentID, version)) + return + } + + cmdio.LogString(ctx, "Current Deployment Version: "+workspaceurls.DeploymentURL(*baseURL, deploymentID, version)) +} + +// deploymentMetadata describes the bundle this deploy came from and where it +// landed, mirroring what bundle/deploy/metadata computes for the metadata file. +func deploymentMetadata(b *bundle.Bundle) dms.Metadata { + p := dms.Metadata{ + DisplayName: b.Config.Bundle.Name, + TargetName: b.Config.Bundle.Target, + Mode: deploymentModeToSDK(b.Config.Bundle.Mode), + } + + ws := &bundledeployments.WorkspaceInfo{ + RootPath: b.Config.Workspace.RootPath, + FilePath: b.Config.Workspace.FilePath, + } + // In a source-linked deployment files are not copied, so resources read them + // from the sync root instead of file_path (see bundle/deploy/metadata.Compute). + if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { + ws.FilePath = b.SyncRootPath + ws.SourceLinked = true + } + // Only a deploy from a Databricks Git folder has one; a local worktree does not. + // bundle_root_path is relative to it, so the service requires both or neither. + if b.WorktreeRoot != nil && strings.HasPrefix(b.WorktreeRoot.Native(), "/Workspace/") { + ws.GitFolderPath = b.WorktreeRoot.Native() + ws.BundleRootPath = b.Config.Bundle.Git.BundleRootPath + } + p.Workspace = ws + return p +} + +// deploymentModeToSDK maps the bundle target's mode to the DMS enum. An unset mode +// maps to empty, which the service reads as "not reported". +func deploymentModeToSDK(mode config.Mode) bundledeployments.DeploymentMode { + switch mode { + case config.Development: + return bundledeployments.DeploymentModeDeploymentModeDevelopment + case config.Production: + return bundledeployments.DeploymentModeDeploymentModeProduction + default: + return "" + } +} diff --git a/bundle/phases/dms_test.go b/bundle/phases/dms_test.go new file mode 100644 index 00000000000..f6854999db9 --- /dev/null +++ b/bundle/phases/dms_test.go @@ -0,0 +1,81 @@ +package phases + +import ( + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStagedOperationsCoversEveryTouchedResource(t *testing.T) { + // The version fixes its operation set, so anything the apply will write has to appear + // here. Keys go out in the service's form, without the CLI's "resources." prefix. + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.foo": {Action: deployplan.Create}, + "resources.pipelines.bar": {Action: deployplan.Recreate}, + "resources.schemas.baz": {Action: deployplan.Delete}, + "resources.clusters.small": {Action: deployplan.Resize}, + }} + + staged, err := stagedOperations(plan) + require.NoError(t, err) + + assert.ElementsMatch(t, []dms.StagedOperation{ + {ResourceKey: "resources.jobs.foo", ActionType: bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {ResourceKey: "resources.pipelines.bar", ActionType: bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {ResourceKey: "resources.schemas.baz", ActionType: bundledeployments.OperationActionTypeOperationActionTypeDelete}, + {ResourceKey: "resources.clusters.small", ActionType: bundledeployments.OperationActionTypeOperationActionTypeResize}, + }, staged) +} + +func TestStagedOperationsLeavesOutUntouchedResources(t *testing.T) { + // A skipped resource is never applied, so staging it would leave an operation pending for + // the life of the version. Undefined is not a real action either. + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.touched": {Action: deployplan.Update}, + "resources.jobs.unchanged": {Action: deployplan.Skip}, + "resources.jobs.unknown": {Action: deployplan.Undefined}, + }} + + staged, err := stagedOperations(plan) + require.NoError(t, err) + + assert.Equal(t, []dms.StagedOperation{ + {ResourceKey: "resources.jobs.touched", ActionType: bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + }, staged) +} + +func TestStagedOperationsEmptyPlan(t *testing.T) { + staged, err := stagedOperations(&deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{}}) + require.NoError(t, err) + assert.Empty(t, staged) +} + +func TestActionToSDK(t *testing.T) { + cases := []struct { + action deployplan.ActionType + want bundledeployments.OperationActionType + }{ + {deployplan.Create, bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {deployplan.Update, bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + {deployplan.UpdateWithID, bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId}, + {deployplan.Recreate, bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {deployplan.Resize, bundledeployments.OperationActionTypeOperationActionTypeResize}, + {deployplan.Delete, bundledeployments.OperationActionTypeOperationActionTypeDelete}, + } + for _, c := range cases { + got, err := actionToSDK(c.action) + require.NoError(t, err) + assert.Equal(t, c.want, got) + } + + // Nothing is applied for these, so stagedOperations leaves them out rather than + // mapping them; the guard is here in case a new action type arrives without a mapping. + _, err := actionToSDK(deployplan.Skip) + assert.Error(t, err) + _, err = actionToSDK(deployplan.Undefined) + assert.Error(t, err) +} diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 7b88290930c..5d54e8702e5 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -172,7 +172,8 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s }() var planBundle direct.DeploymentBundle - if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false); err != nil { + // This plan check does not record deployment history, so no DMS client or deployment id. + if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, nil, dstate.WithDeploymentHistory(false), ""); err != nil { return fmt.Errorf("opening migrated state for plan check: %w", err) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 086ec1d600a..14bc85777c1 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 6d938c5e03d..ea37ff50990 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index fa5471e8f99..1d671fbc306 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -4,13 +4,17 @@ import ( "context" "errors" "fmt" + "path" "path/filepath" + "strconv" "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" @@ -21,11 +25,15 @@ import ( "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" "golang.org/x/mod/semver" ) @@ -189,6 +197,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, nil, err } + // The current deployment read from the service (nil, id "" if there is none yet). Used for the + // metadata diff and to reject a saved plan that predates the deployment's recorded version. + var dmsDeployment *bundledeployments.Deployment + var dmsDeploymentID string + shouldReadState := opts.ReadState || opts.AlwaysPull || opts.InitIDs || opts.ErrorOnEmptyState || opts.PreDeployChecks || opts.Deploy || opts.ReadPlanPath != "" if shouldReadState { @@ -232,11 +245,65 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle needDirectState := stateDesc.Engine.IsDirect() && (opts.InitIDs || opts.ErrorOnEmptyState || opts.Deploy || opts.ReadPlanPath != "" || opts.PreDeployChecks || opts.PostStateFunc != nil) if needDirectState { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + + if b.ConfiguresDeploymentHistory(ctx) { + deploymentID, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + if err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } + + dmsDeploymentID = deploymentID + dmsDeployment = deployment + + // Stamp the deployment and the version this run records onto every job and pipeline so + // the plan carries them. version_id is always known (last recorded + 1); deployment_id + // does not exist until a first deploy creates it, so it is left off here and the deploy + // phase stamps the created id. + lastVersionID := "" + if deployment != nil { + lastVersionID = deployment.LastVersionId + } + nextVersion, verr := dms.NextVersion(lastVersionID) + if verr != nil { + logdiag.LogError(ctx, verr) + return b, stateDesc, root.ErrAlreadyPrinted + } + muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} + if deploymentID != "" { + bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { + b.Config.Bundle.Deployment.History = &config.DeploymentHistory{ + DeploymentID: deploymentID, + LatestVersionID: deployment.LastVersionId, + } + }) + muts = append(muts, metadata.AnnotateDeployment(deploymentID)) + } + bundle.ApplySeqContext(ctx, b, muts...) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), b.WorkspaceClient(ctx), dstate.WithDeploymentHistory(b.ConfiguresDeploymentHistory(ctx)), dmsDeploymentID); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } + // Open built the DMS client from the workspace client when recording; hand it to the + // phases that create versions and deployments. + if b.ConfiguresDeploymentHistory(ctx) { + b.DeploymentBundle.DmsApiClient = b.DeploymentBundle.StateDB.DmsClient() + } + + // The service holds this deployment, so it has to keep being recorded: deploying + // without recording would leave it describing resources that have moved on. + if !b.ConfiguresDeploymentHistory(ctx) && b.DeploymentBundle.StateDB.RequiresDeploymentHistory() { + logdiag.LogError(ctx, errors.New(`unsetting experimental.record_deployment_history is not supported + +This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`)) + return b, stateDesc, root.ErrAlreadyPrinted + } + // Warn when the state was last written by a newer CLI than the one // running now. The state schema version is a hard gate (dstate.Open // rejects a too-new state_version), but a state can be written by a @@ -277,6 +344,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } } + } var plan *deployplan.Plan @@ -300,8 +368,27 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle log.Warnf(ctx, "Plan was created with CLI version %s but current version is %s", plan.CLIVersion, currentVersion) } - // Validate that the plan's lineage and serial match the current state - // This must happen before any file operations + // The plan records the DMS deployment and version it targeted. Reject it if the live + // deployment moved on - a newer version (someone deployed since, possibly from another + // machine) or a different id (deleted and recreated) - so a stale plan is never applied on + // top of newer state. This runs before the lineage/serial check and is the authoritative + // stale guard for recorded bundles, whose local state is only a tombstone (its serial does + // not catch a deploy from elsewhere). Both sides are empty when not recording, a no-op there. + remoteLastVersion := "" + if dmsDeployment != nil { + remoteLastVersion = dmsDeployment.LastVersionId + } + if plan.LastVersionId != remoteLastVersion { + logdiag.LogError(ctx, fmt.Errorf("this plan predates the deployment's current version %s; run 'bundle plan' again", remoteLastVersion)) + return b, stateDesc, root.ErrAlreadyPrinted + } + if plan.DeploymentId != dmsDeploymentID { + logdiag.LogError(ctx, errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again")) + return b, stateDesc, root.ErrAlreadyPrinted + } + + // Validate that the plan's lineage and serial match the local state. This is the stale guard + // for non-recorded bundles (recorded ones are covered by the version check above). err = direct.ValidatePlanAgainstState(&b.DeploymentBundle.StateDB, plan) if err != nil { logdiag.LogError(ctx, err) @@ -373,7 +460,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } t3 := time.Now() - phases.Deploy(ctx, b, outputHandler, stateDesc.Engine, requiredEngine, libs, plan) + phases.Deploy(ctx, b, outputHandler, stateDesc.Engine, requiredEngine, libs, plan, dmsDeployment) b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ Key: "phases.Deploy", Value: time.Since(t3).Milliseconds(), @@ -427,6 +514,27 @@ func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineS return engine.EngineSetting{Type: engine.Default, Source: engine.SourceDefault, IsDefault: true}, nil } +// Lookup and return the deployment object from ${workspace.state_path}/resources.deployment.json +func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { + nodePath := path.Join(statePath, dms.DeploymentNodeName) + + obj, err := w.Workspace.GetStatusByPath(ctx, nodePath) + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + return "", nil, nil + } + if err != nil { + return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + } + deploymentID := strconv.FormatInt(obj.ObjectId, 10) + deployment, err := w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: dms.DeploymentName(deploymentID), + }) + if err != nil { + return "", nil, err + } + return deploymentID, deployment, nil +} + // isNewerVersion reports whether the state's recorded CLI version is strictly // newer than the running build. Both are bare versions without a leading "v". // An empty stateVersion (state not written by any CLI yet) or an unparseable diff --git a/libs/dms/client.go b/libs/dms/client.go new file mode 100644 index 00000000000..8dbf9915399 --- /dev/null +++ b/libs/dms/client.go @@ -0,0 +1,250 @@ +package dms + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// DeploymentNodeName is the workspace node DMS creates per deployment. Must +// match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. +const DeploymentNodeName = "resources.deployment.json" + +// statePrefix is what a bundle state key carries and a DMS resource key does not: state calls a +// job "resources.jobs.foo", DMS calls it "jobs.foo". Every exported name here takes the state +// form; the prefix comes off where a request is built, and back on where a resource is read. +const statePrefix = "resources." + +// Client carries the calls the CLI makes to DMS, as methods below. Each one goes out through one +// of two halves: the generated client for the calls it can express, and hand-written requests +// for the two it cannot. +// +// TODO: Remove this and replace with the SDK. +type Client struct { + // Service is the generated client. + Service bundledeployments.BundleDeploymentsInterface + + // raw sends what the generated client cannot; see requester. + raw requester +} + +// NewClient returns a Client for the workspace w. +func NewClient(w *databricks.WorkspaceClient) (*Client, error) { + api, err := client.New(w.Config) + if err != nil { + return nil, err + } + return &Client{Service: w.BundleDeployments, raw: &rawClient{client: api}}, nil +} + +// DeploymentName and versionName are the two resource-name formats the service uses. Every +// call builds its name here, so a caller only ever passes ids. +func DeploymentName(deploymentID string) string { + return "deployments/" + deploymentID +} + +func versionName(deploymentID string, version int64) string { + return fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version) +} + +// CreateDeployment registers a deployment under parentPath and returns the id the server +// assigned it, which is the id of the workspace node it creates there. +func (c *Client) CreateDeployment(ctx context.Context, parentPath string, metadata Metadata) (string, error) { + dep := metadata.deployment() + dep.InitialParentPath = parentPath + + created, err := c.Service.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{Deployment: dep}) + if err != nil { + return "", err + } + return deploymentIDFromName(created.Name) +} + +// UpdateDeployment writes the fields mask names onto the deployment. The service ignores every +// other field, so the mask is what decides the write. +func (c *Client) UpdateDeployment(ctx context.Context, deploymentID string, metadata Metadata, mask string) error { + return c.raw.UpdateDeployment(ctx, deploymentID, metadata.deployment(), mask) +} + +// DeleteDeployment removes the deployment record, which a completed destroy does. +func (c *Client) DeleteDeployment(ctx context.Context, deploymentID string) error { + return c.Service.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ + Name: DeploymentName(deploymentID), + }) +} + +// CreateVersion claims the version and stages the operations body carries. +func (c *Client) CreateVersion(ctx context.Context, deploymentID string, version int64, body CreateVersionRequest) (*bundledeployments.Version, error) { + return c.raw.CreateVersion(ctx, deploymentID, strconv.FormatInt(version, 10), body) +} + +// CompleteVersion closes the version out, which is what stops the service expiring its lease. +func (c *Client) CompleteVersion(ctx context.Context, deploymentID string, version int64, reason bundledeployments.VersionComplete) error { + _, err := c.Service.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: versionName(deploymentID, version), + CompletionReason: reason, + }) + return err +} + +// UpdateOperation fills in one operation the version staged, and returns the sequence id the +// next update for that resource must send. +func (c *Client) UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (string, error) { + return c.raw.UpdateOperation(ctx, deploymentID, version, stateKey, sequenceID, update) +} + +// deploymentIDFromName extracts the deployment ID from a DMS resource name of +// the form "deployments/{deployment_id}". +func deploymentIDFromName(name string) (string, error) { + id, ok := strings.CutPrefix(name, DeploymentName("")) + if !ok || id == "" { + return "", fmt.Errorf("unexpected deployment name %q from deployment metadata service", name) + } + return id, nil +} + +// requester sends the two requests the generated client cannot express, so a test can capture +// what the CLI puts on the wire. Both are TODO(DMS): drop them once the spec catches up. +type requester interface { + // CreateVersion is hand-written because the generated struct has no operations: the field + // is at DEVELOPMENT stage, which keeps it out of the SDK until it is promoted. + CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) + + // UpdateDeployment is hand-written because the generated client has no such call yet. + UpdateDeployment(ctx context.Context, deploymentID string, deployment bundledeployments.Deployment, mask string) error + + // UpdateOperation is hand-written because the SDK types sequence_id as an int64 while + // the service sends a JSON string, so it cannot read the response. sequenceID is the + // token the previous update for this resource returned, or 0 for the first, which is + // what staging leaves. + UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (next string, err error) +} + +// CreateVersionRequest is the CreateVersion request body. +type CreateVersionRequest struct { + CliVersion string `json:"cli_version"` + VersionType VersionType `json:"version_type"` + // PreviousVersionId is the deployment's most recent version, unset for a + // deployment's first version. + PreviousVersionId string `json:"previous_version_id,omitempty"` + // GitInfo records where this version's source came from. The rest of the provenance - + // display name, target, mode, workspace paths - belongs to the deployment. + GitInfo *bundledeployments.GitInfo `json:"git_info,omitempty"` + // Operations is every resource this version will touch; see StagedOperation. It sits in this + // body with the version's own fields because the request binds body: "version", and is input + // only - the response never carries it back. + Operations []StagedOperation `json:"operations,omitempty"` +} + +// StagedOperation is one resource the version will record an operation for. The service +// creates it in OPERATION_STATUS_PENDING at sequence id 0, and the CLI fills in the outcome +// with UpdateOperation as the resource is applied. +type StagedOperation struct { + // ResourceKey is the bundle state key; the request carries the form the service uses. + ResourceKey string `json:"resource_key"` + ActionType bundledeployments.OperationActionType `json:"action_type"` +} + +// operationResponse is the part of an operation response the CLI reads back. +type operationResponse struct { + // SequenceId is the concurrency token for the next update, typed as the service sends it. + SequenceId string `json:"sequence_id,omitempty"` +} + +// rawClient sends the requests the generated client cannot express. +type rawClient struct { + client *client.DatabricksClient +} + +func (r *rawClient) CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) { + staged := make([]StagedOperation, len(body.Operations)) + for i, op := range body.Operations { + op.ResourceKey = strings.TrimPrefix(op.ResourceKey, statePrefix) + staged[i] = op + } + body.Operations = staged + + var version bundledeployments.Version + path := "/api/2.0/bundle/" + DeploymentName(deploymentID) + "/versions" + err := r.client.Do(ctx, http.MethodPost, path, + auth.WorkspaceIDHeaders(r.client.Config), + map[string]any{"version_id": versionID}, + body, &version) + if err != nil { + return nil, err + } + return &version, nil +} + +// newDeploymentUpdate builds the request body for update, holding exactly the masked fields for +// the same reason newUpdateRequest does. A map, not the SDK struct, whose omitempty tags would +// drop a masked field that is empty - which is how deployment_mode is cleared when a target stops +// setting mode. +func newDeploymentUpdate(deployment bundledeployments.Deployment, mask string) map[string]any { + values := map[string]any{ + "display_name": deployment.DisplayName, + "target_name": deployment.TargetName, + "deployment_mode": deployment.DeploymentMode, + "workspace_info": deployment.WorkspaceInfo, + } + + body := map[string]any{} + for field := range strings.SplitSeq(mask, ",") { + body[field] = values[field] + } + return body +} + +func (r *rawClient) UpdateDeployment(ctx context.Context, deploymentID string, deployment bundledeployments.Deployment, mask string) error { + path := "/api/2.0/bundle/" + DeploymentName(deploymentID) + return r.client.Do(ctx, http.MethodPatch, path, + auth.WorkspaceIDHeaders(r.client.Config), + map[string]any{"update_mask": mask}, + newDeploymentUpdate(deployment, mask), nil) +} + +// newUpdateRequest builds the request body for update. A field is in the body when the mask +// names it and absent otherwise, which is what the service requires: it rejects an update +// whose mask names a field the body leaves out, and an empty value is how a field is cleared - +// no state means the resource is gone, no error_message means an earlier failure is resolved. +// A map, not a struct, so presence cannot drift from the mask through an omitempty tag. +func newUpdateRequest(update OperationUpdate, sequenceID string) map[string]any { + body := map[string]any{"sequence_id": sequenceID} + // A masked state with no value is how the service is told the resource is gone, so a nil + // state is left out rather than sent empty. + if update.Fields.Has(FieldState) && update.State != nil { + body["state"] = string(update.State) + } + if update.Fields.Has(FieldResourceID) { + body["resource_id"] = update.ResourceID + } + if update.Fields.Has(FieldErrorMessage) { + body["error_message"] = update.ErrorMessage + } + if update.Fields.Has(FieldStatus) { + body["status"] = update.Status + } + return body +} + +func (r *rawClient) UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (string, error) { + body := newUpdateRequest(update, sequenceID) + + var result operationResponse + path := "/api/2.0/bundle/" + versionName(deploymentID, version) + "/operations/" + strings.TrimPrefix(stateKey, statePrefix) + err := r.client.Do(ctx, http.MethodPatch, path, + auth.WorkspaceIDHeaders(r.client.Config), + map[string]any{"update_mask": update.Fields.Mask()}, + body, &result) + if err != nil { + return "", err + } + return result.SequenceId, nil +} diff --git a/libs/dms/client_test.go b/libs/dms/client_test.go new file mode 100644 index 00000000000..867655bca4e --- /dev/null +++ b/libs/dms/client_test.go @@ -0,0 +1,67 @@ +package dms + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClientNamesEveryResourceTheSameWay(t *testing.T) { + // One format each, so a call only ever passes ids. + assert.Equal(t, "deployments/dep-1", DeploymentName("dep-1")) + assert.Equal(t, "deployments/dep-1/versions/2", versionName("dep-1", 2)) +} + +func TestDeploymentIDFromName(t *testing.T) { + id, err := deploymentIDFromName("deployments/abc-123") + require.NoError(t, err) + assert.Equal(t, "abc-123", id) + + _, err = deploymentIDFromName("abc-123") + assert.Error(t, err) + + _, err = deploymentIDFromName("deployments/") + assert.Error(t, err) +} + +func TestUpdateRequestSendsExactlyTheMaskedFields(t *testing.T) { + // The service rejects an update whose mask names a field the body leaves out, and treats a + // field it does carry as written - so the body has to hold every masked field and nothing + // else, empty values included. Both masks the CLI builds are asserted on the wire by + // acceptance/bundle/dms; this pins the rule they both rely on. + update := OperationUpdate{ + Fields: DescribesResource, + State: json.RawMessage(`{"state":{"name":"foo"}}`), + ResourceID: "job-1", + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + } + + // A successful write reports no error, and the empty value is what clears an earlier one. + assert.Equal(t, map[string]any{ + "state": `{"state":{"name":"foo"}}`, + "resource_id": "job-1", + "error_message": "", + "status": bundledeployments.OperationStatusOperationStatusSucceeded, + "sequence_id": "3", + }, newUpdateRequest(update, "3")) + + // A failure keeps the recorded state, so state is absent rather than empty: naming it + // would clear what the resource last recorded. + assert.Equal(t, map[string]any{ + "error_message": "boom", + "status": bundledeployments.OperationStatusOperationStatusFailed, + "sequence_id": "3", + }, newUpdateRequest(NewFailureUpdate("job-1", nil, errors.New("boom")), "3")) + + // The deployment's own fields follow the same rule. Clearing deployment_mode - a target that + // stops setting mode - sends it empty, which the SDK struct's omitempty would have dropped. + deployment := Metadata{DisplayName: "b", TargetName: "t"}.deployment() + assert.Equal(t, map[string]any{ + "target_name": "t", + "deployment_mode": bundledeployments.DeploymentMode(""), + }, newDeploymentUpdate(deployment, "target_name,deployment_mode")) +} diff --git a/libs/dms/fields.go b/libs/dms/fields.go new file mode 100644 index 00000000000..05d39b387d7 --- /dev/null +++ b/libs/dms/fields.go @@ -0,0 +1,59 @@ +package dms + +import "strings" + +// Fields is the set of operation fields an update writes, sent as its update mask. The +// service rejects any other path, so this is the whole vocabulary. +type Fields uint8 + +const ( + FieldState Fields = 1 << iota + FieldErrorMessage + FieldResourceID + FieldStatus +) + +// DescribesResource is what a write that says how the resource looks claims: every field +// an update may change. +const DescribesResource = FieldState | FieldErrorMessage | FieldResourceID | FieldStatus + +// ClearsState is what a write that leaves no resource claims: a delete, and the delete half of +// a recreate. State is named so the service reads the absent value as a clear, which is what +// drops the resource from the deployment. It cannot name resource_id either: the service counts +// only a state it was given as recording one, so an id alongside a cleared state is refused. +const ClearsState = FieldState | FieldErrorMessage | FieldStatus + +// KeepsState is what a failure claims: mark it failed and leave state alone. State means +// the resource is as it was written; no state means a delete went through and nothing +// replaced it, so the resource really is gone and the deployment should say so. +// +// It cannot name resource_id: the service requires state in any mask that names the id, and +// naming state here would overwrite what the resource last recorded. +const KeepsState = FieldErrorMessage | FieldStatus + +// wireNames pairs each field with its name on the wire, in the order a mask lists them. +var wireNames = []struct { + field Fields + name string +}{ + {FieldState, "state"}, + {FieldErrorMessage, "error_message"}, + {FieldResourceID, "resource_id"}, + {FieldStatus, "status"}, +} + +// Has reports whether f contains every field in other. +func (f Fields) Has(other Fields) bool { + return f&other == other +} + +// Mask renders f as the update_mask the service expects, always in the same order. +func (f Fields) Mask() string { + names := make([]string, 0, len(wireNames)) + for _, w := range wireNames { + if f.Has(w.field) { + names = append(names, w.name) + } + } + return strings.Join(names, ",") +} diff --git a/libs/dms/fields_test.go b/libs/dms/fields_test.go new file mode 100644 index 00000000000..198a55efca8 --- /dev/null +++ b/libs/dms/fields_test.go @@ -0,0 +1,23 @@ +package dms + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFieldsMask(t *testing.T) { + // The order is fixed, so the same set always sends the same mask. + assert.Equal(t, "state,error_message,resource_id,status", DescribesResource.Mask()) + assert.Equal(t, "error_message,status", KeepsState.Mask()) + assert.Equal(t, "state", FieldState.Mask()) + assert.Empty(t, Fields(0).Mask()) +} + +func TestFieldsHas(t *testing.T) { + assert.True(t, DescribesResource.Has(FieldState)) + assert.False(t, KeepsState.Has(FieldState)) + // Has asks for every field, not any of them. + assert.True(t, DescribesResource.Has(FieldState|FieldStatus)) + assert.False(t, KeepsState.Has(FieldState|FieldStatus)) +} diff --git a/libs/dms/metadata.go b/libs/dms/metadata.go new file mode 100644 index 00000000000..d63154a4790 --- /dev/null +++ b/libs/dms/metadata.go @@ -0,0 +1,95 @@ +package dms + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// VersionType identifies the kind of deployment a version records. +type VersionType = bundledeployments.VersionType + +const ( + VersionTypeDeploy VersionType = bundledeployments.VersionTypeVersionTypeDeploy + VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy +) + +// Metadata is what a version records about the bundle, its source and where it +// landed. The service copies these onto the deployment, so they describe it as of +// its most recent version. +type Metadata struct { + // DisplayName is the bundle's name, which the deployment is listed under. + DisplayName string + // TargetName is the bundle target that was deployed. + TargetName string + // Mode is the bundle target's mode, empty when the target sets none. + Mode bundledeployments.DeploymentMode + Workspace *bundledeployments.WorkspaceInfo +} + +// deploymentFields are the deployment's own metadata, in the order a mask lists them. Git is not +// among them: the service derives the deployment's from the version that carried it. +var deploymentFields = []string{"display_name", "target_name", "deployment_mode", "workspace_info"} + +// sameWorkspaceInfo compares the paths alone. The SDK records which fields a response carried in +// ForceSendFields, so a record read back never deep-equals one built here, and comparing the +// structs whole would report every run as a change. +func sameWorkspaceInfo(want, current *bundledeployments.WorkspaceInfo) bool { + if want == nil || current == nil { + return want == nil && current == nil + } + + a, b := *want, *current + a.ForceSendFields, b.ForceSendFields = nil, nil + return reflect.DeepEqual(a, b) +} + +// deployment renders the metadata the deployment owns. +func (m Metadata) deployment() bundledeployments.Deployment { + return bundledeployments.Deployment{ + DisplayName: m.DisplayName, + TargetName: m.TargetName, + DeploymentMode: m.Mode, + WorkspaceInfo: m.Workspace, + } +} + +// StaleFields returns the mask that brings current up to m, empty when the deployment already +// says what this run would say. current is nil before the first recorded deploy. +func (m Metadata) StaleFields(current *bundledeployments.Deployment) string { + if current == nil { + return strings.Join(deploymentFields, ",") + } + + want := m.deployment() + var stale []string + if want.DisplayName != current.DisplayName { + stale = append(stale, "display_name") + } + if want.TargetName != current.TargetName { + stale = append(stale, "target_name") + } + if want.DeploymentMode != current.DeploymentMode { + stale = append(stale, "deployment_mode") + } + if !sameWorkspaceInfo(want.WorkspaceInfo, current.WorkspaceInfo) { + stale = append(stale, "workspace_info") + } + return strings.Join(stale, ",") +} + +// NextVersion is the version number this run will create, given the deployment's most recent +// one. lastVersionID is empty before the deployment has any version. +func NextVersion(lastVersionID string) (int64, error) { + if lastVersionID == "" { + return 1, nil + } + last, err := strconv.ParseInt(lastVersionID, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse last_version_id %q: %w", lastVersionID, err) + } + return last + 1, nil +} diff --git a/libs/dms/metadata_test.go b/libs/dms/metadata_test.go new file mode 100644 index 00000000000..e3446466407 --- /dev/null +++ b/libs/dms/metadata_test.go @@ -0,0 +1,25 @@ +package dms + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNextVersion(t *testing.T) { + for _, tt := range []struct { + last string + want int64 + }{ + {"", 1}, + {"4", 5}, + } { + got, err := NextVersion(tt.last) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + } + + _, err := NextVersion("not-a-number") + assert.ErrorContains(t, err, "last_version_id") +} diff --git a/libs/dms/operation.go b/libs/dms/operation.go new file mode 100644 index 00000000000..16e1c1a3588 --- /dev/null +++ b/libs/dms/operation.go @@ -0,0 +1,139 @@ +package dms + +import ( + "encoding/json" + "fmt" + "unicode/utf8" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// maxStateSize is the largest serialized state DMS accepts per operation. More than this +// and the resource cannot be recorded at all, so the deploy fails rather than leaving the +// service holding a resource with no state. +const maxStateSize = 64 * 1024 + +// maxErrorMessageSize is how much of a failure's message the service stores. +const maxErrorMessageSize = 16 * 1024 + +// StatusPending is what the service calls an operation it has recorded but not applied, which +// is where a recreate sits between deleting the old resource and creating the new one. Not in +// the SDK: the enum ships server-first while it is staged DEVELOPMENT, so the generated client +// carries only the terminal two. +const StatusPending bundledeployments.OperationStatus = "OPERATION_STATUS_PENDING" + +// OperationUpdate is one write to an operation the version staged: the fields it claims +// and their values. It is built where the outcome is known, so a malformed one fails the +// resource that produced it rather than the upload at the end of apply. +type OperationUpdate struct { + // Fields is the mask to send. It is taken literally: a field named here is written, + // one left out keeps the value it had. + Fields Fields + + // State is the serialized state after the operation, and nil for a delete. + State json.RawMessage + + ResourceID string + Status bundledeployments.OperationStatus + ErrorMessage string +} + +// NewStateUpdate describes how the resource looks now: state is the serialized envelope the +// state DB just persisted, and nil for a delete. inProgress marks a write that is half of a +// larger change - a recreate's delete - so an interrupted deploy does not report it finished. +func NewStateUpdate(resourceID string, state json.RawMessage, inProgress bool) (OperationUpdate, error) { + if len(state) > maxStateSize { + return OperationUpdate{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxStateSize) + } + + status := bundledeployments.OperationStatusOperationStatusSucceeded + if inProgress { + status = StatusPending + } + + // No state means nothing is left to describe, so the write clears it rather than recording + // it as empty: an empty value is a state the service refuses on a delete that succeeded, + // while an absent one is how the resource stops being listed at all. + if state == nil { + return OperationUpdate{ + Fields: ClearsState, + Status: status, + }, nil + } + + return OperationUpdate{ + Fields: DescribesResource, + State: state, + ResourceID: resourceID, + Status: status, + }, nil +} + +// failureMessage summarizes cause for the history, cut to what the service stores. +func failureMessage(cause error) string { + // Summarized, not cause.Error(): for an API failure that adds the status and error + // code, which is often the most actionable part of the history. + message := diag.FormatAPIErrorSummary(cause) + if len(message) > maxErrorMessageSize { + message = message[:maxErrorMessageSize] + // The cut can land inside a rune, and the service stores a string. Drop the partial + // one: at most UTFMax-1 bytes of it can be left, so a message that was already + // invalid loses those bytes rather than being stripped away entirely. + for range utf8.UTFMax - 1 { + if utf8.ValidString(message) { + break + } + message = message[:len(message)-1] + } + } + return message +} + +// failureUpdate marks an operation failed and carries the message, with no state of its own: it +// claims KeepsState so a Merge onto the resource's last write keeps that write's state and id. +// OperationBuffer.RecordFailure uses it when a write already recorded the resource's state. +func failureUpdate(cause error) OperationUpdate { + return OperationUpdate{ + Fields: KeepsState, + Status: bundledeployments.OperationStatusOperationStatusFailed, + ErrorMessage: failureMessage(cause), + } +} + +// NewFailureUpdate records that an operation did not apply, so the history says why a resource +// failed rather than leaving it pending. state is what the resource still has, which the service +// requires of a failure on a live one; nil when nothing is left to describe. +func NewFailureUpdate(resourceID string, state json.RawMessage, cause error) OperationUpdate { + u := failureUpdate(cause) + if state != nil { + // The id travels with the state, which the service requires of any mask naming one. + u.Fields = DescribesResource + u.State = state + u.ResourceID = resourceID + } + return u +} + +// Merge folds a later update into u, for a resource written twice before either upload ran. +// Each field comes from whichever update claimed it, newer winning when both did, and the +// mask is the union. What an update claims is decided where it is built, not here. +func (u OperationUpdate) Merge(newer OperationUpdate) OperationUpdate { + merged := u + merged.Fields = u.Fields | newer.Fields + + if newer.Fields.Has(FieldState) { + merged.State = newer.State + } + if newer.Fields.Has(FieldResourceID) { + merged.ResourceID = newer.ResourceID + } + if newer.Fields.Has(FieldErrorMessage) { + merged.ErrorMessage = newer.ErrorMessage + } + if newer.Fields.Has(FieldStatus) { + merged.Status = newer.Status + } + + return merged +} diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go new file mode 100644 index 00000000000..ac5cab63dd8 --- /dev/null +++ b/libs/dms/operation_buffer.go @@ -0,0 +1,197 @@ +package dms + +import ( + "context" + "encoding/json" + "fmt" + "sync" +) + +// bufferedOperations caps how far ahead of the service a deploy may get; DMS is what the next +// plan reads. +const bufferedOperations = 10 + +// stagedSequenceID is what version creation leaves on a staged operation, so a resource's first +// update sends it as the precondition. +const stagedSequenceID = "0" + +// OperationBuffer records each state write with DMS for one deployment version, off the apply +// path: writes are queued and sent on one background goroutine. It exists only while a bundle +// records deployment history; callers hold a nil buffer otherwise and must not call it. +type OperationBuffer struct { + client *Client + deploymentID string + versionNum int64 + + // The buffer. queue holds bundle state keys, and pending the newest update per key, so a + // second write for a resource replaces the first. + queue chan string + done chan struct{} + stopQueue func() + + // sequenceIDs holds the token the last update for a resource returned. A resource absent + // from it has only what staging left, so its first update sends that. Unguarded: run is the + // only goroutine that writes, one update at a time. + sequenceIDs map[string]string + + // mu guards the fields below. + mu sync.Mutex + + // pending holds the newest update per resource key, absent once the writer takes it. + pending map[string]OperationUpdate + + // latestState is the newest state-bearing update per resource key, kept for the buffer's + // whole life - unlike pending, which the writer removes on take. RecordFailure merges its + // status and message onto this so a failure keeps the state the last write recorded (which + // the service may already hold), not the pre-deploy state, which would overwrite it. + latestState map[string]OperationUpdate + + err error +} + +// StartOperationBuffer opens the buffer for the version the caller just created. The version +// must already exist: operations record under it, and nothing here creates it. +func StartOperationBuffer(ctx context.Context, client *Client, deploymentID string, versionNum int64) *OperationBuffer { + b := &OperationBuffer{ + client: client, + deploymentID: deploymentID, + versionNum: versionNum, + queue: make(chan string, bufferedOperations), + done: make(chan struct{}), + pending: make(map[string]OperationUpdate), + latestState: make(map[string]OperationUpdate), + sequenceIDs: make(map[string]string), + } + b.stopQueue = sync.OnceFunc(func() { close(b.queue) }) + go b.run(ctx) + return b +} + +// RecordOperation records a state write. state is the serialized envelope, nil for a delete. +// An earlier failure does not stop it: keep recording, best effort. +func (b *OperationBuffer) RecordOperation(ctx context.Context, resourceKey string, inProgress bool, resourceID string, state json.RawMessage) { + update, err := NewStateUpdate(resourceID, state, inProgress) + if err != nil { + b.setErr(fmt.Errorf("recording operation for %s: %w", resourceKey, err)) + return + } + + // Remember the state this write recorded, kept past the writer taking it, so a later failure + // for this resource keeps this state rather than the pre-deploy one (see RecordFailure). + b.mu.Lock() + b.latestState[resourceKey] = update + b.mu.Unlock() + + b.record(resourceKey, update) +} + +// RecordFailure records that a resource did not apply, so the history says why rather than +// leaving the resource out. When a write already recorded this resource's state, the failure +// keeps that state - which the service may already hold - and changes only status and message; +// recording the pre-deploy state instead would overwrite the newer state, and after a rename +// pair the new id with the old state. state is the fallback, used only when no write recorded +// anything: the operation failed before it saved state. +func (b *OperationBuffer) RecordFailure(resourceKey, resourceID string, state json.RawMessage, cause error) { + b.mu.Lock() + latest, recorded := b.latestState[resourceKey] + b.mu.Unlock() + + if recorded { + b.record(resourceKey, latest.Merge(failureUpdate(cause))) + return + } + + b.record(resourceKey, NewFailureUpdate(resourceID, state, cause)) +} + +// record makes update the one waiting for resourceKey, waiting itself while the queue is +// full. Recording after Drain panics, so every caller must return before Drain. +func (b *OperationBuffer) record(resourceKey string, update OperationUpdate) { + b.mu.Lock() + waiting, queued := b.pending[resourceKey] + if queued { + update = waiting.Merge(update) + } + b.pending[resourceKey] = update + b.mu.Unlock() + + // when already queued: the writer reads the map when it gets to the key, + // so it picks up what was just stored. + // + // Only when the resource is not already queued, enqueue it. + if !queued { + b.queue <- resourceKey + } +} + +// take claims the update waiting for resourceKey. +func (b *OperationBuffer) take(resourceKey string) (OperationUpdate, bool) { + b.mu.Lock() + defer b.mu.Unlock() + + update, ok := b.pending[resourceKey] + delete(b.pending, resourceKey) + return update, ok +} + +func (b *OperationBuffer) run(ctx context.Context) { + defer close(b.done) + + for resourceKey := range b.queue { + update, ok := b.take(resourceKey) + if !ok { + // Unreachable: a key is queued only when nothing was waiting for it. Guard so + // a stray key could never write a zero-valued update. + continue + } + + // Keep going after a failure, so one bad write does not drop everything behind it. + if err := b.write(ctx, resourceKey, update); err != nil { + b.setErr(fmt.Errorf("recording operation for %s: %w", resourceKey, err)) + } + } +} + +// write sends one update, at the sequence id the resource is at. +func (b *OperationBuffer) write(ctx context.Context, key string, update OperationUpdate) error { + sequenceID, written := b.sequenceIDs[key] + if !written { + sequenceID = stagedSequenceID + } + + next, err := b.client.UpdateOperation(ctx, b.deploymentID, b.versionNum, key, sequenceID, update) + if err != nil { + return err + } + + // The next write for this resource echoes the sequence id this one earned. + b.sequenceIDs[key] = next + + return nil +} + +// Drain waits for everything buffered to reach the service and returns the first write error, +// which fails the deploy: DMS is the source of truth for what exists. Safe to call twice. +func (b *OperationBuffer) Drain() error { + b.stopQueue() + <-b.done + return b.Err() +} + +// setErr keeps the first error; one failure is enough to fail the deploy. +func (b *OperationBuffer) setErr(err error) { + b.mu.Lock() + defer b.mu.Unlock() + + if b.err == nil { + b.err = err + } +} + +// Err returns the first recording error, or nil. +func (b *OperationBuffer) Err() error { + b.mu.Lock() + defer b.mu.Unlock() + + return b.err +} diff --git a/libs/dms/operation_buffer_test.go b/libs/dms/operation_buffer_test.go new file mode 100644 index 00000000000..b4fba8ce124 --- /dev/null +++ b/libs/dms/operation_buffer_test.go @@ -0,0 +1,116 @@ +package dms + +import ( + "encoding/json" + "errors" + "strings" + "sync" + "testing" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// What the service ends up holding - the version a deploy claims, the operations it stages, the +// completion it reports - is asserted end to end by acceptance/bundle/dms. What is left here is +// what a deploy cannot reach: the queue's coalescing and the size limit. No test here goes +// through the API - the queue is driven directly, so nothing waits on a request. + +// queued builds a buffer whose queue nothing drains, so a test drives record and take itself. +func queued() *OperationBuffer { + return &OperationBuffer{ + queue: make(chan string, bufferedOperations), + pending: make(map[string]OperationUpdate), + latestState: make(map[string]OperationUpdate), + } +} + +func stateUpdate(t *testing.T, name string) OperationUpdate { + t.Helper() + update, err := NewStateUpdate("id-1", json.RawMessage(`{"state":{"name":"`+name+`"}}`), false) + require.NoError(t, err) + return update +} + +func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { + s := queued() + + // Two writes for one resource with nothing draining. They carry the resource's full + // state, so only the newest needs to go: one slot in the queue, not two. + s.record("resources.jobs.foo", stateUpdate(t, "v1")) + s.record("resources.jobs.foo", stateUpdate(t, "v2")) + + assert.Len(t, s.queue, 1) + update, ok := s.take("resources.jobs.foo") + require.True(t, ok) + assert.JSONEq(t, `{"state":{"name":"v2"}}`, string(update.State)) + + // Taken means a request has it, and an in-flight request cannot be recalled, so the next + // write gets its own slot rather than joining it. + s.record("resources.jobs.foo", stateUpdate(t, "v3")) + assert.Len(t, s.queue, 2) + update, ok = s.take("resources.jobs.foo") + require.True(t, ok) + assert.JSONEq(t, `{"state":{"name":"v3"}}`, string(update.State)) +} + +func TestBufferFailsOnOversizedState(t *testing.T) { + // The service will not take a state this large, so the resource cannot be recorded. + // Failing here says so, where reporting nothing would leave DMS without the resource + // and the next plan would create it again. + s := queued() + + s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", json.RawMessage(strings.Repeat("x", maxStateSize+1))) + + assert.ErrorContains(t, s.Err(), "exceeds the 65536 byte limit") + assert.Empty(t, s.queue) + assert.Empty(t, s.pending) +} + +func TestDrainIsIdempotent(t *testing.T) { + // Nothing recorded, so no transport is needed: this is the second drain, which must not + // panic on an already closed queue. + s := queued() + s.done = make(chan struct{}) + s.stopQueue = sync.OnceFunc(func() { close(s.queue) }) + go s.run(t.Context()) + + require.NoError(t, s.Drain()) + require.NoError(t, s.Drain()) +} + +func TestRecordFailureKeepsTheStateTheLastWriteRecorded(t *testing.T) { + s := queued() + + // A write records the resource's post-apply state - a rename, so a new id and new state - and + // the writer sends it (take), leaving nothing pending. + s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-new", json.RawMessage(`{"state":{"name":"v2"}}`)) + _, ok := s.take("resources.jobs.foo") + require.True(t, ok) + require.Empty(t, s.pending) + + // The post-update wait then fails. The failure carries the pre-deploy state (old id, old + // state), which it must not use: the write already recorded v2, which the service may hold. + s.RecordFailure("resources.jobs.foo", "id-old", json.RawMessage(`{"state":{"name":"v1"}}`), errors.New("wait timed out")) + + update, ok := s.take("resources.jobs.foo") + require.True(t, ok) + assert.JSONEq(t, `{"state":{"name":"v2"}}`, string(update.State), "keeps the state the write recorded") + assert.Equal(t, "id-new", update.ResourceID, "keeps the id the write recorded, not the pre-deploy id") + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, update.Status) + assert.Contains(t, update.ErrorMessage, "wait timed out") +} + +func TestRecordFailureUsesTheGivenStateWhenNoWriteRecorded(t *testing.T) { + // No write recorded anything - the operation failed before it saved state, e.g. a create that + // never landed - so the failure falls back to the state passed in, what the resource still has. + s := queued() + + s.RecordFailure("resources.jobs.foo", "id-1", json.RawMessage(`{"state":{"name":"v1"}}`), errors.New("create failed")) + + update, ok := s.take("resources.jobs.foo") + require.True(t, ok) + assert.JSONEq(t, `{"state":{"name":"v1"}}`, string(update.State)) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, update.Status) +} diff --git a/libs/dms/operation_test.go b/libs/dms/operation_test.go new file mode 100644 index 00000000000..2bae9a81144 --- /dev/null +++ b/libs/dms/operation_test.go @@ -0,0 +1,83 @@ +package dms + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "unicode/utf8" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// What these updates put on the wire - the mask, the status, the state a write carries and a +// failure leaves alone - is asserted by acceptance/bundle/dms. What is left here are the limits +// and the merge, which a deploy cannot reach. + +func TestNewFailureUpdateTruncatesLongError(t *testing.T) { + // Truncated rather than rejected: a message over the limit would make recording + // fail and hide the error it is reporting. + update := NewFailureUpdate("job-123", nil, errors.New(strings.Repeat("x", maxErrorMessageSize+100))) + + assert.Len(t, update.ErrorMessage, maxErrorMessageSize) +} + +func TestNewFailureUpdatePreservesUTF8OnTruncation(t *testing.T) { + // The cut lands one byte into the emoji, so a byte-wise truncation would leave a partial + // rune behind and the service stores state and messages as strings. + msg := strings.Repeat("a", maxErrorMessageSize-1) + "❌" + "x" + + update := NewFailureUpdate("job-123", nil, errors.New(msg)) + + assert.True(t, utf8.ValidString(update.ErrorMessage)) + // The whole emoji went, so the message is shorter than the limit rather than exactly it. + assert.Equal(t, strings.Repeat("a", maxErrorMessageSize-1), update.ErrorMessage) +} + +func TestMergeLetsAWriteSupersedeAFailure(t *testing.T) { + // A failure is not the last word. A retry that writes state wins whole - state, id and + // mask - and the mask names error_message so the recorded failure is cleared. The service + // rejects a succeeded operation that still carries an error. + failed := NewFailureUpdate("id-old", nil, errors.New("boom")) + retried, err := NewStateUpdate("id-new", json.RawMessage(`{"state":{"name":"after"}}`), false) + require.NoError(t, err) + + merged := failed.Merge(retried) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, merged.Status) + assert.Empty(t, merged.ErrorMessage) + assert.Equal(t, "id-new", merged.ResourceID) + assert.JSONEq(t, `{"state":{"name":"after"}}`, string(merged.State)) + assert.Equal(t, DescribesResource, merged.Fields) +} + +func TestMergeKeepsTheWritesStateAndMask(t *testing.T) { + // A failure claims only status and error_message, so the write's state, id and mask + // survive: the resource stays listed as it was written, now marked failed. + write, err := NewStateUpdate("id-new", json.RawMessage(`{"state":{"name":"before"}}`), false) + require.NoError(t, err) + failed := NewFailureUpdate("id-old", nil, errors.New("boom")) + + merged := write.Merge(failed) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, merged.Status) + assert.Equal(t, "boom", merged.ErrorMessage) + assert.Equal(t, "id-new", merged.ResourceID) + assert.JSONEq(t, `{"state":{"name":"before"}}`, string(merged.State)) + assert.Equal(t, DescribesResource, merged.Fields) +} + +func TestMergeLetsADeleteClearTheState(t *testing.T) { + // A delete legitimately carries no state, and merging must let it through: the resource + // is gone, and keeping the state it replaces would leave it listed. + write, err := NewStateUpdate("id-1", json.RawMessage(`{"state":{"name":"before"}}`), false) + require.NoError(t, err) + deleted, err := NewStateUpdate("id-1", nil, false) + require.NoError(t, err) + + merged := write.Merge(deleted) + + assert.Nil(t, merged.State) +} diff --git a/libs/dms/resources.go b/libs/dms/resources.go new file mode 100644 index 00000000000..565571415f6 --- /dev/null +++ b/libs/dms/resources.go @@ -0,0 +1,41 @@ +package dms + +import ( + "context" + "fmt" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// Resource is what DMS holds for one resource of a deployment, as of the last operation +// that recorded it. +type Resource struct { + // Key is the bundle state key, as everything outside this package spells it. + Key string + ID string + + // State is the state the last operation recorded, as the opaque string the service + // stores, and empty when no operation recorded one. + State string +} + +// ListResources returns every resource DMS holds for the deployment. +func (c *Client) ListResources(ctx context.Context, deploymentID string) ([]Resource, error) { + it := c.Service.ListResources(ctx, bundledeployments.ListResourcesRequest{ + Parent: DeploymentName(deploymentID), + }) + + var out []Resource + for it.HasNext(ctx) { + res, err := it.Next(ctx) + if err != nil { + return nil, fmt.Errorf("listing resources from deployment metadata service: %w", err) + } + out = append(out, Resource{ + Key: statePrefix + res.ResourceKey, + ID: res.ResourceId, + State: res.State, + }) + } + return out, nil +} diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go new file mode 100644 index 00000000000..3545036297a --- /dev/null +++ b/libs/testserver/bundledeployments.go @@ -0,0 +1,615 @@ +package testserver + +import ( + "bytes" + "encoding/json" + "fmt" + "path" + "slices" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. + +// maxOperationsPerVersion mirrors the service's compiled-in default. A bundle past it cannot +// be recorded, since the operation set is fixed when the version is created. +const maxOperationsPerVersion = 800 + +// operationStatusPending is what CreateVersion leaves on a staged operation. Declared here +// because the SDK enum is generated from the OpenAPI spec, which trails the service proto. +const operationStatusPending bundledeployments.OperationStatus = "OPERATION_STATUS_PENDING" + +// State is kept in FakeWorkspace.DmsDeployments, keyed by deployment ID. + +// dmsDeploymentNodeName is the workspace node name the service uses for deployments. +// It must match DEPLOYMENT_NODE_NAME on the service side (DeploymentWhsClient). +const dmsDeploymentNodeName = "resources.deployment.json" + +// dmsUpdatableOperationFields are the update_mask paths UpdateOperation accepts. Any +// other path is rejected, and action_type in particular is fixed when the operation is +// created. +var dmsUpdatableOperationFields = []string{"state", "error_message", "resource_id", "status"} + +// DmsDeployment holds a deployment record together with the versions and +// resources recorded under it, so the read APIs (ListVersions/ListResources) +// can serve back what deploys wrote. +type DmsDeployment struct { + Deployment bundledeployments.Deployment + Versions map[string]*bundledeployments.Version + // resources is the latest resource state per resource key, updated as + // operations are recorded. + Resources map[string]bundledeployments.Resource + // operations holds the recorded operations by resource name. The service keeps + // one per resource per version, so a resource written twice in a version updates + // its operation rather than adding another. + Operations map[string]*bundledeployments.Operation + // lastSuccessfulVersionID is the highest version completed successfully. + // The read path treats a non-empty value as "DMS owns the state"; + // the SDK Deployment struct does not yet carry this field. + LastSuccessfulVersionID string +} + +func (s *FakeWorkspace) CreateDeployment(req Request) Response { + var dep bundledeployments.Deployment + if err := json.Unmarshal(req.Body, &dep); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + if dep.InitialParentPath == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": "initial_parent_path is required"}, + } + } + + defer s.LockUnlock()() + + // The service registers the deployment as a workspace node under + // initial_parent_path and uses that node's ID as the deployment ID, so a + // get-status on the node path is how clients look the deployment back up. + nodePath := path.Join(dep.InitialParentPath, dmsDeploymentNodeName) + if resp, ok := s.requireParentDirectory(nodePath); !ok { + return resp + } + objectID := nextID() + s.files[nodePath] = FileEntry{ + Info: workspace.ObjectInfo{ + // The service registers its own node type, which get-status reports back; + // a plain FILE is what the fake used to claim and the real API never returns. + ObjectType: "BUNDLE_DEPLOYMENT", + Path: nodePath, + ObjectId: objectID, + }, + } + + // The record carries no version yet; last_version_id stays empty until + // the first CreateVersion. A failed registration leaves a record with no versions. + deploymentID := strconv.FormatInt(objectID, 10) + s.DmsDeploymentNodes[deploymentID] = nodePath + + if resp, ok := checkWorkspaceInfo(dep.WorkspaceInfo); !ok { + return resp + } + + dep.Name = "deployments/" + deploymentID + dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive + s.DmsDeployments[deploymentID] = &DmsDeployment{ + Deployment: dep, + Versions: map[string]*bundledeployments.Version{}, + Resources: map[string]bundledeployments.Resource{}, + Operations: map[string]*bundledeployments.Operation{}, + } + return Response{Body: dep} +} + +func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + body, err := deploymentBody(d) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + return Response{Body: body} +} + +// deploymentBody renders a deployment with last_successful_version_id, which +// the SDK struct doesn't yet carry. Embedding in a wrapper won't work because +// Deployment.MarshalJSON silently drops sibling fields. +func deploymentBody(d *DmsDeployment) (map[string]any, error) { + raw, err := json.Marshal(d.Deployment) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + if d.LastSuccessfulVersionID != "" { + body["last_successful_version_id"] = d.LastSuccessfulVersionID + } + return body, nil +} + +// dmsUpdatableDeploymentFields are the update_mask paths UpdateDeployment accepts. +var dmsUpdatableDeploymentFields = []string{"display_name", "target_name", "deployment_mode", "workspace_info"} + +// checkWorkspaceInfo rejects a bundle_root_path without the git_folder_path it is relative to, +// which is what the service does. +func checkWorkspaceInfo(ws *bundledeployments.WorkspaceInfo) (Response, bool) { + if ws != nil && (ws.GitFolderPath == "") != (ws.BundleRootPath == "") { + return dmsInvalidArgument("workspace_info.git_folder_path and workspace_info.bundle_root_path must be set together"), false + } + return Response{}, true +} + +func (s *FakeWorkspace) UpdateDeployment(req Request, deploymentID string) Response { + var raw map[string]json.RawMessage + if err := json.Unmarshal(req.Body, &raw); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + var dep bundledeployments.Deployment + if err := json.Unmarshal(req.Body, &dep); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + updateMask := req.URL.Query().Get("update_mask") + if updateMask == "" { + return dmsInvalidArgument("update_mask is required") + } + update := map[string]bool{} + for path := range strings.SplitSeq(updateMask, ",") { + path = strings.TrimSpace(path) + if !slices.Contains(dmsUpdatableDeploymentFields, path) { + return dmsInvalidArgument("update_mask path " + path + " is not updatable") + } + if _, sent := raw[path]; !sent { + // An empty value is how a field is cleared, so the field has to be there to say so. + return dmsInvalidArgument(path + " is required when '" + path + "' is in update_mask (an empty value clears it)") + } + update[path] = true + } + + if resp, ok := checkWorkspaceInfo(dep.WorkspaceInfo); !ok { + return resp + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Only the masked paths are written; every other field of the body is ignored. + if update["display_name"] { + d.Deployment.DisplayName = dep.DisplayName + } + if update["target_name"] { + d.Deployment.TargetName = dep.TargetName + } + if update["deployment_mode"] { + d.Deployment.DeploymentMode = dep.DeploymentMode + } + if update["workspace_info"] { + d.Deployment.WorkspaceInfo = dep.WorkspaceInfo + } + + body, err := deploymentBody(d) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + return Response{Body: body} +} + +func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + // The service trashes the deployment's workspace node, so a later get-status + // on the node path reports the deployment as absent. + if nodePath, ok := s.DmsDeploymentNodes[deploymentID]; ok { + delete(s.files, nodePath) + } + delete(s.DmsDeploymentNodes, deploymentID) + delete(s.DmsDeployments, deploymentID) + return Response{Body: map[string]any{}} +} + +func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response { + versionID := req.URL.Query().Get("version_id") + + var version bundledeployments.Version + if err := json.Unmarshal(req.Body, &version); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + // operations rides in the version body like every other field, but is absent from the + // generated struct while the field is at DEVELOPMENT stage, so it is read separately. + // Embedding Version instead would promote its own UnmarshalJSON and silently drop this + // field. It is input only, so it is read here and never returned. + var staged struct { + Operations []struct { + ResourceKey string `json:"resource_key"` + ActionType bundledeployments.OperationActionType `json:"action_type"` + } `json:"operations"` + } + if err := json.Unmarshal(req.Body, &staged); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // version_id must be numerically greater than the most recent version, + // and previous_version_id must name that version to detect concurrent deploys. + next, err := strconv.ParseInt(versionID, 10, 64) + if err != nil || next < 1 { + return dmsInvalidArgument("version_id must be a positive integer, got " + versionID) + } + var last int64 + if d.Deployment.LastVersionId != "" { + last, _ = strconv.ParseInt(d.Deployment.LastVersionId, 10, 64) + } + if next <= last { + return dmsInvalidArgument("version_id " + versionID + " must be greater than the most recent version " + d.Deployment.LastVersionId) + } + if version.PreviousVersionId != d.Deployment.LastVersionId { + return dmsAborted("previous_version_id is outdated; the deployment's most recent version is " + d.Deployment.LastVersionId) + } + + // Note: deployment lock not modelled. Tests kill the CLI mid-apply, leaving + // the version in-progress forever, whereas the real service lets the lease expire. + + // A version records its whole operation set up front; there is no API to add one later. + if len(staged.Operations) > maxOperationsPerVersion { + return Response{ + StatusCode: 429, + Body: map[string]string{ + "error_code": "RESOURCE_EXHAUSTED", + "message": fmt.Sprintf("a version may stage at most %d operations, got %d", maxOperationsPerVersion, len(staged.Operations)), + }, + } + } + seen := make(map[string]bool, len(staged.Operations)) + for _, staged := range staged.Operations { + switch { + case staged.ResourceKey == "": + return dmsInvalidArgument("operations.resource_key is required") + case !strings.Contains(staged.ResourceKey, "."): + return dmsInvalidArgument("operations.resource_key must have a known resource type prefix (e.g. 'jobs.', 'pipelines.'): " + staged.ResourceKey) + case staged.ActionType == "": + return dmsInvalidArgument("operations.action_type is required and must not be UNSPECIFIED for resource " + staged.ResourceKey) + case seen[staged.ResourceKey]: + return dmsInvalidArgument("operations must have distinct resource_keys; duplicate: " + staged.ResourceKey) + } + seen[staged.ResourceKey] = true + } + + d.Deployment.LastVersionId = versionID + version.Name = "deployments/" + deploymentID + "/versions/" + versionID + version.VersionId = versionID + version.Status = bundledeployments.VersionStatusVersionStatusInProgress + d.Versions[versionID] = &version + + // The deployment's git provenance is derived from the version that carried it; the rest of + // its metadata is written through CreateDeployment and UpdateDeployment. + d.Deployment.GitInfo = version.GitInfo + + // Each staged operation starts pending at sequence 0, and the CLI fills in its outcome + // with UpdateOperation as the resource is applied. + for _, staged := range staged.Operations { + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + staged.ResourceKey + d.Operations[opName] = &bundledeployments.Operation{ + Name: opName, + ResourceKey: staged.ResourceKey, + ActionType: staged.ActionType, + Status: operationStatusPending, + SequenceId: 0, + } + } + + return Response{Body: version} +} + +func (s *FakeWorkspace) CompleteVersion(req Request, deploymentID, versionID string) Response { + var completeReq bundledeployments.CompleteVersionRequest + if err := json.Unmarshal(req.Body, &completeReq); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + v, ok := d.Versions[versionID] + if !ok { + return dmsNotFound("version " + versionID) + } + + v.Status = bundledeployments.VersionStatusVersionStatusCompleted + v.CompletionReason = completeReq.CompletionReason + if completeReq.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { + d.LastSuccessfulVersionID = versionID + } + return Response{Body: *v} +} + +func (s *FakeWorkspace) Heartbeat() Response { + return Response{Body: bundledeployments.HeartbeatResponse{}} +} + +// operationBody renders an operation the way the service does: sequence_id as a +// JSON string, which the SDK struct cannot express (it types the field int64). +func operationBody(op *bundledeployments.Operation) (map[string]any, error) { + raw, err := json.Marshal(op) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + body["sequence_id"] = strconv.FormatInt(op.SequenceId, 10) + return body, nil +} + +// UpdateOperation applies a later write for a resource already recorded in this +// version. sequence_id is the concurrency precondition and increments on success. +func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, resourceKey string) Response { + // sequence_id arrives as a string, which the SDK struct cannot hold (it types the + // field int64), so read the body twice: once for the typed fields with that key + // removed, and once for the precondition alone. + var raw map[string]json.RawMessage + if err := json.Unmarshal(req.Body, &raw); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + var precondition struct { + SequenceId string `json:"sequence_id"` + } + if err := json.Unmarshal(req.Body, &precondition); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + delete(raw, "sequence_id") + typedBody, err := json.Marshal(raw) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + var op bundledeployments.Operation + if err := json.Unmarshal(typedBody, &op); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + updateMask := req.URL.Query().Get("update_mask") + if updateMask == "" { + return dmsInvalidArgument("update_mask is required") + } + // Only masked paths are written; other fields keep their values. + // Omitting state keeps the already-recorded state. + update := map[string]bool{} + for path := range strings.SplitSeq(updateMask, ",") { + path = strings.TrimSpace(path) + if !slices.Contains(dmsUpdatableOperationFields, path) { + return dmsInvalidArgument("update_mask path " + path + " is not updatable") + } + // state is the exception: masked with no value is how the service is told the resource + // is gone, which is what stops it being listed. + if _, sent := raw[path]; !sent && path != "state" { + // An empty value is how a field is cleared, so the field has to be there to say so. + return dmsInvalidArgument(path + " is required when '" + path + "' is in update_mask (an empty value clears it)") + } + update[path] = true + } + + // The service parses the status as a proto enum, so a value outside the enum arrives as + // UNSPECIFIED and is refused. Checked here too: a status the API does not have otherwise + // passes every local test and fails only against a real workspace. + if update["status"] { + switch op.Status { + case bundledeployments.OperationStatusOperationStatusSucceeded, + bundledeployments.OperationStatusOperationStatusFailed, + operationStatusPending: + default: + return dmsInvalidArgument("status is required and must not be UNSPECIFIED when 'status' is in update_mask") + } + } + + // The service counts only a state it was given as recording one, so these ask whether a + // value arrived rather than whether the mask names the field. + _, stateSent := raw["state"] + recordsState := update["state"] && stateSent + + // A resource_id can be written but not cleared, and only alongside the state it belongs to. + if update["resource_id"] { + if op.ResourceId == "" { + return dmsInvalidArgument("resource_id is required when 'resource_id' is in update_mask") + } + if !recordsState { + return dmsInvalidArgument("state must be in update_mask when 'resource_id' is") + } + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + existing, ok := d.Operations[opName] + if !ok { + return dmsNotFound("operation " + opName) + } + if precondition.SequenceId != strconv.FormatInt(existing.SequenceId, 10) { + return dmsAborted("sequence_id is outdated; the operation is at " + strconv.FormatInt(existing.SequenceId, 10)) + } + + // Invariants check the operation after the update, not the request. + // The mask leaves unspecified fields unchanged. + after := *existing + if update["state"] { + after.State = op.State + } + if update["error_message"] { + after.ErrorMessage = op.ErrorMessage + } + if update["resource_id"] { + after.ResourceId = op.ResourceId + } + if update["status"] { + after.Status = op.Status + } + + failed := after.Status == bundledeployments.OperationStatusOperationStatusFailed + if !failed && after.ErrorMessage != "" { + return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") + } + + // A failure may leave no state only where nothing is left to describe: a create that never + // landed, or a recreate whose delete already went through. + if failed && after.State == "" && leavesLiveResource(after.ActionType) { + return dmsInvalidArgument("state is required for a " + string(after.ActionType) + + " operation, because it acts on a resource that already exists and cannot destroy it, even when it fails") + } + + // An operation with state must identify its resource via resource_id, + // even for failed operations reporting prior state. + if after.State != "" && after.ResourceId == "" { + return dmsInvalidArgument("resource_id is required for an operation that records state") + } + + // Only the masked fields change; action_type and resource_key stay as created. + if update["state"] { + existing.State = op.State + } + if update["error_message"] { + existing.ErrorMessage = op.ErrorMessage + } + if update["resource_id"] { + existing.ResourceId = op.ResourceId + } + if update["status"] { + existing.Status = op.Status + } + existing.SequenceId++ + + body, err := operationBody(existing) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + + // Only an update that names state moves the resource: naming it with no value clears + // it and removes the resource, and an update that leaves it out - a failure reporting + // its outcome - must not disturb what the deployment already holds. Every version + // stages its operations without state, so re-deriving this from the operation + // regardless of the mask would drop a resource whose deploy failed before writing. + if update["state"] { + if existing.State == "" { + delete(d.Resources, resourceKey) + } else { + d.Resources[resourceKey] = bundledeployments.Resource{ + Name: "deployments/" + deploymentID + "/resources/" + resourceKey, + ResourceKey: resourceKey, + ResourceId: existing.ResourceId, + ResourceType: existing.ResourceType, + LastActionType: existing.ActionType, + LastVersionId: versionID, + State: existing.State, + } + } + } else if resource, projected := d.Resources[resourceKey]; projected && update["resource_id"] { + // resource_id is mirrored too, for a resource the deployment still holds. + resource.ResourceId = existing.ResourceId + d.Resources[resourceKey] = resource + } + + return Response{Body: body} +} + +func (s *FakeWorkspace) ListResources(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Sort by resource key so the response order is deterministic. + keys := make([]string, 0, len(d.Resources)) + for key := range d.Resources { + keys = append(keys, key) + } + slices.Sort(keys) + + resources := make([]bundledeployments.Resource, 0, len(keys)) + for _, key := range keys { + resources = append(resources, d.Resources[key]) + } + return Response{Body: bundledeployments.ListResourcesResponse{Resources: resources}} +} + +// leavesLiveResource reports whether a failed operation of this type leaves a resource behind +// that it still has to describe. A create leaves nothing, and a recreate already deleted. +func leavesLiveResource(action bundledeployments.OperationActionType) bool { + switch action { + case bundledeployments.OperationActionTypeOperationActionTypeCreate, + bundledeployments.OperationActionTypeOperationActionTypeRecreate: + return false + default: + return true + } +} + +// dmsNotFound returns the RESOURCE_DOES_NOT_EXIST error shape the DMS API uses, +// which the SDK maps to apierr.ErrNotFound. +func dmsNotFound(what string) Response { + return Response{ + StatusCode: 404, + // Content-Type is required for the SDK to parse the body into a typed error, + // which is what callers match against apierr.ErrResourceDoesNotExist. + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": what + " does not exist", + }, + } +} + +// dmsAborted returns the 409 ABORTED error the server uses for the version +// optimistic-concurrency check. +func dmsInvalidArgument(message string) Response { + return Response{ + StatusCode: 400, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } +} + +func dmsAborted(message string) Response { + return Response{ + StatusCode: 409, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "ABORTED", "message": message}, + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 2f077b1b631..b544c17538e 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -248,6 +248,15 @@ type FakeWorkspace struct { // clusterVenvs caches Python venvs per existing cluster ID, // matching cloud behavior where libraries are cached on running clusters. clusterVenvs map[string]*clusterEnv + + // DmsDeployments holds Deployment Metadata Service (DMS) records, keyed by + // deployment ID. Each record carries its versions and latest resource state. + DmsDeployments map[string]*DmsDeployment + + // DmsDeploymentNodes maps deployment ID to the workspace node CreateDeployment made for + // it. An ID appears here before DmsDeployments has a record, which its first version + // creates, so the node is what makes the ID valid in between. + DmsDeploymentNodes map[string]string } func (s *FakeWorkspace) LockUnlock() func() { @@ -496,6 +505,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { postgresImplicitBranches: map[string]bool{}, postgresImplicitEndpoints: map[string]bool{}, clusterVenvs: map[string]*clusterEnv{}, + DmsDeployments: map[string]*DmsDeployment{}, + DmsDeploymentNodes: map[string]string{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{}, ModelRegistryModels: map[string]ml.Model{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8c44d97eae7..33a742cc2ce 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -279,6 +279,35 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.JobsCreate(req) }) + // Deployment Metadata Service (DMS) endpoints. + server.Handle("POST", "/api/2.0/bundle/deployments", func(req Request) any { + return req.Workspace.CreateDeployment(req) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.GetDeployment(req.Vars["deployment_id"]) + }) + server.Handle("PATCH", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.UpdateDeployment(req, req.Vars["deployment_id"]) + }) + server.Handle("DELETE", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.DeleteDeployment(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/complete", func(req Request) any { + return req.Workspace.CompleteVersion(req, req.Vars["deployment_id"], req.Vars["version_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/heartbeat", func(req Request) any { + return req.Workspace.Heartbeat() + }) + server.Handle("PATCH", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations/{resource_key}", func(req Request) any { + return req.Workspace.UpdateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"], req.Vars["resource_key"]) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/resources", func(req Request) any { + return req.Workspace.ListResources(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.2/jobs/delete", func(req Request) any { var request jobs.DeleteJob if err := json.Unmarshal(req.Body, &request); err != nil { diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index 61c3f271468..0b6a80fd71b 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "strings" ) @@ -71,6 +72,22 @@ func ResourceTypes() []string { return names } +// DeploymentURL returns the workspace URL for a bundle deployment: +// /deployments/?version=. Version pins the page to the deploy that produced it. +func DeploymentURL(baseURL url.URL, deploymentID string, version int64) string { + if deploymentID == "" { + return "" + } + + baseURL.Path = "deployments/" + deploymentID + if version > 0 { + values := baseURL.Query() + values.Set("version", strconv.FormatInt(version, 10)) + baseURL.RawQuery = values.Encode() + } + return baseURL.String() +} + // JobRunPath returns the modern workspace path for a job run, of the form // // jobs//runs/ diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index a398167e027..b92df037c84 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -141,6 +141,55 @@ func TestResourceURL(t *testing.T) { } } +func TestDeploymentURL(t *testing.T) { + tests := []struct { + name string + deploymentID string + version int64 + base url.URL + expected string + }{ + { + name: "id and version", + deploymentID: "996980114684409", + version: 2, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409?version=2", + }, + { + // The version is only known once CreateVersion has run, so link to the + // deployment itself rather than emitting version=0. + name: "zero version omits the query", + deploymentID: "996980114684409", + version: 0, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409", + }, + { + // A base URL carrying ?w= keeps it, so a vanity or legacy + // host still addresses the right workspace. + name: "preserves an existing query", + deploymentID: "42", + version: 7, + base: url.URL{Scheme: "https", Host: "host.com", RawQuery: "w=123"}, + expected: "https://host.com/deployments/42?version=7&w=123", + }, + { + name: "empty id returns empty", + deploymentID: "", + version: 1, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, DeploymentURL(tt.base, tt.deploymentID, tt.version)) + }) + } +} + func TestHasWorkspaceIDInHostname(t *testing.T) { tests := []struct { name string From a9ce3a277f55e3c96729cb2a2141acf4f1d4c828 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 02:02:34 +0000 Subject: [PATCH 002/164] acceptance/dms: list-resources shows empty tombstone for no-resources Co-authored-by: Isaac --- acceptance/bundle/dms/no-resources/output.txt | 6 ++++++ acceptance/bundle/dms/no-resources/script | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index a4f993a04e5..5c80f26a912 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -78,3 +78,9 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "completion_reason": "VERSION_COMPLETE_SUCCESS" } } + +=== The service lists no resources for the deployment: nothing was recorded beyond the deployment itself +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + +] diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 13924f3664f..81444e00ccf 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -8,3 +8,9 @@ trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_U title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get + +title "The service lists no resources for the deployment: nothing was recorded beyond the deployment itself" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json +# Discard the get-status + list-resources reads above so out.requests.txt is not a golden. +rm -f out.requests.txt From 4abbea01168c1a6a56a5696c26374b02605646e9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 02:12:56 +0000 Subject: [PATCH 003/164] acceptance/dms: assert declined deploy leaves recorded resources unchanged Co-authored-by: Isaac --- acceptance/bundle/dms/declined-deploy/output.txt | 10 ++++++++++ acceptance/bundle/dms/declined-deploy/script | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index d09e8d18743..faab3106201 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -76,6 +76,16 @@ Files: 4 uploaded, 0 deleted === Nothing was recorded for the declined deploy - no version, so none to abort >>> print_requests.py --dms //api/2.0/bundle +=== The recorded resources are unchanged by the decline: the schema is still at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.foo diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index e67ff786b19..c8aa0a40301 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -18,5 +18,9 @@ title "Nothing was recorded for the declined deploy - no version, so none to abo # history has no entry that reads like a deploy which failed or did nothing. trace print_requests.py --dms //api/2.0/bundle +title "The recorded resources are unchanged by the decline: the schema is still at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-declined-deploy-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + trace $CLI bundle destroy --auto-approve rm -f out.requests.txt From 2a56b8be41f15f66fc3cd3ec8740776457553aec Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 02:16:39 +0000 Subject: [PATCH 004/164] acceptance/dms: list-resources coverage for depends-on and emptied-resource Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 15 +++++++++++++++ acceptance/bundle/dms/depends-on/script | 4 ++++ acceptance/bundle/dms/emptied-resource/output.txt | 10 ++++++++++ acceptance/bundle/dms/emptied-resource/script | 4 ++++ 4 files changed, 33 insertions(+) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 4807497d63c..b36e5cfb154 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -37,6 +37,21 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } +=== The service lists both jobs, each recorded at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.child" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.parent" + } +] + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.child diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 7f5576ce72b..dd168459b9a 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -4,5 +4,9 @@ title "Deploy a job that references another: depends_on is recorded alongside th trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations +title "The service lists both jobs, each recorded at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-depends-on-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 483b3d8249d..7d0ad36a596 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -130,6 +130,16 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +=== The grants node emptied out, so the service no longer lists it - only the schema remains +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.foo diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index b38be294ce1..1e0606a31c0 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -16,5 +16,9 @@ trace print_requests.py --dms //api/2.0/bundle title "Plan again: reading state back from the service works and reports no work" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" +title "The grants node emptied out, so the service no longer lists it - only the schema remains" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-emptied-resource-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + trace $CLI bundle destroy --auto-approve rm -f out.requests.txt From af3ffde6270668d8d73e0340a90a08e7126c0f9a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 02:21:37 +0000 Subject: [PATCH 005/164] acceptance/dms: inline bundle summary into no-drift + declined-deploy, drop dedicated summary test Co-authored-by: Isaac --- .../bundle/dms/declined-deploy/output.txt | 4 ++ acceptance/bundle/dms/declined-deploy/script | 2 + acceptance/bundle/dms/no-drift/output.txt | 12 ++++++ acceptance/bundle/dms/no-drift/script | 2 + .../bundle/dms/summary/databricks.yml.tmpl | 9 ----- acceptance/bundle/dms/summary/out.test.toml | 3 -- acceptance/bundle/dms/summary/output.txt | 37 ------------------- acceptance/bundle/dms/summary/script | 17 --------- 8 files changed, 20 insertions(+), 66 deletions(-) delete mode 100644 acceptance/bundle/dms/summary/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/summary/out.test.toml delete mode 100644 acceptance/bundle/dms/summary/output.txt delete mode 100644 acceptance/bundle/dms/summary/script diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index faab3106201..ef8a2cb048b 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -96,3 +96,7 @@ This action will result in the deletion of the following UC schemas. Any underly All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default Destroy: 1 deleted + +=== After the destroy the deployment is gone, so the summary reports no history rather than a dangling id +>>> [CLI] bundle summary -o json +false diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index c8aa0a40301..df539b83b29 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -23,4 +23,6 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle destroy --auto-approve +title "After the destroy the deployment is gone, so the summary reports no history rather than a dangling id" +trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' rm -f out.requests.txt diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 7b23792edcd..952b5b2731e 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -21,6 +21,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } ] +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "1" +} + === Plan without touching anything: the deployment stamp must not show as drift >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -121,6 +127,12 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } ] +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "2" +} + === Plan again: no residual drift, and the version stamps are ignored >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 8a18d13f258..dfcb0b55ef5 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -5,6 +5,7 @@ trace $CLI bundle deploy deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-drift-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +trace $CLI bundle summary -o json | jq .bundle.deployment.history # Only deploy stamps deployment.deployment_id. Plan sees it in state but not in config, # so it must ignore it rather than report spurious changes. @@ -17,6 +18,7 @@ trace update_file.py databricks.yml 'serverless: true' 'serverless: true trace $CLI bundle deploy trace print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +trace $CLI bundle summary -o json | jq .bundle.deployment.history title "Plan again: no residual drift, and the version stamps are ignored" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" diff --git a/acceptance/bundle/dms/summary/databricks.yml.tmpl b/acceptance/bundle/dms/summary/databricks.yml.tmpl deleted file mode 100644 index 1c2fa3c7f20..00000000000 --- a/acceptance/bundle/dms/summary/databricks.yml.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -bundle: - name: dms-summary-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - foo: - name: foo diff --git a/acceptance/bundle/dms/summary/out.test.toml b/acceptance/bundle/dms/summary/out.test.toml deleted file mode 100644 index 23c07f70dca..00000000000 --- a/acceptance/bundle/dms/summary/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt deleted file mode 100644 index 233f2416dc3..00000000000 --- a/acceptance/bundle/dms/summary/output.txt +++ /dev/null @@ -1,37 +0,0 @@ - -=== Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default/files... -Created jobs.foo -Files: 5 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> [CLI] bundle summary -o json -{ - "deployment_id": "[NUMID]", - "latest_version_id": "1" -} - -=== Redeploying advances the version the summary reports ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default/files... -Files: 2 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 1 unchanged - ->>> [CLI] bundle summary -o json -{ - "deployment_id": "[NUMID]", - "latest_version_id": "2" -} - -=== After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.foo - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default - -Destroy: 1 deleted - ->>> [CLI] bundle summary -o json -false diff --git a/acceptance/bundle/dms/summary/script b/acceptance/bundle/dms/summary/script deleted file mode 100644 index 75ec8612ffe..00000000000 --- a/acceptance/bundle/dms/summary/script +++ /dev/null @@ -1,17 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on" -trace $CLI bundle deploy -trace $CLI bundle summary -o json | jq .bundle.deployment.history - -title "Redeploying advances the version the summary reports" -trace $CLI bundle deploy -trace $CLI bundle summary -o json | jq .bundle.deployment.history - -title "After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID" -trace $CLI bundle destroy --auto-approve -trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' - -# This test asserts the summary output, not the requests behind it; the file uploads -# recorded here are ordered nondeterministically. -rm -f out.requests.txt From 0978bb8d140fa143e04c589ad00fc6f3408d3d64 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 08:17:06 +0000 Subject: [PATCH 006/164] libs/dms: document the three SDK gaps the raw client works around; drop redundant operation-upload-message test Co-authored-by: Isaac --- .../operation-upload-message/databricks.yml.tmpl | 9 --------- .../dms/operation-upload-message/out.test.toml | 3 --- .../dms/operation-upload-message/output.txt | 16 ---------------- .../bundle/dms/operation-upload-message/script | 5 ----- .../dms/operation-upload-message/test.toml | 8 -------- libs/dms/client.go | 9 ++++++++- 6 files changed, 8 insertions(+), 42 deletions(-) delete mode 100644 acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/operation-upload-message/out.test.toml delete mode 100644 acceptance/bundle/dms/operation-upload-message/output.txt delete mode 100644 acceptance/bundle/dms/operation-upload-message/script delete mode 100644 acceptance/bundle/dms/operation-upload-message/test.toml diff --git a/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl deleted file mode 100644 index bc5b4ec97ad..00000000000 --- a/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -bundle: - name: dms-operation-upload-message-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - foo: - name: foo diff --git a/acceptance/bundle/dms/operation-upload-message/out.test.toml b/acceptance/bundle/dms/operation-upload-message/out.test.toml deleted file mode 100644 index 9921e91a794..00000000000 --- a/acceptance/bundle/dms/operation-upload-message/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-message/output.txt b/acceptance/bundle/dms/operation-upload-message/output.txt deleted file mode 100644 index 0791010b1f1..00000000000 --- a/acceptance/bundle/dms/operation-upload-message/output.txt +++ /dev/null @@ -1,16 +0,0 @@ - -=== What a user sees when an operation cannot be recorded ->>> fault.py PATCH /api/2.0/bundle/* 500 0 1 INTERNAL_ERROR - ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-operation-upload-message-[UNIQUE_NAME]/default/files... -Error: recording operation for resources.jobs.foo: Fault injected by test. (500 INTERNAL_ERROR) - -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus -HTTP Status: 500 Internal Server Error -API error_code: INTERNAL_ERROR -API message: Fault injected by test. - -Files: 5 uploaded, 0 deleted - -Exit code: 1 diff --git a/acceptance/bundle/dms/operation-upload-message/script b/acceptance/bundle/dms/operation-upload-message/script deleted file mode 100644 index c6b796e4149..00000000000 --- a/acceptance/bundle/dms/operation-upload-message/script +++ /dev/null @@ -1,5 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "What a user sees when an operation cannot be recorded" -trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 INTERNAL_ERROR -trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/dms/operation-upload-message/test.toml b/acceptance/bundle/dms/operation-upload-message/test.toml deleted file mode 100644 index 1dbd55da1f0..00000000000 --- a/acceptance/bundle/dms/operation-upload-message/test.toml +++ /dev/null @@ -1,8 +0,0 @@ -# Local only: this case needs an injected failure - DMS itself must reject the operation upload - and fault.py registers -# its rule on the fake, which a cloud run does not have. -Cloud = false - -# One resource, so which operation is refused is fixed and the error a user sees can be -# asserted. operation-upload-fails covers a deploy of several resources, where which ones -# were reached before the failure landed is not. -RecordRequests = false diff --git a/libs/dms/client.go b/libs/dms/client.go index 8dbf9915399..fc1f482c6b2 100644 --- a/libs/dms/client.go +++ b/libs/dms/client.go @@ -158,7 +158,14 @@ type operationResponse struct { SequenceId string `json:"sequence_id,omitempty"` } -// rawClient sends the requests the generated client cannot express. +// rawClient sends the three DMS requests the generated SDK client cannot express yet, while the +// endpoints are at DEVELOPMENT stage. TODO(DMS): drop each once the SDK catches up. +// - UpdateDeployment: the SDK exposes no method for it. +// - CreateVersion: the SDK's Version type has no operations field, so a version cannot stage +// its operations through it. +// - UpdateOperation: the SDK types sequence_id as int64, but the service sends and expects it +// as a JSON string. +// CreateDeployment, DeleteDeployment, CompleteVersion and ListResources go through the SDK (see Client). type rawClient struct { client *client.DatabricksClient } From 6163285765742e9b2a9af70bcf47ffd74413b5f1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 08:21:59 +0000 Subject: [PATCH 007/164] acceptance/dms: fold serialized-plan header assertion into stale-plan, drop serialized-plan --- .../dms/serialized-plan/databricks.yml.tmpl | 9 ---- .../bundle/dms/serialized-plan/out.test.toml | 3 -- .../bundle/dms/serialized-plan/output.txt | 41 ------------------- acceptance/bundle/dms/serialized-plan/script | 20 --------- acceptance/bundle/dms/stale-plan/output.txt | 8 ++++ acceptance/bundle/dms/stale-plan/script | 3 ++ 6 files changed, 11 insertions(+), 73 deletions(-) delete mode 100644 acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/serialized-plan/out.test.toml delete mode 100644 acceptance/bundle/dms/serialized-plan/output.txt delete mode 100644 acceptance/bundle/dms/serialized-plan/script diff --git a/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl b/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl deleted file mode 100644 index 01d891c3880..00000000000 --- a/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -bundle: - name: dms-serialized-plan-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - foo: - name: foo diff --git a/acceptance/bundle/dms/serialized-plan/out.test.toml b/acceptance/bundle/dms/serialized-plan/out.test.toml deleted file mode 100644 index 23c07f70dca..00000000000 --- a/acceptance/bundle/dms/serialized-plan/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/serialized-plan/output.txt b/acceptance/bundle/dms/serialized-plan/output.txt deleted file mode 100644 index fc84deb9f4c..00000000000 --- a/acceptance/bundle/dms/serialized-plan/output.txt +++ /dev/null @@ -1,41 +0,0 @@ - -=== Deploy once to create the deployment (records version 1) ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files... -Created jobs.foo -Files: 5 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - -=== A saved plan carries the deployment id and the versions it targets (last 1, next 2) ->>> [CLI] bundle plan -o json - ->>> jq {deployment_id, last_version_id, next_version_id} tmp.plan.json -{ - "deployment_id": "[NUMID]", - "last_version_id": "1", - "next_version_id": "2" -} - -=== Change the job and deploy again: advances the deployment to version 2 ->>> update_file.py databricks.yml name: foo name: foo-v2 - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files... -Updated jobs.foo -Files: 4 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged - -=== Replaying the earlier plan is rejected: it predates the deployment's current version ->>> musterr [CLI] bundle deploy --plan tmp.plan.json -Error: this plan predates the deployment's current version 2; run 'bundle plan' again - - -=== Only versions 1 and 2 were recorded; the rejected replay created none ->>> print_requests.py --dms //api/2.0/bundle --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-serialized-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"2\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo-v2\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/serialized-plan/script b/acceptance/bundle/dms/serialized-plan/script deleted file mode 100644 index 0800e1a8265..00000000000 --- a/acceptance/bundle/dms/serialized-plan/script +++ /dev/null @@ -1,20 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "Deploy once to create the deployment (records version 1)" -trace $CLI bundle deploy - -title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" -trace $CLI bundle plan -o json > tmp.plan.json -trace jq '{deployment_id, last_version_id, next_version_id}' tmp.plan.json - -title "Change the job and deploy again: advances the deployment to version 2" -trace update_file.py databricks.yml "name: foo" "name: foo-v2" -trace $CLI bundle deploy - -title "Replaying the earlier plan is rejected: it predates the deployment's current version" -trace musterr $CLI bundle deploy --plan tmp.plan.json - -title "Only versions 1 and 2 were recorded; the rejected replay created none" -trace print_requests.py --dms //api/2.0/bundle --oneline - -rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 2ce3b598d1c..f39b3049697 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -8,6 +8,14 @@ Created jobs.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +=== A saved plan carries the deployment id and the versions it targets (last 1, next 2) +>>> [CLI] bundle plan -o json +{ + "deployment_id": "[NUMID]", + "last_version_id": "1", + "next_version_id": "2" +} + === Replay: re-deploying the same plan is rejected - it predates the recorded version >>> musterr [CLI] bundle deploy --plan tmp.plan.json Error: this plan predates the deployment's current version 1; run 'bundle plan' again diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index 1f9e792f30f..09724d7d545 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -4,6 +4,9 @@ title "Save a plan, then deploy it: records version 1" trace $CLI bundle plan -o json > tmp.plan.json trace $CLI bundle deploy --plan tmp.plan.json +title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" +trace $CLI bundle plan -o json | jq '{deployment_id, last_version_id, next_version_id}' + title "Replay: re-deploying the same plan is rejected - it predates the recorded version" trace musterr $CLI bundle deploy --plan tmp.plan.json From 632b8ea121425052cae68a71173324266fbeda33 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 08:27:05 +0000 Subject: [PATCH 008/164] acceptance/bin: centralize recorded-state reconstruction in dms_resources.get_recorded_state --- acceptance/bin/dms_resources.py | 12 ++++++++++++ acceptance/bin/print_state.py | 10 ++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/acceptance/bin/dms_resources.py b/acceptance/bin/dms_resources.py index 25e469c5bad..77b1da5b23f 100755 --- a/acceptance/bin/dms_resources.py +++ b/acceptance/bin/dms_resources.py @@ -103,3 +103,15 @@ def get_resources(target): page_token = listed.get("next_page_token") if not page_token: return result + + +def get_recorded_state(target): + """The recorded resources in the on-disk state file's `state` shape (resources. -> + {__id__, state, depends_on}), so a recording run prints the same shape as a non-recording one.""" + state = {} + for key, value in sorted(get_resources(target).items()): + entry = {"__id__": value["id"], "state": value["state"]} + if value["depends_on"]: + entry["depends_on"] = value["depends_on"] + state[f"resources.{key}"] = entry + return state diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 2959a76f81b..f63189c02b3 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -65,16 +65,10 @@ def print_recorded_state(filename, target): printing it raw would show an empty state and differ from the same test's non-recording run. """ # Imported here rather than at module level: dms_resources reads get_state_file from this module. - from dms_resources import get_resources + from dms_resources import get_recorded_state data = json.loads(open(filename).read()) - state = {} - for key, value in sorted(get_resources(target).items()): - entry = {"__id__": value["id"], "state": value["state"]} - if value["depends_on"]: - entry["depends_on"] = value["depends_on"] - state[f"resources.{key}"] = entry - data["state"] = state + data["state"] = get_recorded_state(target) print(json.dumps(data, indent=1)) From 1b0b45f15cec3f66a1bd1d8470ee02e499a924de Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 08:59:40 +0000 Subject: [PATCH 009/164] acceptance/dms: record-failure uses a real bogus node type + a scoped stub instead of fault.py --- .../bundle/dms/record-failure/databricks.yml.tmpl | 8 ++++++++ acceptance/bundle/dms/record-failure/out.test.toml | 2 +- acceptance/bundle/dms/record-failure/output.txt | 8 +++----- acceptance/bundle/dms/record-failure/script | 1 - acceptance/bundle/dms/record-failure/test.toml | 14 +++++++++----- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/dms/record-failure/databricks.yml.tmpl b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl index 87f0eb4d1a9..f1aeba83d9a 100644 --- a/acceptance/bundle/dms/record-failure/databricks.yml.tmpl +++ b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl @@ -7,3 +7,11 @@ resources: jobs: doomed: name: doomed + tasks: + - task_key: doomed + new_cluster: + node_type_id: nonexistent-node-type + spark_version: 15.4.x-scala2.12 + num_workers: 1 + notebook_task: + notebook_path: /nonexistent diff --git a/acceptance/bundle/dms/record-failure/out.test.toml b/acceptance/bundle/dms/record-failure/out.test.toml index 9921e91a794..23c07f70dca 100644 --- a/acceptance/bundle/dms/record-failure/out.test.toml +++ b/acceptance/bundle/dms/record-failure/out.test.toml @@ -1,3 +1,3 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 479acca7a84..3556ba54b95 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -1,15 +1,13 @@ === A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource ->>> fault.py POST /api/2.2/jobs/create 400 0 1 INVALID_PARAMETER_VALUE - >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/files... -Error: cannot create resources.jobs.doomed: Fault injected by test. (400 INVALID_PARAMETER_VALUE) +Error: cannot create resources.jobs.doomed: Cluster validation failed: node type nonexistent-node-type is not supported (400 INVALID_PARAMETER_VALUE) Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Fault injected by test. +API message: Cluster validation failed: node type nonexistent-node-type is not supported Files: 6 uploaded, 0 deleted @@ -51,7 +49,7 @@ Files: 6 uploaded, 0 deleted "update_mask": "error_message,status" }, "body": { - "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "error_message": "Cluster validation failed: node type nonexistent-node-type is not supported (400 INVALID_PARAMETER_VALUE)", "sequence_id": "0", "status": "OPERATION_STATUS_FAILED" } diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index bc0947a1d91..8b629dfd6be 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -1,7 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" -trace fault.py "POST /api/2.2/jobs/create" 400 0 1 INVALID_PARAMETER_VALUE trace musterr $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/record-failure/test.toml b/acceptance/bundle/dms/record-failure/test.toml index 8a850ba07c5..e4b49d83c1c 100644 --- a/acceptance/bundle/dms/record-failure/test.toml +++ b/acceptance/bundle/dms/record-failure/test.toml @@ -1,5 +1,9 @@ -# Local only: the failure is injected, and fault.py registers its rule on the fake, which -# a cloud run does not have. Simulating it for real instead - a catalog or principal that -# does not exist - does not work either: the fake accepts both, so the case would stop -# failing locally. Making it portable needs the fake to model that existence. -Cloud = false +# The job's cluster names a node type that does not exist, so the create fails for real: on cloud +# the Jobs API rejects it. The local fake is lax and would accept it, so this one test stubs the +# create to return the same error - scoped here, not the shared fake, so no other test is affected. +Cloud = true + +[[Server]] +Pattern = "POST /api/2.2/jobs/create" +Response.StatusCode = 400 +Response.Body = '''{"error_code": "INVALID_PARAMETER_VALUE", "message": "Cluster validation failed: node type nonexistent-node-type is not supported"}''' From 09f61c3d98863ed3d802c9e656eb8331b2c40a88 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 09:30:01 +0000 Subject: [PATCH 010/164] libs/dms: gofmt the rawClient doc comment --- libs/dms/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/dms/client.go b/libs/dms/client.go index fc1f482c6b2..611bdf9c11b 100644 --- a/libs/dms/client.go +++ b/libs/dms/client.go @@ -165,6 +165,7 @@ type operationResponse struct { // its operations through it. // - UpdateOperation: the SDK types sequence_id as int64, but the service sends and expects it // as a JSON string. +// // CreateDeployment, DeleteDeployment, CompleteVersion and ListResources go through the SDK (see Client). type rawClient struct { client *client.DatabricksClient From d42b08e82806a3ebc0a644267223208dfb9d7d94 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 10:58:50 +0000 Subject: [PATCH 011/164] bundle: make the recorded-state marker the source of truth for DMS; Open reads the workspace client from context Co-authored-by: Isaac --- .../bundle/dms/record-failure/output.txt | 6 +- .../bundle/dms/record-failure/test.toml | 7 +- bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +-- bundle/direct/dstate/state.go | 94 ++++++++++--------- bundle/direct/dstate/state_test.go | 36 +++---- bundle/statemgmt/direct_migration.go | 2 +- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 34 +++---- 11 files changed, 103 insertions(+), 96 deletions(-) diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 3556ba54b95..819004844d7 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -2,12 +2,12 @@ === A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/files... -Error: cannot create resources.jobs.doomed: Cluster validation failed: node type nonexistent-node-type is not supported (400 INVALID_PARAMETER_VALUE) +Error: cannot create resources.jobs.doomed: Node type nonexistent-node-type is not supported. Supported node types: [NODE_TYPES] (400 INVALID_PARAMETER_VALUE) Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Cluster validation failed: node type nonexistent-node-type is not supported +API message: Node type nonexistent-node-type is not supported. Supported node types: [NODE_TYPES] Files: 6 uploaded, 0 deleted @@ -49,7 +49,7 @@ Files: 6 uploaded, 0 deleted "update_mask": "error_message,status" }, "body": { - "error_message": "Cluster validation failed: node type nonexistent-node-type is not supported (400 INVALID_PARAMETER_VALUE)", + "error_message": "Node type nonexistent-node-type is not supported. Supported node types: [NODE_TYPES] (400 INVALID_PARAMETER_VALUE)", "sequence_id": "0", "status": "OPERATION_STATUS_FAILED" } diff --git a/acceptance/bundle/dms/record-failure/test.toml b/acceptance/bundle/dms/record-failure/test.toml index e4b49d83c1c..49f63c399ac 100644 --- a/acceptance/bundle/dms/record-failure/test.toml +++ b/acceptance/bundle/dms/record-failure/test.toml @@ -3,7 +3,12 @@ # create to return the same error - scoped here, not the shared fake, so no other test is affected. Cloud = true +# The Jobs API error lists every supported node type, and the set grows over time; mask it. +[[Repls]] +Old = 'Supported node types: [^(\n]*[^ (\n]' +New = 'Supported node types: [NODE_TYPES]' + [[Server]] Pattern = "POST /api/2.2/jobs/create" Response.StatusCode = 400 -Response.Body = '''{"error_code": "INVALID_PARAMETER_VALUE", "message": "Cluster validation failed: node type nonexistent-node-type is not supported"}''' +Response.Body = '''{"error_code": "INVALID_PARAMETER_VALUE", "message": "Node type nonexistent-node-type is not supported. Supported node types: r3.xlarge, m5.large"}''' diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 40511ae1dcd..f5afb8d5b1a 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index c16655a3518..462c0b4c71f 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -156,7 +156,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 9c4ab260e03..a65cacd7eb4 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -64,7 +64,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac // The state is opened without a DMS client, so the writes below record nothing; // phases.Bind and phases.Unbind refuse to run at all when recording is enabled. var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -88,7 +88,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -111,7 +111,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), "") + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -147,7 +147,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -167,7 +167,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), "") + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -217,7 +217,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") if err != nil { return err } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 4e2f8c03344..6a9d1c4d2ee 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -18,9 +18,9 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go" "github.com/google/uuid" ) @@ -364,7 +364,7 @@ type ( // recorded deploy); lineage and serial still come from the file, since that is what the write path // increments. Open only reads through the client - StartRecording installs the write path once a // version exists. -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, wsClient *databricks.WorkspaceClient, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { db.mu.Lock() defer db.mu.Unlock() @@ -372,7 +372,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W panic(fmt.Sprintf("state already opened: %v, cannot open %v", db.Path, path)) } - err := db.unlockedOpen(ctx, path, withRecovery, withWrite, wsClient, withDeploymentHistory, dmsDeploymentID) + err := db.unlockedOpen(ctx, path, withRecovery, withWrite, withDeploymentHistory, dmsDeploymentID) if err != nil { // A failed open must leave the receiver closed. unlockedOpen assigns // db.Path before every fallible step, so without this the receiver stays @@ -394,20 +394,11 @@ func (db *DeploymentState) reset() { db.stateIDs = nil } -func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, wsClient *databricks.WorkspaceClient, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { +func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { db.Path = path - if withDeploymentHistory { - // The service is the source of truth for a recorded deployment, so a leftover WAL is - // discarded below rather than replayed: force recovery off regardless of what was asked. - withRecovery = false - client, err := dms.NewClient(wsClient) - if err != nil { - return err - } - db.dmsClient = client - } - + // The state file is the source of truth for whether this deployment records history: read it + // first, then reconcile the config against the marker it carries. data, err := os.ReadFile(db.Path) if err != nil { if errors.Is(err, fs.ErrNotExist) { @@ -421,6 +412,16 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe } } + if err := migrateState(&db.Data); err != nil { + return fmt.Errorf("migrating state %s: %w", path, err) + } + + if err := checkStateFeatures(db.Data.Features); err != nil { + return err + } + + recording := bool(withDeploymentHistory) + db.stateIDs = make(map[string]string) for key, entry := range db.Data.State { db.stateIDs[key] = entry.ID @@ -433,56 +434,57 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe return fmt.Errorf("failed to stat WAL file %s: %w", walPath, statErr) } - // A recorded deployment gets its resources from the service (applyDMSState below), so the WAL - - // a local log of an uncommitted deploy - is not merged. A leftover from a crash or a declined - // deploy is removed so the next write's exclusive create of the WAL does not fail. Without + // A recording open discards a leftover WAL - the service owns the resources, so a local log from + // a crash or a declined deploy is irrelevant, even when recording is refused below. Without // recording, replay the WAL as usual. if walExists { - if withDeploymentHistory { + switch { + case recording: if err := os.Remove(walPath); err != nil { return fmt.Errorf("removing WAL file %s: %w", walPath, err) } - } else if withRecovery { + case bool(withRecovery): if err := db.replayWAL(ctx); err != nil { return fmt.Errorf("reading state from %s: %w", path, err) } - } else { + default: return fmt.Errorf("unexpected WAL file found at %s", walPath) } } - if err := migrateState(&db.Data); err != nil { - return fmt.Errorf("migrating state %s: %w", path, err) - } - - if err := checkStateFeatures(db.Data.Features); err != nil { - return err - } - - if withDeploymentHistory { - // The service is the source of truth for a recorded deployment; local state is irrelevant. - // This state was pulled from the workspace, so it reflects the remote and not just a local - // cache, and a leftover WAL was discarded above rather than replayed. It records the feature - // once recording owns it, so a recorded deployment carries no resources here (a tombstone) - // and applyDMSState below loads the service's. A state that still tracks resources without - // the feature is a deployment made before recording: treating the service as authoritative - // would create those resources a second time, so refuse it (adopting one is a followup). - if _, recorded := db.Data.Features[featureRecordDeploymentHistory]; !recorded && len(db.Data.State) > 0 { - return errors.New(`this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + // Reconcile the config against the state. A brand-new deployment is where recording begins, so + // the config bootstraps the marker there. On an existing deployment the state is authoritative: + // the config can neither start recording one that was not (its resources would be created a + // second time) nor stop recording one that is (the service still holds it). + _, recorded := db.Data.Features[featureRecordDeploymentHistory] + switch { + case recording && !recorded && len(db.Data.State) == 0: + if db.Data.Features == nil { + db.Data.Features = make(map[string]struct{}, 1) + } + db.Data.Features[featureRecordDeploymentHistory] = struct{}{} + recorded = true + case recording && !recorded: + return errors.New(`this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: 1. remove experimental.record_deployment_history from your bundle configuration 2. run "databricks bundle destroy" to delete the existing resources 3. add experimental.record_deployment_history back and deploy again`) - } + case !recording && recorded: + return errors.New(`unsetting experimental.record_deployment_history is not supported - // Mark the state as depending on the service so this CLI keeps recording it (see - // RequiresDeploymentHistory). unlockedSave then writes the header alone. Bumping the state - // version so older CLIs refuse a recorded state is a followup. - if db.Data.Features == nil { - db.Data.Features = make(map[string]struct{}, 1) +This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) + } + + if recorded { + // The service is the source of truth for a recorded deployment; the file is a tombstone + // carrying only the marker, and applyDMSState loads the resources the service holds. + client, err := dms.NewClient(cmdctx.WorkspaceClient(ctx)) + if err != nil { + return err } - db.Data.Features[featureRecordDeploymentHistory] = struct{}{} + db.dmsClient = client if dmsDeploymentID != "" { resources, err := db.dmsClient.ListResources(ctx, dmsDeploymentID) diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 8ccd6277ecb..f7d605e39f8 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -21,14 +21,14 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) @@ -38,7 +38,7 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) mustFinalize(t, &db) _, err := os.Stat(path) @@ -94,10 +94,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "") + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "") }) mustFinalize(t, &db) } @@ -115,12 +115,12 @@ func TestCLIVersionRecordsLastWriter(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) mustFinalize(t, &db) var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, build.GetInfo().Version, reopened.Data.CLIVersion) assert.Equal(t, 2, reopened.Data.Serial) mustFinalize(t, &reopened) @@ -142,7 +142,7 @@ func TestHeaderOnlyWALDoesNotUpdateCLIVersion(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, "0.1.2", recovered.Data.CLIVersion, "a header-only WAL wrote no state, so the version must not move") assert.Equal(t, 1, recovered.Data.Serial) mustFinalize(t, &recovered) @@ -154,12 +154,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -175,7 +175,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -186,17 +186,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", false)) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -208,7 +208,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), "")) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -225,7 +225,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } @@ -239,13 +239,13 @@ func TestOpenFailureLeavesStateClosed(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte("{not json"), 0o600)) var db DeploymentState - require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) assert.Empty(t, db.Path) // Once the state file is readable, the same receiver opens without panicking. seed := `{"state_version":2,"cli_version":"0.1.2","lineage":"test-lineage","serial":1,"state":{}}` require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) assert.Equal(t, "test-lineage", db.Data.Lineage) mustFinalize(t, &db) } diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 5d54e8702e5..f2140823b28 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -173,7 +173,7 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s var planBundle direct.DeploymentBundle // This plan check does not record deployment history, so no DMS client or deployment id. - if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, nil, dstate.WithDeploymentHistory(false), ""); err != nil { + if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, dstate.WithDeploymentHistory(false), ""); err != nil { return fmt.Errorf("opening migrated state for plan check: %w", err) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 14bc85777c1..6047655b412 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index ea37ff50990..f87529b869a 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 1d671fbc306..3074ba4272c 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -23,6 +23,7 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" @@ -283,25 +284,24 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } - } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), b.WorkspaceClient(ctx), dstate.WithDeploymentHistory(b.ConfiguresDeploymentHistory(ctx)), dmsDeploymentID); err != nil { - logdiag.LogError(ctx, err) - return b, stateDesc, root.ErrAlreadyPrinted - } + // StateDB.Open builds the DMS client from the workspace client on the context. A bundle + // command resolves that client lazily through b, so it is not otherwise on the context. + if !cmdctx.HasWorkspaceClient(ctx) { + ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) + } + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dmsDeploymentID); err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } - // Open built the DMS client from the workspace client when recording; hand it to the - // phases that create versions and deployments. - if b.ConfiguresDeploymentHistory(ctx) { + // Open built the DMS client from the workspace client on the context; hand it to the + // phases that create versions and deployments. b.DeploymentBundle.DmsApiClient = b.DeploymentBundle.StateDB.DmsClient() - } - - // The service holds this deployment, so it has to keep being recorded: deploying - // without recording would leave it describing resources that have moved on. - if !b.ConfiguresDeploymentHistory(ctx) && b.DeploymentBundle.StateDB.RequiresDeploymentHistory() { - logdiag.LogError(ctx, errors.New(`unsetting experimental.record_deployment_history is not supported - -This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`)) - return b, stateDesc, root.ErrAlreadyPrinted + } else { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } } // Warn when the state was last written by a newer CLI than the one From 8dc139da2eb2fbc5da634faffabb4119c74c5eec Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 11:14:19 +0000 Subject: [PATCH 012/164] bundle: move the DMS client and operation buffer into StateDB; drop them from DeploymentBundle The phases now reach DMS through StateDB (DmsClient, OperationBuffer, TakeOperationBuffer) rather than DeploymentBundle mirror fields. The operation drain stays before CompleteVersion. Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 4 ++-- bundle/direct/dstate/state.go | 22 +++++++++++++++++++--- bundle/direct/pkg.go | 8 -------- bundle/phases/deploy.go | 2 +- bundle/phases/destroy.go | 4 ++-- bundle/phases/dms.go | 17 +++++++++-------- cmd/bundle/utils/process.go | 4 ---- 7 files changed, 33 insertions(+), 28 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 72465fad346..818a1e95a23 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -77,7 +77,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } // Stop resource CRUD once recording state with DMS has failed. - if buf := b.DmsAsyncOperationClient; buf != nil { + if buf := b.StateDB.OperationBuffer(); buf != nil { if err := buf.Err(); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false @@ -183,7 +183,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Wait for the uploads and report a failure here, with the deploy's other errors. The phase // completes the version afterwards and drains again, quietly, so this is not printed twice. - if buf := b.DmsAsyncOperationClient; buf != nil { + if buf := b.StateDB.OperationBuffer(); buf != nil { if err := buf.Drain(); err != nil { logdiag.LogError(ctx, err) } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 6a9d1c4d2ee..ebddd3683b3 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -138,14 +138,30 @@ type WALEntry struct { // StartRecording opens the operation buffer that records each subsequent state write with DMS // under deploymentID and versionID, so what the service holds mirrors the WAL. It uses the client // Open built, so it is called only when the deployment records history, and once the version -// exists (after approval) - which is why it is not an Open option. Returns the buffer so the -// deploy can drain it. -func (db *DeploymentState) StartRecording(ctx context.Context, deploymentID string, versionID int64) *dms.OperationBuffer { +// exists (after approval) - which is why it is not an Open option. +func (db *DeploymentState) StartRecording(ctx context.Context, deploymentID string, versionID int64) { buf := dms.StartOperationBuffer(ctx, db.dmsClient, deploymentID, versionID) db.mu.Lock() defer db.mu.Unlock() db.operationBuffer = buf +} + +// OperationBuffer returns the buffer recording state writes to DMS, or nil when the bundle does not +// record deployment history. Apply reads it to surface a recording failure and to drain uploads. +func (db *DeploymentState) OperationBuffer() *dms.OperationBuffer { + db.mu.Lock() + defer db.mu.Unlock() + return db.operationBuffer +} + +// TakeOperationBuffer returns the operation buffer and clears it, so a deferred safety-net +// completion after an explicit one is a no-op. +func (db *DeploymentState) TakeOperationBuffer() *dms.OperationBuffer { + db.mu.Lock() + defer db.mu.Unlock() + buf := db.operationBuffer + db.operationBuffer = nil return buf } diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 764397d5fc3..ea7ae1e51b6 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/statemgmt/resourcestate" - "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/structs/structvar" ) @@ -51,13 +50,6 @@ type DeploymentBundle struct { Plan *deployplan.Plan RemoteStateCache sync.Map StateCache structvar.Cache - - // DMS carries deployment-history recording across the phases; nil unless the bundle records - // history. DmsApiClient is set where the state is opened. The deployment id and version live - // in bundle.deployment.history, not here. DmsAsyncOperationClient is opened once the deploy - // phase creates the version, and Apply drains it before returning. - DmsApiClient *dms.Client - DmsAsyncOperationClient *dms.OperationBuffer } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 5fe25dd2eee..b35757af167 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -326,7 +326,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. - if b.DeploymentBundle.DmsApiClient != nil { + if b.DeploymentBundle.StateDB.DmsClient() != nil { existingID, _ := deploymentAndNextVersion(b) createOrUpdateDeployment(ctx, b, dmsDeployment) if logdiag.HasError(ctx) { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 720f4ef7fa9..420e2a882d3 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -168,7 +168,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e // A completed destroy's resources are gone, so its deployment record is deleted too. if completed { deploymentID, _ := deploymentAndNextVersion(b) - if err := b.DeploymentBundle.DmsApiClient.DeleteDeployment(ctx, deploymentID); err != nil { + if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) return } @@ -222,7 +222,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { } else if completed { // A completed destroy's resources are gone, so its deployment record is deleted too. deploymentID, _ := deploymentAndNextVersion(b) - if err := b.DeploymentBundle.DmsApiClient.DeleteDeployment(ctx, deploymentID); err != nil { + if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) } } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 40257fe5f5c..fc578aa62a6 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -85,18 +85,19 @@ func deploymentAndNextVersion(b *bundle.Bundle) (string, int64) { // no deployment behind; a first deploy's new id is then stamped into the plan (StampDeploymentID). func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bundledeployments.Deployment) { db := &b.DeploymentBundle + dmsClient := db.StateDB.DmsClient() metadata := deploymentMetadata(b) deploymentID, _ := deploymentAndNextVersion(b) switch mask := metadata.StaleFields(current); { case deploymentID == "": - id, err := db.DmsApiClient.CreateDeployment(ctx, b.Config.Workspace.StatePath, metadata) + id, err := dmsClient.CreateDeployment(ctx, b.Config.Workspace.StatePath, metadata) if err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to create deployment: %w", err)) return } deploymentID = id case mask != "": - if err := db.DmsApiClient.UpdateDeployment(ctx, deploymentID, metadata, mask); err != nil { + if err := dmsClient.UpdateDeployment(ctx, deploymentID, metadata, mask); err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to update deployment: %w", err)) return } @@ -116,7 +117,8 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu // A no-op when the bundle does not record deployment history. func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.VersionType, staged []dms.StagedOperation) error { db := &b.DeploymentBundle - if db.DmsApiClient == nil { + dmsClient := db.StateDB.DmsClient() + if dmsClient == nil { return nil } deploymentID, versionID := deploymentAndNextVersion(b) @@ -139,7 +141,7 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version OriginUrl: git.OriginURL, } } - version, err := db.DmsApiClient.CreateVersion(ctx, deploymentID, versionID, dms.CreateVersionRequest{ + version, err := dmsClient.CreateVersion(ctx, deploymentID, versionID, dms.CreateVersionRequest{ CliVersion: build.GetInfo().Version, VersionType: versionType, PreviousVersionId: previousVersionID, @@ -151,7 +153,7 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version } log.Infof(ctx, "Created deployment version: deployment=%s version=%s", deploymentID, version.VersionId) - db.DmsAsyncOperationClient = db.StateDB.StartRecording(ctx, deploymentID, versionID) + db.StateDB.StartRecording(ctx, deploymentID, versionID) return nil } @@ -162,11 +164,10 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version // was created. func drainOperationsAndCompleteVersion(ctx context.Context, b *bundle.Bundle, success bool) (bool, error) { db := &b.DeploymentBundle - buf := db.DmsAsyncOperationClient + buf := db.StateDB.TakeOperationBuffer() if buf == nil { return false, nil } - db.DmsAsyncOperationClient = nil // Drain first; a recording error fails the deploy even when the resources applied. Its error // is left to whoever drained (apply), not returned twice. @@ -179,7 +180,7 @@ func drainOperationsAndCompleteVersion(ctx context.Context, b *bundle.Bundle, su reason = bundledeployments.VersionCompleteVersionCompleteFailure } deploymentID, versionID := deploymentAndNextVersion(b) - if err := db.DmsApiClient.CompleteVersion(ctx, deploymentID, versionID, reason); err != nil { + if err := db.StateDB.DmsClient().CompleteVersion(ctx, deploymentID, versionID, reason); err != nil { return false, err } log.Infof(ctx, "Completed deployment version: deployment=%s version=%d reason=%s", deploymentID, versionID, reason) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 3074ba4272c..620d6ae9048 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -293,10 +293,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } - - // Open built the DMS client from the workspace client on the context; hand it to the - // phases that create versions and deployments. - b.DeploymentBundle.DmsApiClient = b.DeploymentBundle.StateDB.DmsClient() } else { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) From 8ec86f85862575e1c4a07a5e32e5a8aeb3639884 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 12:43:31 +0000 Subject: [PATCH 013/164] bundle/dms: StorageBackend enum as source of truth; hide operation buffer in StateDB - add dstate.StorageBackend (WORKSPACE_FILESYSTEM/DEPLOYMENT_METADATA_SERVICE), set by Open from the state marker; the plan carries its string form so deploy --plan can reject a plan whose target switched backends - StateDB owns the recording lifecycle: RecordingError() and CompleteVersion() (idempotent via versionCompleted), with the operation-buffer drain moved into Finalize; drop the exposed OperationBuffer/TakeOperationBuffer getters and the recorder() indirection - gate DMS behavior on StorageBackend()==DEPLOYMENT_METADATA_SERVICE / ConfiguresDeploymentHistory instead of DmsClient()!=nil - consolidate plan validation in validatePlan (config-vs-plan backend match + stale-plan guard) Co-authored-by: Isaac --- bundle/deployplan/plan.go | 5 ++ bundle/direct/bundle_apply.go | 15 +---- bundle/direct/bundle_plan.go | 6 ++ bundle/direct/dstate/state.go | 110 +++++++++++++++++++++++++++------- bundle/phases/deploy.go | 11 +++- bundle/phases/destroy.go | 4 +- bundle/phases/dms.go | 31 ---------- cmd/bundle/utils/process.go | 63 ++++++++++++------- 8 files changed, 152 insertions(+), 93 deletions(-) diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 3f24c82ee43..726e0de9a92 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -29,6 +29,11 @@ type Plan struct { NextVersionId string `json:"next_version_id,omitempty"` LastVersionId string `json:"last_version_id,omitempty"` + // StorageBackend is where this deployment's state lives, the string form of a + // dstate.StorageBackend. deploy --plan checks it against the target's config so a recorded plan + // is never applied to a non-recording target or the reverse. Set only for a recorded plan. + StorageBackend string `json:"storage_backend,omitempty"` + Plan map[string]*PlanEntry `json:"plan,omitzero"` // NotSelected is the number of resources removed by FilterToSelected via the diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 818a1e95a23..8c2de1f29ee 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -77,11 +77,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } // Stop resource CRUD once recording state with DMS has failed. - if buf := b.StateDB.OperationBuffer(); buf != nil { - if err := buf.Err(); err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) - return false - } + if err := b.StateDB.RecordingError(); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false } adapter, err := b.getAdapterForKey(resourceKey) @@ -181,13 +179,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) - // Wait for the uploads and report a failure here, with the deploy's other errors. The phase - // completes the version afterwards and drains again, quietly, so this is not printed twice. - if buf := b.StateDB.OperationBuffer(); buf != nil { - if err := buf.Drain(); err != nil { - logdiag.LogError(ctx, err) - } - } } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 45d5b2d3663..4c1c4f4e3a8 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -172,6 +172,12 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // deploy --plan can reject a plan the deployment has moved on from, and pass last_version_id // as previous_version_id. History is set only while recording, so other plans leave these empty. // configRoot is nil for destroy, which records its version separately. + // The plan records where its state lives so deploy --plan can reject a plan whose target has + // since switched backends. Set from StateDB (the source of truth) so a first deploy - which has + // no History yet - still carries it. + if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + plan.StorageBackend = string(dstate.StorageBackendDeploymentMetadataService) + } if configRoot != nil && configRoot.Bundle.Deployment.History != nil { h := configRoot.Bundle.Deployment.History next, err := dms.NextVersion(h.LatestVersionID) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ebddd3683b3..2e410ee25b8 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -21,6 +21,7 @@ import ( "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -51,6 +52,16 @@ const featureRecordDeploymentHistory = "record_deployment_history" // state that depends on a feature it does not recognize. const featuresDocURL = "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" +// StorageBackend identifies where a deployment's resource state lives: the workspace filesystem +// (the state file) or the deployment metadata service. It is the source of truth for gating DMS +// behavior after Open; the plan carries its string form to check against the target's config. +type StorageBackend string + +const ( + StorageBackendWorkspaceFilesystem StorageBackend = "WORKSPACE_FILESYSTEM" + StorageBackendDeploymentMetadataService StorageBackend = "DEPLOYMENT_METADATA_SERVICE" +) + // recognizedFeatures is the set of state feature flags this CLI understands. A state depending on // any feature not listed here is refused (see checkStateFeatures), so a newer CLI's feature is not // silently clobbered by this one. @@ -95,6 +106,20 @@ type DeploymentState struct { // dmsClient talks to the deployment metadata service. Open builds it from the workspace // client when the deployment records history; nil otherwise. dmsClient *dms.Client + + // storageBackend is where this deployment's state lives, set by Open from the feature marker. + // It is the source of truth for gating DMS behavior after Open, in place of nil client checks. + storageBackend StorageBackend + + // deploymentID and versionID identify the recorded deployment and the version this run writes, + // stored by StartRecording. CompleteVersion reads them after Finalize's reset has cleared Path, + // Data and stateIDs, so - like operationBuffer and dmsClient - they must survive reset. + deploymentID string + versionID int64 + + // versionCompleted makes CompleteVersion a no-op after the first call, so a deferred safety-net + // completion after an explicit one does nothing. + versionCompleted bool } type Header struct { @@ -145,31 +170,55 @@ func (db *DeploymentState) StartRecording(ctx context.Context, deploymentID stri db.mu.Lock() defer db.mu.Unlock() db.operationBuffer = buf + db.deploymentID = deploymentID + db.versionID = versionID } -// OperationBuffer returns the buffer recording state writes to DMS, or nil when the bundle does not -// record deployment history. Apply reads it to surface a recording failure and to drain uploads. -func (db *DeploymentState) OperationBuffer() *dms.OperationBuffer { +// RecordingError reports whether recording state writes to the service has failed, so the apply +// stops touching resources once the service is no longer keeping up. Nil when the bundle does not +// record deployment history or recording is healthy. +func (db *DeploymentState) RecordingError() error { db.mu.Lock() - defer db.mu.Unlock() - return db.operationBuffer + buf := db.operationBuffer + db.mu.Unlock() + if buf == nil { + return nil + } + return buf.Err() } -// TakeOperationBuffer returns the operation buffer and clears it, so a deferred safety-net -// completion after an explicit one is a no-op. -func (db *DeploymentState) TakeOperationBuffer() *dms.OperationBuffer { +// CompleteVersion marks the recorded version done, reporting whether it completed here. A no-op +// returning false when no version was created (the bundle does not record history, or a deploy was +// declined) or the version was already completed, so a deferred safety-net call after an explicit +// one does nothing. Finalize has already drained the buffered operations. It reads only fields that +// survive Finalize's reset and asserts nothing, so it is safe to call after the state is closed. +func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (bool, error) { db.mu.Lock() - defer db.mu.Unlock() - buf := db.operationBuffer - db.operationBuffer = nil - return buf + if db.operationBuffer == nil || db.versionCompleted { + db.mu.Unlock() + return false, nil + } + db.versionCompleted = true + deploymentID, versionID, client := db.deploymentID, db.versionID, db.dmsClient + db.mu.Unlock() + + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if !success { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + if err := client.CompleteVersion(ctx, deploymentID, versionID, reason); err != nil { + return false, err + } + log.Infof(ctx, "Completed deployment version: deployment=%s version=%d reason=%s", deploymentID, versionID, reason) + // Report the success value the version completed with, not merely that it completed: a destroy + // deletes the deployment record only for a version that completed successfully. + return success, nil } // RecordFailure records that a resource did not apply, so the history says why rather than // leaving the resource out. resourceID is the id it had before the failure. func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause error) { - r := db.recorder() - if r == nil { + if db.StorageBackend() != StorageBackendDeploymentMetadataService { return } @@ -186,15 +235,11 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e } } - r.RecordFailure(resourceKey, resourceID, recorded, cause) -} - -// recorder reads the buffer under db.mu, which guards it, and returns nil when the bundle does -// not record deployment history. -func (db *DeploymentState) recorder() *dms.OperationBuffer { + // A failure is recorded only while a version is open (after StartRecording), so the buffer is set. db.mu.Lock() - defer db.mu.Unlock() - return db.operationBuffer + buf := db.operationBuffer + db.mu.Unlock() + buf.RecordFailure(resourceKey, resourceID, recorded, cause) } func NewDatabase(lineage string, serial int) Database { @@ -333,6 +378,15 @@ func (db *DeploymentState) RequiresDeploymentHistory() bool { return ok } +// StorageBackend reports where this deployment's state lives - the source of truth for whether the +// bundle records deployment history. Set by Open; valid only after the state is opened. +func (db *DeploymentState) StorageBackend() StorageBackend { + db.AssertOpenedForReadOrWrite() + db.mu.Lock() + defer db.mu.Unlock() + return db.storageBackend +} + // DmsClient returns the deployment metadata service client Open built from the workspace client, // or nil when the deployment does not record history. func (db *DeploymentState) DmsClient() *dms.Client { @@ -493,7 +547,10 @@ To record this bundle's history, start it over as a new deployment: This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) } + db.storageBackend = StorageBackendWorkspaceFilesystem if recorded { + db.storageBackend = StorageBackendDeploymentMetadataService + // The service is the source of truth for a recorded deployment; the file is a tombstone // carrying only the marker, and applyDMSState loads the resources the service holds. client, err := dms.NewClient(cmdctx.WorkspaceClient(ctx)) @@ -697,6 +754,15 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported err = db.replayWAL(ctx) } + // A recorded deployment uploads each write to the service on a background goroutine; block until + // they land or fail so the version completes only over state the service actually holds. A nil + // buffer means the deployment does not record history. + if db.operationBuffer != nil { + if drainErr := db.operationBuffer.Drain(); drainErr != nil { + err = errors.Join(err, drainErr) + } + } + state := ExportStateFromData(db.Data) db.reset() diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index b35757af167..9ffe6eb3b44 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" @@ -204,8 +205,12 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // The version is created only after approval; CompleteVersion is deferred before // lock.Release and no-ops until then. defer func() { - if _, err := drainOperationsAndCompleteVersion(ctx, b, !logdiag.HasError(ctx)); err != nil { - logdiag.LogError(ctx, err) + // Skip entirely when not recording; CompleteVersion runs after Finalize has closed the state, + // so it is gated on the config rather than the (reset) StateDB. + if b.ConfiguresDeploymentHistory(ctx) { + if _, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() @@ -326,7 +331,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. - if b.DeploymentBundle.StateDB.DmsClient() != nil { + if b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { existingID, _ := deploymentAndNextVersion(b) createOrUpdateDeployment(ctx, b, dmsDeployment) if logdiag.HasError(ctx) { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 420e2a882d3..309adb120da 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -160,7 +160,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e } // Complete version before deleting remote files; the deployment node is under statePath. - completed, err := drainOperationsAndCompleteVersion(ctx, b, true) + completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) if err != nil { logdiag.LogError(ctx, err) return @@ -216,7 +216,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // destroy records nothing. Deferred before lock.Release to hold the lock; a no-op once // destroyCore has completed the version. defer func() { - completed, err := drainOperationsAndCompleteVersion(ctx, b, !logdiag.HasError(ctx)) + completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)) if err != nil { logdiag.LogError(ctx, err) } else if completed { diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index fc578aa62a6..2db8bf57dca 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -157,37 +157,6 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version return nil } -// drainOperationsAndCompleteVersion drains the buffered operations and completes the version. -// It returns whether the version completed successfully, which a destroy uses to decide whether -// to delete the deployment. Idempotent: the first call clears the buffer, so a deferred -// safety-net after an explicit completion is a no-op returning false. Also false when no version -// was created. -func drainOperationsAndCompleteVersion(ctx context.Context, b *bundle.Bundle, success bool) (bool, error) { - db := &b.DeploymentBundle - buf := db.StateDB.TakeOperationBuffer() - if buf == nil { - return false, nil - } - - // Drain first; a recording error fails the deploy even when the resources applied. Its error - // is left to whoever drained (apply), not returned twice. - if buf.Drain() != nil { - success = false - } - - reason := bundledeployments.VersionCompleteVersionCompleteSuccess - if !success { - reason = bundledeployments.VersionCompleteVersionCompleteFailure - } - deploymentID, versionID := deploymentAndNextVersion(b) - if err := db.StateDB.DmsClient().CompleteVersion(ctx, deploymentID, versionID, reason); err != nil { - return false, err - } - log.Infof(ctx, "Completed deployment version: deployment=%s version=%d reason=%s", deploymentID, versionID, reason) - - return success, nil -} - // logDeploymentVersion logs the deployment version URL. Workspace ID is omitted // so the page stays clickable in a terminal and redirects correctly without it. func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, deploymentID string, version int64) { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 620d6ae9048..74909c20997 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -364,29 +364,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle log.Warnf(ctx, "Plan was created with CLI version %s but current version is %s", plan.CLIVersion, currentVersion) } - // The plan records the DMS deployment and version it targeted. Reject it if the live - // deployment moved on - a newer version (someone deployed since, possibly from another - // machine) or a different id (deleted and recreated) - so a stale plan is never applied on - // top of newer state. This runs before the lineage/serial check and is the authoritative - // stale guard for recorded bundles, whose local state is only a tombstone (its serial does - // not catch a deploy from elsewhere). Both sides are empty when not recording, a no-op there. - remoteLastVersion := "" - if dmsDeployment != nil { - remoteLastVersion = dmsDeployment.LastVersionId - } - if plan.LastVersionId != remoteLastVersion { - logdiag.LogError(ctx, fmt.Errorf("this plan predates the deployment's current version %s; run 'bundle plan' again", remoteLastVersion)) - return b, stateDesc, root.ErrAlreadyPrinted - } - if plan.DeploymentId != dmsDeploymentID { - logdiag.LogError(ctx, errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again")) - return b, stateDesc, root.ErrAlreadyPrinted - } - - // Validate that the plan's lineage and serial match the local state. This is the stale guard - // for non-recorded bundles (recorded ones are covered by the version check above). - err = direct.ValidatePlanAgainstState(&b.DeploymentBundle.StateDB, plan) - if err != nil { + if err := validatePlan(ctx, b, plan, dmsDeployment, dmsDeploymentID); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } @@ -484,6 +462,45 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, nil } +// validatePlan rejects a --plan file that no longer matches the target: its recording shape must +// match the target's config, and a plan that targets an existing recorded deployment must not +// predate the deployment the service now holds. It ends with the local lineage/serial guard. +func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, dmsDeployment *bundledeployments.Deployment, dmsDeploymentID string) error { + // A plan records its storage backend, which must match the target's current one, or it would + // deploy against the wrong backend. StateDB.StorageBackend is the target's source of truth, and + // it is set even for a first recorded deploy (unlike the version ids, which are empty then). + isDMSPlan := plan.StorageBackend == string(dstate.StorageBackendDeploymentMetadataService) + switch records := b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService; { + case isDMSPlan && !records: + return errors.New("this plan records deployment history, but the target no longer does; run 'bundle plan' again") + case !isDMSPlan && records: + return errors.New("this plan does not record deployment history, but the target now does; run 'bundle plan' again") + } + + // The plan records the DMS deployment and version it targeted. Reject it if the live deployment + // moved on - a newer version (someone deployed since, possibly from another machine) or a + // different id (deleted and recreated) - so a stale plan is never applied on top of newer state. + // This is the authoritative stale guard for a recorded bundle, whose local state is only a + // tombstone (its serial does not catch a deploy from elsewhere). Gated on the plan being a DMS + // plan; a first-deploy plan has empty version ids that still get compared (and match) here. + if isDMSPlan { + remoteLastVersion := "" + if dmsDeployment != nil { + remoteLastVersion = dmsDeployment.LastVersionId + } + if plan.LastVersionId != remoteLastVersion { + return fmt.Errorf("this plan predates the deployment's current version %s; run 'bundle plan' again", remoteLastVersion) + } + if plan.DeploymentId != dmsDeploymentID { + return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") + } + } + + // Validate that the plan's lineage and serial match the local state. This is the stale guard for + // non-recorded bundles (recorded ones are covered by the version check above). + return direct.ValidatePlanAgainstState(&b.DeploymentBundle.StateDB, plan) +} + // ResolveEngineSetting determines the effective engine setting by combining bundle config and env var. // Priority: bundle.engine config > DATABRICKS_BUNDLE_ENGINE env var > engine.Default. func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineSetting, error) { From 3683aeddfcdb2de34ea6bd42f8e768dccd005e38 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 12:55:38 +0000 Subject: [PATCH 014/164] bundle/dms: rename first-version stamp, inline env predicate, state-gate unbind, restore v3 read - rename StampDeploymentID -> StampDeploymentIdForFirstVersion (only used for the first version's id) - inline RecordDeploymentHistoryEnv into RecordsDeploymentHistory (single predicate) - unbind: drop the config-based phase check; DeploymentBundle.Unbind opens without recording, so the state marker (not the config flag) refuses a recorded deployment - restore forward-compat read of featureStateVersion=3: this CLI still writes v2, but reads a v3 state with no features and refuses one that carries a feature Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 6 +++--- bundle/direct/dstate/migrate.go | 18 ++++++++++++++++++ bundle/direct/dstate/state.go | 16 +++++++++++++--- bundle/env/dms.go | 17 +++++++---------- bundle/env/dms_test.go | 4 ++-- bundle/phases/bind.go | 8 +++----- bundle/phases/deploy.go | 2 +- 7 files changed, 47 insertions(+), 24 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 4c1c4f4e3a8..3b6b389bc57 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -64,7 +64,7 @@ func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan. // InitForApply initializes the DeploymentBundle for applying a pre-computed plan: it parses the // plan's entries (loaded as JSON when the plan came from a file) into the typed state cache the // apply reads. A first deployment's id does not exist until the deploy creates it after approval; -// StampDeploymentID fills it in then. StateDB must already be open for write before calling this. +// StampDeploymentIdForFirstVersion fills it in then. StateDB must already be open for write before calling this. func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { b.StateDB.AssertOpenedForWrite() @@ -118,11 +118,11 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. return nil } -// StampDeploymentID fills deploymentID into the plan's jobs and pipelines that still lack one, in +// StampDeploymentIdForFirstVersion fills deploymentID into the plan's jobs and pipelines that still lack one, in // both the state cache the apply reads and the plan JSON. It is called after approval creates the // deployment, for a first deploy whose id did not exist at plan time; the version and any // pre-existing id were stamped then. -func (b *DeploymentBundle) StampDeploymentID(deploymentID string) error { +func (b *DeploymentBundle) StampDeploymentIdForFirstVersion(deploymentID string) error { for resourceKey, entry := range b.Plan.Plan { if entry.NewState == nil || len(entry.NewState.Value) == 0 { continue diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index dccb06686a5..6a891900c36 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -3,6 +3,8 @@ package dstate import ( "encoding/json" "fmt" + "slices" + "strings" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/structs/structpath" @@ -13,6 +15,22 @@ import ( // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { + // A featureStateVersion state carries a feature list this CLI does not write (see the + // featureStateVersion doc comment). With no features it is equivalent to currentStateVersion, so + // accept it and leave the on-disk version as-is; with any feature it depends on capabilities this + // CLI lacks, so refuse it and tell the user to upgrade. + if db.StateVersion == featureStateVersion { + if len(db.Features) == 0 { + return nil + } + features := make([]string, 0, len(db.Features)) + for name := range db.Features { + features = append(features, name) + } + slices.Sort(features) + return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) + } + if db.StateVersion == currentStateVersion { return nil } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 2e410ee25b8..a248f8328fb 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -33,9 +33,19 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // supportedStateVersion is the highest schema version this CLI can read: the - // version it writes. A state newer than this is rejected as too new. - supportedStateVersion = currentStateVersion + // featureStateVersion is the schema version a future CLI will write once it records feature flags + // in the state version itself. This CLI does not write it, but reads such states (see + // migrateState): featureStateVersion with no features is equivalent to currentStateVersion and is + // accepted as-is (the on-disk version is left at featureStateVersion, not flipped down); + // featureStateVersion with any feature depends on capabilities this CLI lacks, so it is refused. + // Forward-compat scaffolding so a later release can write featureStateVersion + features without + // this CLI rejecting them outright. Always 3. + featureStateVersion = 3 + + // supportedStateVersion is the highest schema version this CLI can read. Normally equal to + // currentStateVersion - the version it reads is the version it writes - but this CLI can also + // read (never write) featureStateVersion. A state newer than this is rejected as too new. + supportedStateVersion = featureStateVersion ) // featureRecordDeploymentHistory marks a state whose resources are also recorded with the diff --git a/bundle/env/dms.go b/bundle/env/dms.go index d7258a76867..66779aa3fe4 100644 --- a/bundle/env/dms.go +++ b/bundle/env/dms.go @@ -7,16 +7,13 @@ import "context" // Deliberately undocumented. const RecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY" -// RecordDeploymentHistoryEnv reports whether the environment turns on deployment history -// recording. Only "true" turns it on: anything else - including a typo like "TRUE" or "yes" - -// leaves the feature off rather than silently enabling it. -func RecordDeploymentHistoryEnv(ctx context.Context) bool { +// RecordsDeploymentHistory reports whether recording is on, from config or env var. Single predicate +// for all recording code paths, keeping them in sync. The env var only counts as "true": anything +// else - a typo like "TRUE" or "yes" - leaves recording off rather than silently enabling it. +func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { + if configured { + return true + } value, _ := get(ctx, []string{RecordDeploymentHistoryVariable}) return value == "true" } - -// RecordsDeploymentHistory reports whether recording is on, from config or env var. -// Single predicate for all recording code paths; keeps them in sync. -func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { - return configured || RecordDeploymentHistoryEnv(ctx) -} diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go index 15bfe0e63f0..cfd91c0007a 100644 --- a/bundle/env/dms_test.go +++ b/bundle/env/dms_test.go @@ -22,12 +22,12 @@ func TestRecordDeploymentHistoryEnv(t *testing.T) { {"yes", false}, } { ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, tc.value) - assert.Equal(t, tc.want, RecordDeploymentHistoryEnv(ctx), "value %q", tc.value) + assert.Equal(t, tc.want, RecordsDeploymentHistory(ctx, false), "value %q", tc.value) } } func TestRecordDeploymentHistoryEnvUnset(t *testing.T) { - assert.False(t, RecordDeploymentHistoryEnv(t.Context())) + assert.False(t, RecordsDeploymentHistory(t.Context(), false)) } func TestRecordsDeploymentHistory(t *testing.T) { diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index eb141ea59b7..fc36122d1d8 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -135,11 +135,9 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r }() if engine.IsDirect() { - if b.ConfiguresDeploymentHistory(ctx) { - logdiag.LogError(ctx, errors.New("unbind is not supported for a bundle that records deployment history")) - return - } - + // A recorded deployment cannot be unbound: DeploymentBundle.Unbind opens the state without + // recording, so the state guard refuses it when the marker is set. State is the source of + // truth, so a config flag that disagrees with the marker cannot slip an unbind through. groupName, ok := terraform.TerraformToGroupName[tfResourceType] if !ok { groupName = tfResourceType diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 9ffe6eb3b44..b7dd3ea44d1 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -339,7 +339,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } if existingID == "" { deploymentID, _ := deploymentAndNextVersion(b) - if err := b.DeploymentBundle.StampDeploymentID(deploymentID); err != nil { + if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(deploymentID); err != nil { logdiag.LogError(ctx, err) return } From c0bc24dc908c47337b1d71dbe3a7444883c9adf0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:05:11 +0000 Subject: [PATCH 015/164] acceptance/dms: record plan JSON, list-resources, and raw recorded state Extend declined-deploy, depends-on and deployment-metadata-change to capture the serialized 'bundle plan -o json' (now carrying storage_backend), the DMS list-resources view, and the raw recorded state. deployment-metadata-change pins state_path, which print_state.py cannot resolve, so it relies on list-resources. Co-authored-by: Isaac --- .../bundle/dms/declined-deploy/output.txt | 55 ++++++ acceptance/bundle/dms/declined-deploy/script | 4 + acceptance/bundle/dms/depends-on/output.txt | 156 ++++++++++++++++++ acceptance/bundle/dms/depends-on/script | 6 + .../dms/deployment-metadata-change/output.txt | 104 ++++++++++++ .../dms/deployment-metadata-change/script | 13 ++ 6 files changed, 338 insertions(+) diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index ef8a2cb048b..d0ce19a2389 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -59,6 +59,61 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1 +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_declined_deploy_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "name": "dms_declined_deploy_schema_[UNIQUE_NAME]" + } + } + } +} + === A destructive change without --auto-approve is declined: this console cannot prompt >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index df539b83b29..cc69f672e63 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -4,6 +4,10 @@ title "Deploy a schema, so the deployment and its first version exist" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" +trace $CLI bundle plan -o json | nostamp.py +trace print_state.py + title "A destructive change without --auto-approve is declined: this console cannot prompt" # Changing the catalog recreates the schema, which needs approval. # Unity Catalog has no catalog called "other", so make one to move the schema into. diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index b36e5cfb154..4fb3a8caab5 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -37,6 +37,105 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.child": { + "depends_on": [ + { + "node": "resources.jobs.parent", + "label": "${resources.jobs.parent.id}" + } + ], + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json" + }, + "description": "depends on [NUMID]", + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "child", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.parent": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "parent", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + === The service lists both jobs, each recorded at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ @@ -52,6 +151,63 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } ] +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.child": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "description": "depends on [NUMID]", + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "child", + "queue": { + "enabled": true + } + }, + "depends_on": [ + { + "node": "resources.jobs.parent", + "label": "${resources.jobs.parent.id}" + } + ] + }, + "resources.jobs.parent": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "parent", + "queue": { + "enabled": true + } + } + } + } +} + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.child diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index dd168459b9a..022f8e3dd11 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -4,9 +4,15 @@ title "Deploy a job that references another: depends_on is recorded alongside th trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + title "The service lists both jobs, each recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-depends-on-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +title "The raw recorded state the service holds for this deployment" +trace print_state.py + trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 0b0798eee7f..3422dde62d7 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -59,6 +59,50 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1 +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_metadata_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_metadata_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + === Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed >>> update_file.py databricks.yml dms-metadata-before dms-metadata-after @@ -110,6 +154,56 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged } } +=== The metadata changed but the schema did not: still one deployment, recorded at version 1 +>>> [CLI] bundle plan -t prod -o json +Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed + +A common practice is to use a username or principal name in this path, i.e. use + + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_metadata_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_metadata_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + === Deploy again without touching anything: the record already says this, so no deployment update >>> [CLI] bundle deploy -t prod Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed @@ -142,3 +236,13 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged "completion_reason": "VERSION_COMPLETE_SUCCESS" } } + +=== The no-op redeploy leaves the schema at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script index 03f0c593999..45de9533cab 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/script +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -4,11 +4,24 @@ title "Deploy: the deployment is created carrying this bundle's metadata" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle +title "The plan records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" +trace $CLI bundle plan -o json | nostamp.py +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-metadata-pinned-${UNIQUE_NAME}/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + title "Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed" trace update_file.py databricks.yml dms-metadata-before dms-metadata-after trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle +title "The metadata changed but the schema did not: still one deployment, recorded at version 1" +trace $CLI bundle plan -t prod -o json | nostamp.py +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + title "Deploy again without touching anything: the record already says this, so no deployment update" trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle + +title "The no-op redeploy leaves the schema at version 1" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +rm -f out.requests.txt From eaa31f4dc21b55211fe8e7275bf826802b60849c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:10:45 +0000 Subject: [PATCH 016/164] acceptance/dms: record plan JSON, list-resources, and raw recorded state across the suite Extend no-resources, multiple-resources, record, successful-recreate, no-drift, stale-plan, emptied-resource and provenance to capture the serialized 'bundle plan -o json' (now carrying storage_backend), the DMS list-resources view, and the raw recorded state at each steady-state point. no-drift shows only the updated resource bumping to version 2; successful-recreate shows the recreate at version 2; provenance masks the git commit. Co-authored-by: Isaac --- .../bundle/dms/emptied-resource/output.txt | 56 ++ acceptance/bundle/dms/emptied-resource/script | 6 + .../bundle/dms/multiple-resources/output.txt | 692 ++++++++++++++++++ .../bundle/dms/multiple-resources/script | 22 + acceptance/bundle/dms/no-drift/output.txt | 276 +++++++ acceptance/bundle/dms/no-drift/script | 14 +- acceptance/bundle/dms/no-resources/output.txt | 46 +- acceptance/bundle/dms/no-resources/script | 22 +- acceptance/bundle/dms/provenance/output.txt | 100 +++ acceptance/bundle/dms/provenance/script | 9 + acceptance/bundle/dms/record/output.txt | 94 +++ acceptance/bundle/dms/record/script | 11 + acceptance/bundle/dms/stale-plan/output.txt | 66 ++ acceptance/bundle/dms/stale-plan/script | 11 +- .../bundle/dms/successful-recreate/output.txt | 136 ++++ .../bundle/dms/successful-recreate/script | 20 + 16 files changed, 1571 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 7d0ad36a596..297b8e441db 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -130,6 +130,41 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_emptied_resource_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_emptied_resource_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][1], + "updated_by": "[USERNAME]" + } + } + } +} + === The grants node emptied out, so the service no longer lists it - only the schema remains >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ @@ -140,6 +175,27 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } ] +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_emptied_resource_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "name": "dms_emptied_resource_[UNIQUE_NAME]" + } + } + } +} + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.foo diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 1e0606a31c0..6e5af06cc31 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -16,9 +16,15 @@ trace print_requests.py --dms //api/2.0/bundle title "Plan again: reading state back from the service works and reports no work" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" +title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + title "The grants node emptied out, so the service no longer lists it - only the schema remains" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-emptied-resource-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +title "The raw recorded state the service holds for this deployment" +trace print_state.py + trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index ecc2092225a..4a737c37b6e 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -17,6 +17,352 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.three", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.two", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.five": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.four": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.one": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][2], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.three": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][3], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.two": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][4], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + +=== The service lists all five resources at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.five" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.four" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.one" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.three" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.two" + } +] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.five": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.four": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.one": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.three": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + } + } + } + } +} + === Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... @@ -26,3 +372,349 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged >>> print_requests.py --dms //api/2.0/bundle --sort --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.five": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.four": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.one": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][2], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.three": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][3], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.two": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][4], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + +=== The service still lists all five resources, now at version 2 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.five" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.four" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.one" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.three" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.two" + } +] + +=== The raw recorded state after redeploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.five": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.four": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.one": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.three": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + } + } + } + } +} diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 584423a4d5e..78ab3028390 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -4,6 +4,28 @@ title "Deploy several resources: operations are uploaded from background workers trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline +title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists all five resources at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py + title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --sort --oneline + +title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service still lists all five resources, now at version 2" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after redeploy" +trace print_state.py + +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 952b5b2731e..87d254cf2e2 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -27,6 +27,143 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "latest_version_id": "1" } +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.pipelines.bar": { + "action": "skip", + "remote_state": { + "catalog": "main", + "channel": "CURRENT", + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "id": "[UUID]", + "last_modified": [UNIX_TIME_MILLIS][1], + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "schema": "default", + "serverless": true, + "state": "IDLE" + } + } + } +} + +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.pipelines.bar": { + "__id__": "[UUID]", + "state": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } + } + } +} + === Plan without touching anything: the deployment stamp must not show as drift >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -133,6 +270,145 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "latest_version_id": "2" } +=== The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA_SERVICE as backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.pipelines.bar": { + "action": "skip", + "remote_state": { + "catalog": "main", + "channel": "CURRENT", + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + }, + "development": true, + "edition": "ADVANCED", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "id": "[UUID]", + "last_modified": [UNIX_TIME_MILLIS][1], + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "schema": "default", + "serverless": true, + "state": "IDLE" + } + } + } +} + +=== The raw recorded state the service holds after the update +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.pipelines.bar": { + "__id__": "[UUID]", + "state": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "2" + }, + "development": true, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } + } + } +} + === Plan again: no residual drift, and the version stamps are ignored >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index dfcb0b55ef5..55bff445646 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -7,6 +7,12 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle summary -o json | jq .bundle.deployment.history +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The raw recorded state the service holds for this deployment" +trace print_state.py + # Only deploy stamps deployment.deployment_id. Plan sees it in state but not in config, # so it must ignore it rather than report spurious changes. title "Plan without touching anything: the deployment stamp must not show as drift" @@ -20,9 +26,15 @@ trace print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle summary -o json | jq .bundle.deployment.history +title "The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA_SERVICE as backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The raw recorded state the service holds after the update" +trace print_state.py + title "Plan again: no residual drift, and the version stamps are ignored" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" -# The list-resources and final plan above record requests we do not assert on; drop them so +# The list-resources and plan/state captures above record requests we do not assert on; drop them so # out.requests.txt does not become a nondeterministic golden. rm -f out.requests.txt diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 5c80f26a912..f75ba46e2fa 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -44,6 +44,22 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state/resources.deployment.json" } +=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": {} +} + +=== The service lists no resources for the deployment +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py + === Redeploy: the deployment is resolved from that node, so no second deployment is created >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files... @@ -59,6 +75,18 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -79,8 +107,18 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged } } -=== The service lists no resources for the deployment: nothing was recorded beyond the deployment itself +=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": {} +} + +=== The service still lists no resources >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json -[ - -] +[] + +=== The raw recorded state after redeploy +>>> print_state.py diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 81444e00ccf..001d1e835f5 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -5,12 +5,28 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' +title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists no resources for the deployment" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py + title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get -title "The service lists no resources for the deployment: nothing was recorded beyond the deployment itself" +title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service still lists no resources" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json -# Discard the get-status + list-resources reads above so out.requests.txt is not a golden. +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after redeploy" +trace print_state.py + rm -f out.requests.txt diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 2142dabb700..df026816182 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -51,6 +51,61 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] foo", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "tags": { + "dev": "[USERNAME]" + }, + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + === The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID] { @@ -66,3 +121,48 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev" } } + +=== The service lists the deployed job +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + } +] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py -t dev +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] foo", + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + } + } + } + } +} diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index 8b15822fa4b..1c6661c30d7 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -8,12 +8,21 @@ trace $CLI bundle deploy add_repl.py "$(git rev-parse HEAD)" COMMIT trace print_requests.py --dms //versions +title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance-${UNIQUE_NAME}/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') # MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API # path into C:/Program Files/Git/api/2.0/... before the CLI saw it. trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}" | jq '{target_name, deployment_mode, git_info, workspace_info}' +title "The service lists the deployed job" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py -t dev + # The deploy uploads files in a nondeterministic order; only the requests above are # asserted. rm -f out.requests.txt diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 7e63506bf79..7f62c55528d 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -82,3 +82,97 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "completion_reason": "VERSION_COMPLETE_SUCCESS" } } + +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 3, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + +=== The service lists the job, recorded at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + } +] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 3, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + } + } +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index c521416b158..5c565b792d1 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -14,3 +14,14 @@ trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${C trace $CLI bundle deploy trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' trace print_requests.py --get --dms //api/2.0/bundle + +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists the job, recorded at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index f39b3049697..420d5225dd2 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -1,6 +1,31 @@ === Save a plan, then deploy it: records version 1 >>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + } + } + } +} >>> [CLI] bundle deploy --plan tmp.plan.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files... @@ -8,6 +33,47 @@ Created jobs.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + } +] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + } + } +} + === A saved plan carries the deployment id and the versions it targets (last 1, next 2) >>> [CLI] bundle plan -o json { diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index 09724d7d545..8a49c5fcd4f 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -1,9 +1,16 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Save a plan, then deploy it: records version 1" -trace $CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle plan -o json | nostamp.py trace $CLI bundle deploy --plan tmp.plan.json +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-plan-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py + title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" trace $CLI bundle plan -o json | jq '{deployment_id, last_version_id, next_version_id}' @@ -13,4 +20,4 @@ trace musterr $CLI bundle deploy --plan tmp.plan.json title "Only the first deploy recorded a version; the rejected replay created none" trace print_requests.py --dms //api/2.0/bundle --oneline -rm -f tmp.plan.json +rm -f tmp.plan.json out.requests.txt diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 4d497d81f2b..1fd34e383e4 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -59,6 +59,74 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "v1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +=== The service lists the schema, recorded at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + } + } + } +} + === Recreate writes state twice - the entry is dropped, then the new resource is saved >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] @@ -124,6 +192,74 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } } +=== The plan now shows the recreate action +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "dms_other_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "comment": "v1", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][1], + "updated_by": "[USERNAME]" + } + } + } +} + +=== The service lists the schema at version 2 with recreate action +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_RECREATE", + "last_version_id": "2", + "resource_key": "schemas.foo" + } +] + +=== The raw recorded state after recreate +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "dms_other_[UNIQUE_NAME]", + "comment": "v1", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + } + } + } +} + === Destroy: the delete is recorded with the id and no state >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index 43050cc7c8f..6a8e0fe8914 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -4,6 +4,16 @@ title "Deploy: the state write records the resource" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists the schema, recorded at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-successful-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py + title "Recreate writes state twice - the entry is dropped, then the new resource is saved" # One operation per resource per version; both writes land on the same one. # The drop opens it IN_PROGRESS, the save patches it to SUCCEEDED. @@ -15,6 +25,16 @@ trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_ trace $CLI bundle deploy --auto-approve trace print_requests.py --dms //api/2.0/bundle +title "The plan now shows the recreate action" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists the schema at version 2 with recreate action" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after recreate" +trace print_state.py + title "Destroy: the delete is recorded with the id and no state" trace $CLI bundle destroy --auto-approve trace print_requests.py --dms //api/2.0/bundle +rm -f out.requests.txt From db6a166c6ec8f2f1b15b981db6cd043390697abc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:18:50 +0000 Subject: [PATCH 017/164] bundle/dms: guard StorageBackend() gate on direct engine; bump future-version golden - deploy.go: StorageBackend() asserts the state is open, but only the direct engine opens it, so gate the DMS deployment-create block on stateEngine.IsDirect() first. Fixes a panic ("DeploymentState must be opened first") on terraform deploys, which the direct-only dms tests did not exercise. - future_version: supported state version is now 3 (v3 is read-forward-compatible), so the too-new message reports 3. Co-authored-by: Isaac --- acceptance/bundle/state/future_version/output.txt | 2 +- bundle/phases/deploy.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 7cf98129ee9..0a16971f472 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,3 @@ -state version 999 is newer than supported version 2; upgrade the CLI +state version 999 is newer than supported version 3; upgrade the CLI Exit code: 1 diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index b7dd3ea44d1..88245e107e1 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -331,7 +331,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. - if b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + // IsDirect first: StorageBackend asserts the state is open, and only the direct engine opens it. + if stateEngine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { existingID, _ := deploymentAndNextVersion(b) createOrUpdateDeployment(ctx, b, dmsDeployment) if logdiag.HasError(ctx) { From 0afcb1354c3bc1ccc687a8397fcd7025fcea416a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:41:34 +0000 Subject: [PATCH 018/164] bundle/dms: StateDB owns plan lineage; unbind-specific message; fail version on recording error - C11: StateDB holds DeploymentID/LatestVersionID (set at Open from the service); CalculatePlan stamps the plan's lineage from StateDB instead of the bundle config tree (config.History stays only for 'bundle summary'). Behavior-neutral. - unbind: Open returns a sentinel ErrUnsettingRecording; DeploymentBundle.Unbind maps it to an unbind-specific message instead of the deploy/destroy-flavored guard text. - CompleteVersion now completes as Failure when the drained buffer holds a recording error, even if the caller passed success=true, so a destroy whose uploads failed keeps its deployment record. Co-authored-by: Isaac --- bundle/direct/bind.go | 4 ++++ bundle/direct/bundle_plan.go | 21 ++++++++++----------- bundle/direct/dstate/state.go | 27 +++++++++++++++++++++++---- cmd/bundle/utils/process.go | 3 +++ 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index a65cacd7eb4..4605f34be66 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -2,6 +2,7 @@ package direct import ( "context" + "errors" "fmt" "os" "strings" @@ -219,6 +220,9 @@ func (result *BindResult) Cancel() { func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") if err != nil { + if errors.Is(err, dstate.ErrUnsettingRecording) { + return errors.New("unbind is not supported for a bundle that records deployment history") + } return err } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 3b6b389bc57..775103e6ad9 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -168,24 +168,23 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, fmt.Errorf("reading config: %w", err) } - // Record the DMS deployment and version this plan targets. A saved plan carries them so - // deploy --plan can reject a plan the deployment has moved on from, and pass last_version_id - // as previous_version_id. History is set only while recording, so other plans leave these empty. - // configRoot is nil for destroy, which records its version separately. // The plan records where its state lives so deploy --plan can reject a plan whose target has - // since switched backends. Set from StateDB (the source of truth) so a first deploy - which has - // no History yet - still carries it. + // since switched backends. StateDB is the source of truth, so a first deploy - which has no + // recorded deployment yet - still carries the backend. if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { plan.StorageBackend = string(dstate.StorageBackendDeploymentMetadataService) } - if configRoot != nil && configRoot.Bundle.Deployment.History != nil { - h := configRoot.Bundle.Deployment.History - next, err := dms.NextVersion(h.LatestVersionID) + // Record the DMS deployment and version this plan targets, from StateDB (not the config tree). + // A saved plan carries them so deploy --plan can reject a plan the deployment has moved on from, + // and passes last_version_id as previous_version_id. Empty for a first deploy, which has no + // recorded deployment yet - the deploy phase stamps the created id. + if b.StateDB.DeploymentID != "" { + next, err := dms.NextVersion(b.StateDB.LatestVersionID) if err != nil { return nil, fmt.Errorf("computing next deployment version: %w", err) } - plan.DeploymentId = h.DeploymentID - plan.LastVersionId = h.LatestVersionID + plan.DeploymentId = b.StateDB.DeploymentID + plan.LastVersionId = b.StateDB.LatestVersionID plan.NextVersionId = strconv.FormatInt(next, 10) } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index a248f8328fb..17c4eb9d4e7 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -100,6 +100,13 @@ func checkStateFeatures(features map[string]struct{}) error { // The caller should delete the stale WAL and proceed normally. var errStaleWAL = errors.New("stale WAL") +// ErrUnsettingRecording is returned by Open when a recorded state is opened without recording - the +// config turned the feature off, or an operation that never records (unbind) reached it. Callers +// present an operation-appropriate message via errors.Is. +var ErrUnsettingRecording = errors.New(`unsetting experimental.record_deployment_history is not supported + +This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) + type DeploymentState struct { Path string Data Database @@ -130,6 +137,12 @@ type DeploymentState struct { // versionCompleted makes CompleteVersion a no-op after the first call, so a deferred safety-net // completion after an explicit one does nothing. versionCompleted bool + + // DeploymentID and LatestVersionID are the recorded deployment's id and its most recent version + // at Open time, from the service. CalculatePlan reads them (rather than the bundle config tree) + // to stamp the plan's lineage. Both empty for a first deploy, whose id the deploy phase creates. + DeploymentID string + LatestVersionID string } type Header struct { @@ -204,7 +217,8 @@ func (db *DeploymentState) RecordingError() error { // survive Finalize's reset and asserts nothing, so it is safe to call after the state is closed. func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (bool, error) { db.mu.Lock() - if db.operationBuffer == nil || db.versionCompleted { + buf := db.operationBuffer + if buf == nil || db.versionCompleted { db.mu.Unlock() return false, nil } @@ -212,6 +226,13 @@ func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (b deploymentID, versionID, client := db.deploymentID, db.versionID, db.dmsClient db.mu.Unlock() + // A recording failure fails the version even when the caller counted the deploy a success: the + // service does not then hold everything the WAL does. Finalize already drained and surfaced it; + // this reads the drained buffer's error so a destroy whose uploads failed keeps its record. + if buf.Err() != nil { + success = false + } + reason := bundledeployments.VersionCompleteVersionCompleteSuccess if !success { reason = bundledeployments.VersionCompleteVersionCompleteFailure @@ -552,9 +573,7 @@ To record this bundle's history, start it over as a new deployment: 2. run "databricks bundle destroy" to delete the existing resources 3. add experimental.record_deployment_history back and deploy again`) case !recording && recorded: - return errors.New(`unsetting experimental.record_deployment_history is not supported - -This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) + return ErrUnsettingRecording } db.storageBackend = StorageBackendWorkspaceFilesystem diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 74909c20997..dc11d95306e 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -293,6 +293,9 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } + // StateDB owns the plan's lineage: CalculatePlan reads these, not the config tree. + b.DeploymentBundle.StateDB.DeploymentID = dmsDeploymentID + b.DeploymentBundle.StateDB.LatestVersionID = lastVersionID } else { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) From 0f92c144c4350e7f4556eb5daed33f33b6a5261d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:46:51 +0000 Subject: [PATCH 019/164] acceptance/dms: drop dedicated readplan test; cover deploy --plan via the readplan option The dedicated readplan test duplicated stale-plan, which already deploys from a saved raw plan under recording. Delete it and add the standard READPLAN matrix to stale-plan so its initial deploy runs both from the saved plan and recomputed - the deploy is untraced so the variant-dependent command stays out of the golden. Recorded operations still carry the deployment_id/version_id lineage (asserted in depends-on and others). Co-authored-by: Isaac --- .../bundle/dms/readplan/databricks.yml.tmpl | 19 ---------------- acceptance/bundle/dms/readplan/out.test.toml | 3 --- acceptance/bundle/dms/readplan/output.txt | 22 ------------------- acceptance/bundle/dms/readplan/script | 13 ----------- acceptance/bundle/dms/readplan/transform.py | 1 - .../bundle/dms/stale-plan/out.test.toml | 1 + acceptance/bundle/dms/stale-plan/output.txt | 4 +--- acceptance/bundle/dms/stale-plan/script | 4 +++- acceptance/bundle/dms/stale-plan/test.toml | 4 ++++ 9 files changed, 9 insertions(+), 62 deletions(-) delete mode 100644 acceptance/bundle/dms/readplan/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/readplan/out.test.toml delete mode 100644 acceptance/bundle/dms/readplan/output.txt delete mode 100644 acceptance/bundle/dms/readplan/script delete mode 100644 acceptance/bundle/dms/readplan/transform.py create mode 100644 acceptance/bundle/dms/stale-plan/test.toml diff --git a/acceptance/bundle/dms/readplan/databricks.yml.tmpl b/acceptance/bundle/dms/readplan/databricks.yml.tmpl deleted file mode 100644 index d11621ab1e5..00000000000 --- a/acceptance/bundle/dms/readplan/databricks.yml.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -bundle: - name: dms-readplan-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - foo: - name: foo - pipelines: - bar: - # Unique: the Pipelines API refuses a duplicate name across overlapping runs. - name: bar-$UNIQUE_NAME - catalog: main - schema: default - serverless: true - libraries: - - file: - path: ./transform.py diff --git a/acceptance/bundle/dms/readplan/out.test.toml b/acceptance/bundle/dms/readplan/out.test.toml deleted file mode 100644 index 23c07f70dca..00000000000 --- a/acceptance/bundle/dms/readplan/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/readplan/output.txt b/acceptance/bundle/dms/readplan/output.txt deleted file mode 100644 index 371855dbcc0..00000000000 --- a/acceptance/bundle/dms/readplan/output.txt +++ /dev/null @@ -1,22 +0,0 @@ - -=== Save a plan, then deploy from it: the deploy stamps the id and version onto the job and pipeline the plan predates ->>> [CLI] bundle plan -o json - ->>> [CLI] bundle deploy --plan tmp.plan.json -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files... -Created jobs.foo -Created pipelines.bar -Files: 7 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged - -=== Re-plan: no drift, the deployed job and pipeline carry the deployment stamp ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - -=== Both recorded operations carry deployment_id and version_id (lineage), not just kind ->>> print_requests.py --dms //api/2.0/bundle --sort --oneline -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.bar", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[UUID]", "sequence_id": "0", "state": "{\"state\":{\"catalog\":\"main\",\"channel\":\"CURRENT\",\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edition\":\"ADVANCED\",\"libraries\":[{\"file\":{\"path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files/transform.py\"}}],\"name\":\"bar-[UNIQUE_NAME]\",\"schema\":\"default\",\"serverless\":true}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-readplan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}, {"resource_key": "pipelines.bar", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/readplan/script b/acceptance/bundle/dms/readplan/script deleted file mode 100644 index 40bd110bf00..00000000000 --- a/acceptance/bundle/dms/readplan/script +++ /dev/null @@ -1,13 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "Save a plan, then deploy from it: the deploy stamps the id and version onto the job and pipeline the plan predates" -trace $CLI bundle plan -o json > tmp.plan.json -trace $CLI bundle deploy --plan tmp.plan.json - -title "Re-plan: no drift, the deployed job and pipeline carry the deployment stamp" -trace $CLI bundle plan - -title "Both recorded operations carry deployment_id and version_id (lineage), not just kind" -trace print_requests.py --dms //api/2.0/bundle --sort --oneline | contains.py deployment_id version_id - -rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/readplan/transform.py b/acceptance/bundle/dms/readplan/transform.py deleted file mode 100644 index c07007592a6..00000000000 --- a/acceptance/bundle/dms/readplan/transform.py +++ /dev/null @@ -1 +0,0 @@ -# Minimal source so the pipeline has a library, which the API requires. diff --git a/acceptance/bundle/dms/stale-plan/out.test.toml b/acceptance/bundle/dms/stale-plan/out.test.toml index 23c07f70dca..f7ae4fc905f 100644 --- a/acceptance/bundle/dms/stale-plan/out.test.toml +++ b/acceptance/bundle/dms/stale-plan/out.test.toml @@ -1,3 +1,4 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 420d5225dd2..3b89e18875a 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -26,11 +26,9 @@ } } } - ->>> [CLI] bundle deploy --plan tmp.plan.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files... Created jobs.foo -Files: 6 uploaded, 0 deleted +Files: 7 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index 8a49c5fcd4f..f50dae91193 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -3,7 +3,9 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Save a plan, then deploy it: records version 1" $CLI bundle plan -o json > tmp.plan.json trace $CLI bundle plan -o json | nostamp.py -trace $CLI bundle deploy --plan tmp.plan.json +# READPLAN deploys from the saved raw plan; without it the plan is recomputed. Untraced so the +# variant-dependent command line stays out of the golden (the deploy output is identical either way). +$CLI bundle deploy $(readplanarg tmp.plan.json) deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-plan-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' diff --git a/acceptance/bundle/dms/stale-plan/test.toml b/acceptance/bundle/dms/stale-plan/test.toml new file mode 100644 index 00000000000..4c51e8879d6 --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/test.toml @@ -0,0 +1,4 @@ +# Also deploy from the saved plan (not just compute it), so recorded deploy --plan is covered by the +# standard readplan option rather than a dedicated test. The saved plan is raw (not nostamp.py'd) so +# it carries the DMS fields the replay needs. +EnvMatrix.READPLAN = ["", "1"] From 5f108058d68416b1b570afd05a5cf1cfc79ec8d9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:56:34 +0000 Subject: [PATCH 020/164] bundle/dms: trim redundant comment on the cmdctx workspace-client bridge Co-authored-by: Isaac --- cmd/bundle/utils/process.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index dc11d95306e..f223abc82d4 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -284,8 +284,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } - // StateDB.Open builds the DMS client from the workspace client on the context. A bundle - // command resolves that client lazily through b, so it is not otherwise on the context. + // StateDB.Open builds the DMS client from the workspace client on the context. if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } From abd73573045604c03f76f4052e50aa96bed1f6db Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 14:06:18 +0000 Subject: [PATCH 021/164] acceptance/dms: record plan/list-resources/state for the failure tests Extend the failure-injection tests to show how DMS records a failed or refused deploy: failed-delete/recreate/update, failed-update-permissions, operation-upload-fails-once, record-failure, version-never-created (and a WORKSPACE_FILESYSTEM plan on existing-state's first deploy). Each captures the healthy recorded state and the recorded result of the failure. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 50 +++++ acceptance/bundle/dms/existing-state/script | 1 + .../bundle/dms/failed-delete/output.txt | 105 +++++++++ acceptance/bundle/dms/failed-delete/script | 18 +- .../bundle/dms/failed-recreate/output.txt | 138 +++++++++++- acceptance/bundle/dms/failed-recreate/script | 23 +- .../dms/failed-update-permissions/output.txt | 212 ++++++++++++++++++ .../dms/failed-update-permissions/script | 18 ++ .../bundle/dms/failed-update/output.txt | 161 ++++++++++++- acceptance/bundle/dms/failed-update/script | 23 +- .../operation-upload-fails-once/output.txt | 100 +++++++++ .../dms/operation-upload-fails-once/script | 18 ++ .../bundle/dms/record-failure/output.txt | 45 ++++ acceptance/bundle/dms/record-failure/script | 5 + .../dms/version-never-created/output.txt | 33 +++ .../bundle/dms/version-never-created/script | 4 + 16 files changed, 925 insertions(+), 29 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 9a7439eef8a..95352e1dcb0 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -6,6 +6,56 @@ Created jobs.one Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.jobs.one": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + >>> print_requests.py --dms //api/2.0/bundle --oneline >>> jq {state_version, features} .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index e0784bb406b..5ab20285e56 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -2,6 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" trace $CLI bundle deploy +trace $CLI bundle plan -o json | nostamp.py trace print_requests.py --dms //api/2.0/bundle --oneline trace jq '{state_version, features}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 701b2051c14..17d98e0a4a3 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -59,6 +59,100 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.stuck": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "stuck", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + +=== After deploy, the service lists the resource at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.stuck" + } +] + +=== The raw state after successful deploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.stuck": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "stuck", + "queue": { + "enabled": true + } + } + } + } +} + === A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone >>> fault.py POST /api/2.2/jobs/delete 400 0 1 INVALID_PARAMETER_VALUE @@ -117,6 +211,17 @@ API message: Fault injected by test. } } +=== After failure, the service shows the operation failed +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_DELETE", + "last_version_id": "2", + "resource_key": "jobs.stuck" + } +] + +=== The state shows the resource still exists but the operation is marked failed >>> print_state.py { "state_version": 2, diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index fff56460c21..ed544b27d7f 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -4,13 +4,29 @@ title "Deploy: the job is recorded" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "After deploy, the service lists the resource at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw state after successful deploy" +trace print_state.py + title "A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone" # The state entry survives too, so the next deploy still knows about the resource. trace fault.py "POST /api/2.2/jobs/delete" 400 0 1 INVALID_PARAMETER_VALUE trace musterr $CLI bundle destroy --auto-approve trace print_requests.py --dms //api/2.0/bundle + +title "After failure, the service shows the operation failed" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The state shows the resource still exists but the operation is marked failed" trace print_state.py # print_state.py asks the service for the resources while recording, so drain those reads instead # of leaving them behind for the harness to diff. -print_requests.py //api/2.0 > /dev/null +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 718b8108771..a85298826b2 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -6,6 +6,127 @@ Created schemas.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-failed-recreate-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_recreate_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "v1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_failed_recreate_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +=== After deploy, the service lists the resource at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + +=== The raw state after successful deploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_failed_recreate_schema_[UNIQUE_NAME]" + } + } + } +} + === A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] @@ -77,8 +198,21 @@ Files: 4 uploaded, 0 deleted } === The resource is not listed: state is what projects a resource, and the failed recreate left none ->>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources -{} +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[] + +=== The raw state after failed recreate shows no resource +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "features": { + "record_deployment_history": {} + }, + "state": {} +} === Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index e2d4a68f508..6c0b82a1a69 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -2,7 +2,17 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy the schema, so there is a resource to recreate" trace $CLI bundle deploy -rm -f out.requests.txt +trace print_requests.py --dms //api/2.0/bundle + +title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "After deploy, the service lists the resource at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw state after successful deploy" +trace print_state.py title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy" # Unity Catalog has no catalog called "other", so make one to move the schema into. @@ -17,12 +27,11 @@ trace musterr $CLI bundle deploy --auto-approve trace print_requests.py --dms //api/2.0/bundle title "The resource is not listed: state is what projects a resource, and the failed recreate left none" -# The deployment ID is the workspace node's ID; read it back the way the CLI does. -# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') -# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API -# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. -trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw state after failed recreate shows no resource" +trace print_state.py title "Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists" rm -rf .databricks diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 080f4b6c8be..ac97f5ce901 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -7,6 +7,149 @@ Created jobs.foo.permissions Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +=== The plan records DEPLOYMENT_METADATA_SERVICE backend and the job +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.foo.permissions": { + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.id}" + } + ], + "action": "skip", + "remote_state": { + "object_id": "/jobs/[NUMID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } + } + } +} + +=== The service lists the job at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo.permissions" + } +] + +=== The raw recorded state after deploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.foo.permissions": { + "__id__": "/jobs/[NUMID]", + "state": { + "object_id": "/jobs/[NUMID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.id}" + } + ] + } + } +} + === A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not >>> update_file.py databricks.yml CAN_VIEW CAN_MANAGE @@ -23,6 +166,75 @@ API message: Fault injected by test. Files: 3 uploaded, 0 deleted +=== The service lists the job with the failure +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", + "last_version_id": "2", + "resource_key": "jobs.foo.permissions" + } +] + +=== The raw recorded state after the failure +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.foo.permissions": { + "__id__": "/jobs/[NUMID]", + "state": { + "object_id": "/jobs/[NUMID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.id}" + } + ] + } + } +} + >>> print_requests.py --dms //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo.permissions", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED)", "resource_id": "/jobs/[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", "status": "OPERATION_STATUS_FAILED"}} diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index 56141a30f4c..6883d8f3775 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -4,9 +4,27 @@ title "Deploy the job and its permissions" trace $CLI bundle deploy rm -f out.requests.txt +title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the job" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists the job at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-permissions-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after deploy" +trace print_state.py + title "A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not" trace update_file.py databricks.yml "CAN_VIEW" "CAN_MANAGE" # A 403 is what the API returns when the caller cannot change a job's owner. trace fault.py "PUT /api/2.0/permissions/jobs/*" 403 0 1 PERMISSION_DENIED trace musterr $CLI bundle deploy --auto-approve + +title "The service lists the job with the failure" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-permissions-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after the failure" +trace print_state.py + trace print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index 125c16608b0..2cb6c855d5f 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -6,6 +6,127 @@ Created schemas.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-failed-update-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "v1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_failed_update_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +=== After deploy, the service lists the resource at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + +=== The raw state after successful deploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_failed_update_schema_[UNIQUE_NAME]" + } + } + } +} + === An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it >>> update_file.py databricks.yml comment: v1 comment: v2 @@ -64,19 +185,35 @@ Files: 3 uploaded, 0 deleted } === The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one ->>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", + "last_version_id": "2", + "resource_key": "schemas.foo" + } +] + +=== The raw state after failed update shows the previous version +>>> print_state.py { - "resources": [ - { - "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", - "last_version_id": "2", - "name": "deployments/[NUMID]/resources/schemas.foo", - "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "resource_key": "schemas.foo", - "resource_type": "", - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}" - } - ] + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_failed_update_schema_[UNIQUE_NAME]" + } + } + } } === Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script index 91640bd7631..7d223af223b 100644 --- a/acceptance/bundle/dms/failed-update/script +++ b/acceptance/bundle/dms/failed-update/script @@ -2,7 +2,17 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy the schema, so there is an existing resource to update" trace $CLI bundle deploy -rm -f out.requests.txt +trace print_requests.py --dms //api/2.0/bundle + +title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "After deploy, the service lists the resource at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw state after successful deploy" +trace print_state.py title "An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it" trace update_file.py databricks.yml "comment: v1" "comment: v2" @@ -13,12 +23,11 @@ trace musterr $CLI bundle deploy --auto-approve trace print_requests.py --dms //api/2.0/bundle title "The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one" -# The deployment ID is the workspace node's ID; read it back the way the CLI does. -# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') -# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API -# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. -trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw state after failed update shows the previous version" +trace print_state.py title "Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys" rm -rf .databricks diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index abdeedeb959..8e74957520c 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -6,6 +6,74 @@ Created schemas.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +=== The plan records DEPLOYMENT_METADATA_SERVICE backend and the schema +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "v1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_upload_fails_once_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +=== The service lists the schema at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + +=== The raw recorded state after deploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_upload_fails_once_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + } + } + } +} + === A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one >>> fault.py PATCH /api/2.0/bundle/* 500 0 1 @@ -25,6 +93,38 @@ API message: Fault injected by test. Files: 4 uploaded, 0 deleted +=== The service lists the schema with the failure +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_RECREATE", + "last_version_id": "2", + "resource_key": "schemas.foo" + } +] + +=== The raw recorded state after the failure +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "dms_other_[UNIQUE_NAME]", + "comment": "v1", + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + } + } + } +} + >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index 5b609b1b8ee..d69268d8f87 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -4,6 +4,16 @@ title "Deploy so there is a recorded resource to recreate" trace $CLI bundle deploy rm -f out.requests.txt +title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the schema" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists the schema at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after deploy" +trace print_state.py + title "A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one" trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 # Unity Catalog has no catalog called "other", so make one to move the schema into. @@ -12,4 +22,12 @@ $CLI catalogs create "$other_catalog" &> LOG.catalog trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" trace musterr $CLI bundle deploy --auto-approve + +title "The service lists the schema with the failure" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after the failure" +trace print_state.py + trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 819004844d7..a08dd0df054 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -11,6 +11,46 @@ API message: Node type nonexistent-node-type is not supported. Supported node ty Files: 6 uploaded, 0 deleted +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.doomed": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "doomed", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "node_type_id": "nonexistent-node-type", + "num_workers": 1, + "spark_version": "15.4.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/nonexistent" + }, + "task_key": "doomed" + } + ] + } + } + } + } +} + >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", @@ -66,6 +106,11 @@ Files: 6 uploaded, 0 deleted >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources {} +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[] + +>>> print_state.py + === Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged >>> [CLI] bundle plan create jobs.doomed diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 8b629dfd6be..8fc543ff20d 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -2,6 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" trace musterr $CLI bundle deploy +trace $CLI bundle plan -o json | nostamp.py trace print_requests.py --dms //api/2.0/bundle title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" @@ -12,6 +13,10 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ # path into C:/Program Files/Git/api/2.0/... before the CLI saw it. trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +trace print_state.py + title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" rm -rf .databricks trace $CLI bundle plan diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 8a7e5fe8506..db21f2a000a 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -11,6 +11,33 @@ API message: Internal error Files: 5 uploaded, 0 deleted +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + } + } + } +} + >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json { "object_type": "BUNDLE_DEPLOYMENT" @@ -19,6 +46,11 @@ Files: 5 uploaded, 0 deleted >>> print_requests.py --dms //api/2.0/bundle --get --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-version-never-created", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[] === The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment >>> musterr [CLI] bundle deploy @@ -33,6 +65,7 @@ API message: Internal error Files: 2 uploaded, 0 deleted >>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script index 58178feaf53..aca7b030713 100644 --- a/acceptance/bundle/dms/version-never-created/script +++ b/acceptance/bundle/dms/version-never-created/script @@ -1,8 +1,12 @@ title "The first version fails, so no deployment record exists - only the node naming its ID" trace musterr $CLI bundle deploy +trace $CLI bundle plan -o json | nostamp.py trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' trace print_requests.py --dms //api/2.0/bundle --get --oneline +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" trace musterr $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get --oneline From b4d23d20072be2329e499e995a2999c426c7e87b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 14:10:55 +0000 Subject: [PATCH 022/164] bundle/dms: review fixes - naming, single lineage fields, user-facing wording - rename StartRecording -> InitializeOperationBuffer (it opens the buffer; kept separate from Open because a first deploy's id does not exist until after approval) - collapse the duplicated deployment/version fields to one public pair (DeploymentID/LatestVersionID); InitializeOperationBuffer records the first-deploy id, CompleteVersion derives the version from it - RecordingError is now a no-op unless the backend is the deployment metadata service, keeping the non-recording path cleanly separate (like RecordFailure) - stop surfacing the internal "deployment metadata service" name in user-facing errors; say "deployment history feature enabled" / "the deployment history service" - drop the "in place of nil client checks" comment and restore the original featureStateVersion wording (minimal C12 diff) Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 6 +- bundle/direct/bundle_apply.go | 2 +- bundle/direct/dstate/state.go | 79 +++++++++++-------- bundle/phases/dms.go | 2 +- libs/dms/client.go | 2 +- libs/dms/resources.go | 2 +- 6 files changed, 53 insertions(+), 40 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 95352e1dcb0..0d35577a43c 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -66,7 +66,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time >>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources +Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: 1. remove experimental.record_deployment_history from your bundle configuration @@ -78,7 +78,7 @@ To record this bundle's history, start it over as a new deployment: === The check is against the remote state, not the local cache: wiping .databricks still refuses, because the deploy pulls the state back from the workspace >>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources +Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: 1. remove experimental.record_deployment_history from your bundle configuration @@ -88,7 +88,7 @@ To record this bundle's history, start it over as a new deployment: === A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands >>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources +Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: 1. remove experimental.record_deployment_history from your bundle configuration diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 8c2de1f29ee..74bfa226cbf 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -43,7 +43,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } - // The state DB records every write with DMS from here on (via the buffer StartRecording opened + // The state DB records every write with DMS from here on (via the buffer InitializeOperationBuffer opened // in the deploy phase), so the service mirrors the WAL. Writes go out on one background // goroutine, off the apply path, and are drained below once every worker has finished recording. g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 17c4eb9d4e7..ef67ccfc559 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -33,18 +33,27 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // featureStateVersion is the schema version a future CLI will write once it records feature flags - // in the state version itself. This CLI does not write it, but reads such states (see - // migrateState): featureStateVersion with no features is equivalent to currentStateVersion and is - // accepted as-is (the on-disk version is left at featureStateVersion, not flipped down); - // featureStateVersion with any feature depends on capabilities this CLI lacks, so it is refused. - // Forward-compat scaffolding so a later release can write featureStateVersion + features without - // this CLI rejecting them outright. Always 3. + // featureStateVersion is the schema version a future CLI will write once it + // records deployment state "feature flags" (see Header.Features). This CLI does + // not write it and records no features; it exists now only so this CLI reads + // such states correctly (see migrateState): + // - featureStateVersion with no features -> accept and leave the version as-is + // - featureStateVersion with any feature -> refuse, tell the user to upgrade + // + // A featureStateVersion state with no features is equivalent to + // currentStateVersion, but we deliberately do not flip the on-disk version down + // to currentStateVersion: a state written at featureStateVersion stays at + // featureStateVersion. This is forward-compat scaffolding so that a later release + // can start writing featureStateVersion + features without older CLIs (with this + // change) either mishandling a feature they lack or rejecting a featureless state + // outright. featureStateVersion is always 3. featureStateVersion = 3 - // supportedStateVersion is the highest schema version this CLI can read. Normally equal to - // currentStateVersion - the version it reads is the version it writes - but this CLI can also - // read (never write) featureStateVersion. A state newer than this is rejected as too new. + // supportedStateVersion is the highest schema version this CLI can read. It is + // normally equal to currentStateVersion — the version this CLI reads is the + // version it writes — and exceeds it only during a two-phase version bump like + // the current feature-flag scaffolding, where this CLI reads (but does not + // write) featureStateVersion. A state newer than this is rejected as too new. supportedStateVersion = featureStateVersion ) @@ -105,7 +114,7 @@ var errStaleWAL = errors.New("stale WAL") // present an operation-appropriate message via errors.Is. var ErrUnsettingRecording = errors.New(`unsetting experimental.record_deployment_history is not supported -This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) +This deployment's resources are recorded with the deployment history feature enabled. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) type DeploymentState struct { Path string @@ -117,7 +126,7 @@ type DeploymentState struct { stateIDs map[string]string // operationBuffer records each state write with DMS. Nil unless the bundle records deployment - // history, in which case StartRecording installs it once the version exists. + // history, in which case InitializeOperationBuffer installs it once the version exists. operationBuffer *dms.OperationBuffer // dmsClient talks to the deployment metadata service. Open builds it from the workspace @@ -125,22 +134,18 @@ type DeploymentState struct { dmsClient *dms.Client // storageBackend is where this deployment's state lives, set by Open from the feature marker. - // It is the source of truth for gating DMS behavior after Open, in place of nil client checks. + // It is the source of truth for gating DMS behavior after Open. storageBackend StorageBackend - // deploymentID and versionID identify the recorded deployment and the version this run writes, - // stored by StartRecording. CompleteVersion reads them after Finalize's reset has cleared Path, - // Data and stateIDs, so - like operationBuffer and dmsClient - they must survive reset. - deploymentID string - versionID int64 - // versionCompleted makes CompleteVersion a no-op after the first call, so a deferred safety-net // completion after an explicit one does nothing. versionCompleted bool - // DeploymentID and LatestVersionID are the recorded deployment's id and its most recent version - // at Open time, from the service. CalculatePlan reads them (rather than the bundle config tree) - // to stamp the plan's lineage. Both empty for a first deploy, whose id the deploy phase creates. + // DeploymentID and LatestVersionID are the recorded deployment's id and its most recent version, + // from the service at Open (InitializeOperationBuffer updates DeploymentID to the id a first + // deploy creates, which Open cannot know). CalculatePlan reads them to stamp the plan's lineage; + // CompleteVersion reads them after Finalize's reset - so, like operationBuffer and dmsClient, they + // must survive reset. DeploymentID string LatestVersionID string } @@ -183,24 +188,27 @@ type WALEntry struct { Value *ResourceEntry `json:"v,omitempty"` // nil means delete } -// StartRecording opens the operation buffer that records each subsequent state write with DMS -// under deploymentID and versionID, so what the service holds mirrors the WAL. It uses the client -// Open built, so it is called only when the deployment records history, and once the version -// exists (after approval) - which is why it is not an Open option. -func (db *DeploymentState) StartRecording(ctx context.Context, deploymentID string, versionID int64) { +// InitializeOperationBuffer opens the operation buffer that records each subsequent state write with +// DMS under deploymentID and versionID, so what the service holds mirrors the WAL. It uses the client +// Open built, so it is called only when the deployment records history, and once the version exists +// (after approval) - which is why it is not an Open option. It also records the id a first deploy +// just created, which Open could not know, so CompleteVersion later has it. +func (db *DeploymentState) InitializeOperationBuffer(ctx context.Context, deploymentID string, versionID int64) { buf := dms.StartOperationBuffer(ctx, db.dmsClient, deploymentID, versionID) db.mu.Lock() defer db.mu.Unlock() db.operationBuffer = buf - db.deploymentID = deploymentID - db.versionID = versionID + db.DeploymentID = deploymentID } // RecordingError reports whether recording state writes to the service has failed, so the apply // stops touching resources once the service is no longer keeping up. Nil when the bundle does not // record deployment history or recording is healthy. func (db *DeploymentState) RecordingError() error { + if db.StorageBackend() != StorageBackendDeploymentMetadataService { + return nil + } db.mu.Lock() buf := db.operationBuffer db.mu.Unlock() @@ -223,9 +231,14 @@ func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (b return false, nil } db.versionCompleted = true - deploymentID, versionID, client := db.deploymentID, db.versionID, db.dmsClient + deploymentID, latestVersionID, client := db.DeploymentID, db.LatestVersionID, db.dmsClient db.mu.Unlock() + versionID, err := dms.NextVersion(latestVersionID) + if err != nil { + return false, err + } + // A recording failure fails the version even when the caller counted the deploy a success: the // service does not then hold everything the WAL does. Finalize already drained and surfaced it; // this reads the drained buffer's error so a destroy whose uploads failed keeps its record. @@ -266,7 +279,7 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e } } - // A failure is recorded only while a version is open (after StartRecording), so the buffer is set. + // A failure is recorded only while a version is open (after InitializeOperationBuffer), so the buffer is set. db.mu.Lock() buf := db.operationBuffer db.mu.Unlock() @@ -463,7 +476,7 @@ type ( // When withDeploymentHistory is set it builds a DMS client from wsClient and reads resources from // the service instead, with dmsDeploymentID the id the service holds (empty before the first // recorded deploy); lineage and serial still come from the file, since that is what the write path -// increments. Open only reads through the client - StartRecording installs the write path once a +// increments. Open only reads through the client - InitializeOperationBuffer installs the write path once a // version exists. func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { db.mu.Lock() @@ -566,7 +579,7 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe db.Data.Features[featureRecordDeploymentHistory] = struct{}{} recorded = true case recording && !recorded: - return errors.New(`this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + return errors.New(`this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: 1. remove experimental.record_deployment_history from your bundle configuration diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 2db8bf57dca..fa9a0f65a14 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -153,7 +153,7 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version } log.Infof(ctx, "Created deployment version: deployment=%s version=%s", deploymentID, version.VersionId) - db.StateDB.StartRecording(ctx, deploymentID, versionID) + db.StateDB.InitializeOperationBuffer(ctx, deploymentID, versionID) return nil } diff --git a/libs/dms/client.go b/libs/dms/client.go index 611bdf9c11b..c4f592dec11 100644 --- a/libs/dms/client.go +++ b/libs/dms/client.go @@ -105,7 +105,7 @@ func (c *Client) UpdateOperation(ctx context.Context, deploymentID string, versi func deploymentIDFromName(name string) (string, error) { id, ok := strings.CutPrefix(name, DeploymentName("")) if !ok || id == "" { - return "", fmt.Errorf("unexpected deployment name %q from deployment metadata service", name) + return "", fmt.Errorf("unexpected deployment name %q from the deployment history service", name) } return id, nil } diff --git a/libs/dms/resources.go b/libs/dms/resources.go index 565571415f6..11321161c15 100644 --- a/libs/dms/resources.go +++ b/libs/dms/resources.go @@ -29,7 +29,7 @@ func (c *Client) ListResources(ctx context.Context, deploymentID string) ([]Reso for it.HasNext(ctx) { res, err := it.Next(ctx) if err != nil { - return nil, fmt.Errorf("listing resources from deployment metadata service: %w", err) + return nil, fmt.Errorf("listing resources from the deployment history service: %w", err) } out = append(out, Resource{ Key: statePrefix + res.ResourceKey, From 97ba52add1ba5f877847d5042e48e1070b2180cf Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 00:21:45 +0200 Subject: [PATCH 023/164] hand edits --- bundle/direct/apply.go | 5 +- bundle/direct/bundle_apply.go | 1 - bundle/direct/bundle_plan.go | 35 +++---- bundle/direct/dstate/state.go | 171 +++++++++++++--------------------- bundle/phases/bind.go | 3 - bundle/phases/deploy.go | 42 ++++----- bundle/phases/destroy.go | 44 +++++---- 7 files changed, 128 insertions(+), 173 deletions(-) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index af2a7277c77..f04b56e8b72 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -259,9 +259,8 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, log.Warnf(ctx, "Ignoring permission error when deleting %s id=%s: %s", d.ResourceKey, oldID, err) } else if d.deleteConfirmedGone(ctx, oldID) { log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) - } else { - // The resource is still there, so the history says why rather than leaving its - // operation pending. The state write below is what records a delete that worked. + } else if db.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + // When using the deployment metadata service, record the error. err = fmt.Errorf("deleting id=%s: %w", oldID, err) db.RecordFailure(d.ResourceKey, oldID, err) return err diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 74bfa226cbf..db0a7ab4b3c 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -178,7 +178,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) - } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 775103e6ad9..a2232b13088 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -61,10 +61,8 @@ func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan. return nil } -// InitForApply initializes the DeploymentBundle for applying a pre-computed plan: it parses the -// plan's entries (loaded as JSON when the plan came from a file) into the typed state cache the -// apply reads. A first deployment's id does not exist until the deploy creates it after approval; -// StampDeploymentIdForFirstVersion fills it in then. StateDB must already be open for write before calling this. +// InitForApply initializes the DeploymentBundle for applying a pre-computed plan. +// StateDB must already be open for write before calling this function. func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { b.StateDB.AssertOpenedForWrite() @@ -120,8 +118,9 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. // StampDeploymentIdForFirstVersion fills deploymentID into the plan's jobs and pipelines that still lack one, in // both the state cache the apply reads and the plan JSON. It is called after approval creates the -// deployment, for a first deploy whose id did not exist at plan time; the version and any -// pre-existing id were stamped then. +// deployment, for a first deploy whose deployment_id did not exist at plan time. +// +// For subsequent deployments, the deployment_id is already stamped in the plan. func (b *DeploymentBundle) StampDeploymentIdForFirstVersion(deploymentID string) error { for resourceKey, entry := range b.Plan.Plan { if entry.NewState == nil || len(entry.NewState.Value) == 0 { @@ -169,23 +168,19 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } // The plan records where its state lives so deploy --plan can reject a plan whose target has - // since switched backends. StateDB is the source of truth, so a first deploy - which has no - // recorded deployment yet - still carries the backend. + // since switched backends or has an outdated version or a wrong deployment_id. if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { plan.StorageBackend = string(dstate.StorageBackendDeploymentMetadataService) - } - // Record the DMS deployment and version this plan targets, from StateDB (not the config tree). - // A saved plan carries them so deploy --plan can reject a plan the deployment has moved on from, - // and passes last_version_id as previous_version_id. Empty for a first deploy, which has no - // recorded deployment yet - the deploy phase stamps the created id. - if b.StateDB.DeploymentID != "" { - next, err := dms.NextVersion(b.StateDB.LatestVersionID) - if err != nil { - return nil, fmt.Errorf("computing next deployment version: %w", err) + // Not the first deployment, stamp the deployment_id and version_id into the plan. + if b.StateDB.DeploymentID != "" { + next, err := dms.NextVersion(b.StateDB.LatestVersionID) + if err != nil { + return nil, fmt.Errorf("computing next deployment version: %w", err) + } + plan.DeploymentId = b.StateDB.DeploymentID + plan.LastVersionId = b.StateDB.LatestVersionID + plan.NextVersionId = strconv.FormatInt(next, 10) } - plan.DeploymentId = b.StateDB.DeploymentID - plan.LastVersionId = b.StateDB.LatestVersionID - plan.NextVersionId = strconv.FormatInt(next, 10) } b.Plan = plan diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ef67ccfc559..79d59bd39ee 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -64,7 +64,7 @@ const ( // // The marker is sticky: once a deployment is recorded, the service holds resources that a // CLI which is not recording must not touch. So turning recording off does not clear it, and -// deploying such a state without recording is refused (see RequiresDeploymentHistory). +// deploying such a state without recording is refused. const featureRecordDeploymentHistory = "record_deployment_history" // featuresDocURL explains the state-features mechanism in the error shown when this CLI refuses a @@ -88,10 +88,8 @@ var recognizedFeatures = map[string]struct{}{ featureRecordDeploymentHistory: {}, } -// checkStateFeatures refuses a state that depends on a feature this CLI does not recognize: keeping -// the state and the service in step needs every reader to understand the features in play, so an -// unknown one belongs to a newer CLI and this one refuses rather than clobber it. -func checkStateFeatures(features map[string]struct{}) error { +// assertNoUnsupportedFeatures refuses a state that depends on a feature this CLI does not recognize. +func assertNoUnsupportedFeatures(features map[string]struct{}) error { var unsupported []string for feature := range features { if _, ok := recognizedFeatures[feature]; !ok { @@ -224,6 +222,10 @@ func (db *DeploymentState) RecordingError() error { // one does nothing. Finalize has already drained the buffered operations. It reads only fields that // survive Finalize's reset and asserts nothing, so it is safe to call after the state is closed. func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (bool, error) { + if db.StorageBackend() != StorageBackendDeploymentMetadataService { + return false, nil + } + db.mu.Lock() buf := db.operationBuffer if buf == nil || db.versionCompleted { @@ -301,6 +303,12 @@ func NewDatabase(lineage string, serial int) Database { // SaveState records the resource's state after an operation was applied to it. func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { db.AssertOpenedForWrite() + db.mu.Lock() + defer db.mu.Unlock() + + if db.Data.State == nil { + db.Data.State = make(map[string]ResourceEntry) + } jsonMessage, err := json.Marshal(state) if err != nil { @@ -312,33 +320,23 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta DependsOn: dependsOn, } - db.mu.Lock() - if db.Data.State == nil { - db.Data.State = make(map[string]ResourceEntry) - } err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) if err == nil { db.stateIDs[key] = newID } - buf := db.operationBuffer - db.mu.Unlock() - if err != nil { return err } - if buf == nil { - return nil - } - // Recorded after the WAL write, so DMS never reports state the deploy failed to persist, - // and outside db.mu because recording waits when the service is behind - waiting under the - // lock would hold up every other resource's write. - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) - if err != nil { - return err + // Record operation in DMS. + buf := db.operationBuffer + if db.StorageBackend() == StorageBackendDeploymentMetadataService && buf != nil { + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + if err != nil { + return err + } + buf.RecordOperation(ctx, key, false, newID, recorded) } - buf.RecordOperation(ctx, key, false, newID, recorded) - return nil } @@ -347,10 +345,10 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta // deploy must not leave the resource described as finished. func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgress bool) error { db.AssertOpenedForWrite() - db.mu.Lock() + defer db.mu.Unlock() + if db.Data.State == nil { - db.mu.Unlock() return nil } // Read before the delete: DMS needs the id to say which resource went away. @@ -359,19 +357,14 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgre if err == nil { delete(db.stateIDs, key) } - buf := db.operationBuffer - db.mu.Unlock() - if err != nil { return err } - // State is nil: the resource no longer exists. Recorded outside the lock for the - // same reason as SaveState. - if buf != nil { + buf := db.operationBuffer + if db.StorageBackend() == StorageBackendDeploymentMetadataService && buf != nil { buf.RecordOperation(ctx, key, inProgress, deletedID, nil) } - return nil } @@ -410,24 +403,9 @@ func (db *DeploymentState) StateCLIVersion() string { return db.Data.CLIVersion } -// RequiresDeploymentHistory reports whether the state depends on the deployment metadata -// service recording it. Deploying such a state without recording would leave the service -// holding a deployment that no longer matches, so the caller refuses instead. -func (db *DeploymentState) RequiresDeploymentHistory() bool { - db.AssertOpenedForReadOrWrite() - db.mu.Lock() - defer db.mu.Unlock() - - _, ok := db.Data.Features[featureRecordDeploymentHistory] - return ok -} - // StorageBackend reports where this deployment's state lives - the source of truth for whether the // bundle records deployment history. Set by Open; valid only after the state is opened. func (db *DeploymentState) StorageBackend() StorageBackend { - db.AssertOpenedForReadOrWrite() - db.mu.Lock() - defer db.mu.Unlock() return db.storageBackend } @@ -512,7 +490,7 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe db.Path = path // The state file is the source of truth for whether this deployment records history: read it - // first, then reconcile the config against the marker it carries. + // first. data, err := os.ReadFile(db.Path) if err != nil { if errors.Is(err, fs.ErrNotExist) { @@ -530,68 +508,52 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe return fmt.Errorf("migrating state %s: %w", path, err) } - if err := checkStateFeatures(db.Data.Features); err != nil { + // TODO: We can remove and move this assertion to migrateState once we do the state + // version bump to 3 for this CLI. + if err := assertNoUnsupportedFeatures(db.Data.Features); err != nil { return err } - recording := bool(withDeploymentHistory) - db.stateIDs = make(map[string]string) for key, entry := range db.Data.State { db.stateIDs[key] = entry.ID } walPath := db.Path + walSuffix - _, statErr := os.Stat(walPath) - walExists := statErr == nil - if statErr != nil && !errors.Is(statErr, fs.ErrNotExist) { - return fmt.Errorf("failed to stat WAL file %s: %w", walPath, statErr) - } - - // A recording open discards a leftover WAL - the service owns the resources, so a local log from - // a crash or a declined deploy is irrelevant, even when recording is refused below. Without - // recording, replay the WAL as usual. - if walExists { - switch { - case recording: - if err := os.Remove(walPath); err != nil { - return fmt.Errorf("removing WAL file %s: %w", walPath, err) - } - case bool(withRecovery): + _, err = os.Stat(walPath) + switch { + case errors.Is(err, fs.ErrNotExist): + // no WAL, nothing to do + case err != nil: + return fmt.Errorf("failed to stat WAL file %s: %w", walPath, err) + default: // WAL exists + if withRecovery { if err := db.replayWAL(ctx); err != nil { return fmt.Errorf("reading state from %s: %w", path, err) } - default: + } else if withDeploymentHistory { + // Ignore the WAL file. That is not read for deployments that record history. + } else { return fmt.Errorf("unexpected WAL file found at %s", walPath) } } - // Reconcile the config against the state. A brand-new deployment is where recording begins, so - // the config bootstraps the marker there. On an existing deployment the state is authoritative: - // the config can neither start recording one that was not (its resources would be created a - // second time) nor stop recording one that is (the service still holds it). - _, recorded := db.Data.Features[featureRecordDeploymentHistory] - switch { - case recording && !recorded && len(db.Data.State) == 0: - if db.Data.Features == nil { - db.Data.Features = make(map[string]struct{}, 1) - } - db.Data.Features[featureRecordDeploymentHistory] = struct{}{} - recorded = true - case recording && !recorded: + _, stateHasRecordDeploymentHistory := db.Data.Features[featureRecordDeploymentHistory] + if !stateHasRecordDeploymentHistory && bool(withDeploymentHistory) { return errors.New(`this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources -To record this bundle's history, start it over as a new deployment: - 1. remove experimental.record_deployment_history from your bundle configuration - 2. run "databricks bundle destroy" to delete the existing resources - 3. add experimental.record_deployment_history back and deploy again`) - case !recording && recorded: - return ErrUnsettingRecording + To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again`) } db.storageBackend = StorageBackendWorkspaceFilesystem - if recorded { + if stateHasRecordDeploymentHistory { db.storageBackend = StorageBackendDeploymentMetadataService + if !bool(withDeploymentHistory) { + return ErrUnsettingRecording + } // The service is the source of truth for a recorded deployment; the file is a tombstone // carrying only the marker, and applyDMSState loads the resources the service holds. @@ -716,7 +678,6 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) if header.Serial > expectedSerial { return false, fmt.Errorf("WAL serial (%d) is ahead of expected (%d), state may be corrupted", header.Serial, expectedSerial) } - newSerial = header.Serial newCLIVersion = header.CLIVersion } else { @@ -796,10 +757,8 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported err = db.replayWAL(ctx) } - // A recorded deployment uploads each write to the service on a background goroutine; block until - // they land or fail so the version completes only over state the service actually holds. A nil - // buffer means the deployment does not record history. - if db.operationBuffer != nil { + // Wait until all operations are recorded in the service. + if db.StorageBackend() == StorageBackendDeploymentMetadataService && db.operationBuffer != nil { if drainErr := db.operationBuffer.Drain(); drainErr != nil { err = errors.Join(err, drainErr) } @@ -925,7 +884,7 @@ func (db *DeploymentState) ExportState(ctx context.Context) resourcestate.Export // the WAL. A torn write would therefore leave a state file that Open rejects // next to an intact WAL it never reads. func (db *DeploymentState) unlockedSave() error { - data, err := json.MarshalIndent(db.dataToPersist(), "", " ") + data, err := json.MarshalIndent(db.dataForFile(), "", " ") if err != nil { return err } @@ -961,18 +920,22 @@ func (db *DeploymentState) unlockedSave() error { return nil } -// dataToPersist returns what the state file should hold: db.Data itself, unless the deployment -// records its history, in which case it is the header alone. The service holds those resources and -// is where they are read back from, so writing them here too would be a second, never-read copy. -// Returns a copy, leaving the in-memory state - what the rest of the deploy works from - untouched. -func (db *DeploymentState) dataToPersist() Database { - data := db.Data - if _, recorded := data.Features[featureRecordDeploymentHistory]; !recorded { - return data +// Data to persist in the remote resources.json file. +func (db *DeploymentState) dataForFile() Database { + if db.StorageBackend() == StorageBackendDeploymentMetadataService { + header := db.Data.Header + if header.Features == nil { + header.Features = make(map[string]struct{}) + } + header.Features[featureRecordDeploymentHistory] = struct{}{} + + return Database{ + Header: header, + State: map[string]ResourceEntry{}, + } } - data.State = map[string]ResourceEntry{} - return data + return db.Data } func appendJSONLine(file *os.File, obj any) error { diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index fc36122d1d8..86c9cc80da3 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -135,9 +135,6 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r }() if engine.IsDirect() { - // A recorded deployment cannot be unbound: DeploymentBundle.Unbind opens the state without - // recording, so the state guard refuses it when the marker is set. State is the source of - // truth, so a config flag that disagrees with the marker cannot slip an unbind through. groupName, ok := terraform.TerraformToGroupName[tfResourceType] if !ok { groupName = tfResourceType diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 88245e107e1..531270bfc76 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -205,9 +205,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // The version is created only after approval; CompleteVersion is deferred before // lock.Release and no-ops until then. defer func() { - // Skip entirely when not recording; CompleteVersion runs after Finalize has closed the state, - // so it is gated on the config rather than the (reset) StateDB. - if b.ConfiguresDeploymentHistory(ctx) { + // Close the version when backend type is deployment metadata service. + if b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { if _, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { logdiag.LogError(ctx, err) } @@ -299,8 +298,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } if planFromFile { - // Parse the loaded plan into the state cache the apply reads. A first deploy's deployment id - // is stamped later, after approval creates it (see below). + // Initialize DeploymentBundle for applying the loaded plan err := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan) if err != nil { logdiag.LogError(ctx, err) @@ -315,13 +313,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - haveApproval, approvalErr := approvalForDeploy(ctx, b, plan) + haveApproval, err := approvalForDeploy(ctx, b, plan) if !haveApproval { // No version was created, so the deferred CompleteVersion is a no-op and the version // number is left for the next deploy. Both the user declining and a console that // cannot prompt land here. - if approvalErr != nil { - logdiag.LogError(ctx, approvalErr) + if err != nil { + logdiag.LogError(ctx, err) return } cmdio.LogString(ctx, "Deployment cancelled!") @@ -345,22 +343,22 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } } - } - // Create the version the plan was stamped with, staging an operation for every resource - // it touches. Doing it here rather than before the prompt means a declined deploy never - // claims a version number. - staged, err := stagedOperations(plan) - if err != nil { - logdiag.LogError(ctx, err) - return - } - if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { - logdiag.LogError(ctx, err) - return + // Create the version the plan was stamped with, staging an operation for every resource + // it touches. Doing it here rather than before the prompt means a declined deploy never + // claims a version number. + staged, err := stagedOperations(plan) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { + logdiag.LogError(ctx, err) + return + } + deploymentID, versionID := deploymentAndNextVersion(b) + logDeploymentVersion(ctx, b, deploymentID, versionID) } - deploymentID, versionID := deploymentAndNextVersion(b) - logDeploymentVersion(ctx, b, deploymentID, versionID) deployCore(ctx, b, plan, stateEngine) diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 309adb120da..69523ee7579 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -14,6 +14,7 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" @@ -159,19 +160,21 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } - // Complete version before deleting remote files; the deployment node is under statePath. - completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) - if err != nil { - logdiag.LogError(ctx, err) - return - } - // A completed destroy's resources are gone, so its deployment record is deleted too. - if completed { - deploymentID, _ := deploymentAndNextVersion(b) - if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + if engine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + // Complete version before deleting remote files; the deployment node is under statePath. + completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) + if err != nil { + logdiag.LogError(ctx, err) return } + // A completed destroy's resources are gone, so its deployment record is deleted too. + if completed { + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + return + } + } } bundle.ApplyContext(ctx, b, files.Delete()) @@ -216,14 +219,15 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // destroy records nothing. Deferred before lock.Release to hold the lock; a no-op once // destroyCore has completed the version. defer func() { - completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)) - if err != nil { - logdiag.LogError(ctx, err) - } else if completed { - // A completed destroy's resources are gone, so its deployment record is deleted too. - deploymentID, _ := deploymentAndNextVersion(b) - if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + if engine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)) + if err != nil { + logdiag.LogError(ctx, err) + } else if completed { + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + } } } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) @@ -288,7 +292,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // destroy's recording: it opens the operation buffer that destroyCore drains and completes, // so the drain and the deployment delete follow only when a version was started here. // Destroy never creates or updates the deployment - it is about to be deleted. - if engine.IsDirect() && b.DeploymentBundle.StateDB.RequiresDeploymentHistory() { + if engine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { staged, err := stagedOperations(plan) if err != nil { logdiag.LogError(ctx, err) From dad29930a31f7d6453c4ea16f7e1cb823fc4ab38 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 00:37:00 +0200 Subject: [PATCH 024/164] bundle/dms: fix hand-edit regressions in unlockedOpen and declined-deploy test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs introduced by the previous hand-edit commit: 1. unlockedOpen: the 3-way switch reconciling config vs state was replaced with a simple `if !stateHasRecordDeploymentHistory && withDeploymentHistory`, which errored even on brand-new deployments (no state file → no feature flag → error). Restored the original logic including the bootstrap case for `recording && !recorded && len(db.Data.State) == 0`. 2. unlockedOpen: recording opens must delete a leftover WAL (the service owns the resources; a local log from a crash is irrelevant). The hand edit changed this to "ignore the WAL", breaking version-never-created and existing-state. Restored os.Remove on recording opens. 3. Error message indentation: the multi-line error had tab-indented continuation lines instead of no leading whitespace, mismatching the existing-state golden. Also: declined-deploy/script - add --get to the second print_requests call so the GET requests made during the declined deploy (GetDeployment + ListResources) appear in the output. Move out.requests.txt cleanup into the EXIT trap so the catalog delete's request file is removed and no out.requests.txt is left behind. Co-authored-by: Isaac --- .../dms/declined-deploy/out.requests.txt | 7 --- .../bundle/dms/declined-deploy/output.txt | 24 +++++++++- acceptance/bundle/dms/declined-deploy/script | 7 ++- bundle/direct/dstate/state.go | 45 +++++++++++++------ 4 files changed, 56 insertions(+), 27 deletions(-) delete mode 100644 acceptance/bundle/dms/declined-deploy/out.requests.txt diff --git a/acceptance/bundle/dms/declined-deploy/out.requests.txt b/acceptance/bundle/dms/declined-deploy/out.requests.txt deleted file mode 100644 index eb169f26a0c..00000000000 --- a/acceptance/bundle/dms/declined-deploy/out.requests.txt +++ /dev/null @@ -1,7 +0,0 @@ -{ - "method": "DELETE", - "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", - "q": { - "force": "true" - } -} diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index d0ce19a2389..890e58e72b7 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -6,7 +6,7 @@ Created schemas.foo Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py --dms //api/2.0/bundle +>>> print_requests.py --dms --get //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -129,7 +129,27 @@ To proceed, use --auto-approve after reviewing the plan above. Files: 4 uploaded, 0 deleted === Nothing was recorded for the declined deploy - no version, so none to abort ->>> print_requests.py --dms //api/2.0/bundle +>>> print_requests.py --dms --get //api/2.0/bundle +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} === The recorded resources are unchanged by the decline: the schema is still at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index cc69f672e63..d49fcfbfb53 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy a schema, so the deployment and its first version exist" trace $CLI bundle deploy -trace print_requests.py --dms //api/2.0/bundle +trace print_requests.py --dms --get //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" trace $CLI bundle plan -o json | nostamp.py @@ -13,14 +13,14 @@ title "A destructive change without --auto-approve is declined: this console can # Unity Catalog has no catalog called "other", so make one to move the schema into. other_catalog="dms_other_${UNIQUE_NAME}" $CLI catalogs create "$other_catalog" &> LOG.catalog -trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1; rm -f out.requests.txt' EXIT trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" trace musterr $CLI bundle deploy title "Nothing was recorded for the declined deploy - no version, so none to abort" # The version number it would have used is left for the next deploy to take, so the # history has no entry that reads like a deploy which failed or did nothing. -trace print_requests.py --dms //api/2.0/bundle +trace print_requests.py --dms --get //api/2.0/bundle title "The recorded resources are unchanged by the decline: the schema is still at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-declined-deploy-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -29,4 +29,3 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle destroy --auto-approve title "After the destroy the deployment is gone, so the summary reports no history rather than a dangling id" trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' -rm -f out.requests.txt diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 79d59bd39ee..f43503c7a05 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -519,6 +519,8 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe db.stateIDs[key] = entry.ID } + recording := bool(withDeploymentHistory) + walPath := db.Path + walSuffix _, err = os.Stat(walPath) switch { @@ -527,33 +529,48 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe case err != nil: return fmt.Errorf("failed to stat WAL file %s: %w", walPath, err) default: // WAL exists - if withRecovery { + switch { + case recording: + // A recording open discards a leftover WAL - the service owns the resources, so a local + // log from a crash or a declined deploy is irrelevant, even when recording is refused below. + if err := os.Remove(walPath); err != nil { + return fmt.Errorf("removing WAL file %s: %w", walPath, err) + } + case bool(withRecovery): if err := db.replayWAL(ctx); err != nil { return fmt.Errorf("reading state from %s: %w", path, err) } - } else if withDeploymentHistory { - // Ignore the WAL file. That is not read for deployments that record history. - } else { + default: return fmt.Errorf("unexpected WAL file found at %s", walPath) } } - _, stateHasRecordDeploymentHistory := db.Data.Features[featureRecordDeploymentHistory] - if !stateHasRecordDeploymentHistory && bool(withDeploymentHistory) { + // Reconcile the config against the state. A brand-new deployment is where recording begins, so + // the config bootstraps the marker there. On an existing deployment the state is authoritative: + // the config can neither start recording one that was not (its resources would be created a + // second time) nor stop recording one that is (the service still holds it). + _, recorded := db.Data.Features[featureRecordDeploymentHistory] + switch { + case recording && !recorded && len(db.Data.State) == 0: + if db.Data.Features == nil { + db.Data.Features = make(map[string]struct{}, 1) + } + db.Data.Features[featureRecordDeploymentHistory] = struct{}{} + recorded = true + case recording && !recorded: return errors.New(`this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources - To record this bundle's history, start it over as a new deployment: - 1. remove experimental.record_deployment_history from your bundle configuration - 2. run "databricks bundle destroy" to delete the existing resources - 3. add experimental.record_deployment_history back and deploy again`) +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again`) + case !recording && recorded: + return ErrUnsettingRecording } db.storageBackend = StorageBackendWorkspaceFilesystem - if stateHasRecordDeploymentHistory { + if recorded { db.storageBackend = StorageBackendDeploymentMetadataService - if !bool(withDeploymentHistory) { - return ErrUnsettingRecording - } // The service is the source of truth for a recorded deployment; the file is a tombstone // carrying only the marker, and applyDMSState loads the resources the service holds. From bb5abcdae431ad641df1c9b4b32805fe0dace9bc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 01:02:23 +0200 Subject: [PATCH 025/164] acceptance/dms: serialize deployment_id and version_id in plan output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove | nostamp.py from all bundle/dms test scripts. nostamp was stripping deployment_id, next_version_id, and last_version_id from plan output (its purpose is to make non-DMS tests agnostic to recording mode). DMS-specific tests should assert these fields directly — they are the feature under test. Regenerate all goldens. Plans now show deployment_id/next_version_id/ last_version_id after the first deploy, confirming the fields are serialized. Co-authored-by: Isaac --- .../bundle/dms/declined-deploy/output.txt | 3 + acceptance/bundle/dms/declined-deploy/script | 2 +- acceptance/bundle/dms/depends-on/output.txt | 25 +++- acceptance/bundle/dms/depends-on/script | 2 +- .../dms/deployment-metadata-change/output.txt | 6 + .../dms/deployment-metadata-change/script | 4 +- .../bundle/dms/emptied-resource/output.txt | 3 + acceptance/bundle/dms/emptied-resource/script | 2 +- acceptance/bundle/dms/existing-state/script | 2 +- .../bundle/dms/failed-delete/output.txt | 14 ++- acceptance/bundle/dms/failed-delete/script | 2 +- .../bundle/dms/failed-recreate/output.txt | 3 + acceptance/bundle/dms/failed-recreate/script | 2 +- .../dms/failed-update-permissions/output.txt | 14 ++- .../dms/failed-update-permissions/script | 2 +- .../bundle/dms/failed-update/output.txt | 3 + acceptance/bundle/dms/failed-update/script | 2 +- .../bundle/dms/multiple-resources/output.txt | 116 ++++++++++++++++-- .../bundle/dms/multiple-resources/script | 4 +- acceptance/bundle/dms/no-drift/output.txt | 54 +++++++- acceptance/bundle/dms/no-drift/script | 4 +- acceptance/bundle/dms/no-resources/output.txt | 6 + acceptance/bundle/dms/no-resources/script | 4 +- .../operation-upload-fails-once/output.txt | 3 + .../dms/operation-upload-fails-once/script | 2 +- acceptance/bundle/dms/provenance/output.txt | 14 ++- acceptance/bundle/dms/provenance/script | 2 +- .../bundle/dms/record-failure/output.txt | 7 +- acceptance/bundle/dms/record-failure/script | 2 +- acceptance/bundle/dms/record/output.txt | 14 ++- acceptance/bundle/dms/record/script | 2 +- acceptance/bundle/dms/stale-plan/output.txt | 3 +- acceptance/bundle/dms/stale-plan/script | 2 +- .../bundle/dms/successful-recreate/output.txt | 6 + .../bundle/dms/successful-recreate/script | 4 +- .../dms/version-never-created/output.txt | 6 +- .../bundle/dms/version-never-created/script | 2 +- 37 files changed, 301 insertions(+), 47 deletions(-) diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 890e58e72b7..552567f1cab 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index d49fcfbfb53..d31b65e6129 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms --get //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json trace print_state.py title "A destructive change without --auto-approve is declined: this console cannot prompt" diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 4fb3a8caab5..83d71a1e2ad 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -44,6 +44,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.child": { @@ -58,8 +61,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][0], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "description": "depends on [NUMID]", "edit_mode": "UI_LOCKED", @@ -76,6 +81,13 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -99,8 +111,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][1], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -116,6 +130,13 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 022f8e3dd11..5b721e24c6b 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists both jobs, each recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-depends-on-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 3422dde62d7..99c1c51f301 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { @@ -167,6 +170,9 @@ A common practice is to use a username or principal name in this path, i.e. use "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script index 45de9533cab..5d650aaa341 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/script +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-metadata-pinned-${UNIQUE_NAME}/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' @@ -15,7 +15,7 @@ trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle title "The metadata changed but the schema did not: still one deployment, recorded at version 1" -trace $CLI bundle plan -t prod -o json | nostamp.py +trace $CLI bundle plan -t prod -o json trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "Deploy again without touching anything: the record already says this, so no deployment update" diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 297b8e441db..0b00307c1e3 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -137,6 +137,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 6e5af06cc31..8d49ae09cdd 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -17,7 +17,7 @@ title "Plan again: reading state back from the service works and reports no work trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The grants node emptied out, so the service no longer lists it - only the schema remains" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-emptied-resource-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 5ab20285e56..1b1a9401965 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" trace $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json trace print_requests.py --dms //api/2.0/bundle --oneline trace jq '{state_version, features}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 17d98e0a4a3..4585105b85a 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.stuck": { @@ -74,8 +77,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -91,6 +96,13 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index ed544b27d7f..9f3881f78bd 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index a85298826b2..5f8bfa55121 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 6c0b82a1a69..77b72b94669 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index ac97f5ce901..345f048464a 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -14,6 +14,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -22,8 +25,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -39,6 +44,13 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index 6883d8f3775..8b251b08b7c 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy rm -f out.requests.txt title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the job" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists the job at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-permissions-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index 2cb6c855d5f..54358b62e95 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script index 7d223af223b..6e1acfe26e2 100644 --- a/acceptance/bundle/dms/failed-update/script +++ b/acceptance/bundle/dms/failed-update/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 4a737c37b6e..79370c3d51d 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -24,6 +24,9 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.five": { @@ -32,8 +35,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][0], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -49,6 +54,13 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -72,8 +84,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][1], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -89,6 +103,13 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -112,8 +133,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][2], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -129,6 +152,13 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -152,8 +182,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][3], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -169,6 +201,13 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -192,8 +231,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][4], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -209,6 +250,13 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -380,6 +428,9 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.five": { @@ -388,8 +439,10 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "created_time": [UNIX_TIME_MILLIS][0], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -405,6 +458,13 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -428,8 +488,10 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "created_time": [UNIX_TIME_MILLIS][1], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -445,6 +507,13 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -468,8 +537,10 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "created_time": [UNIX_TIME_MILLIS][2], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -485,6 +556,13 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -508,8 +586,10 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "created_time": [UNIX_TIME_MILLIS][3], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -525,6 +605,13 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -548,8 +635,10 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "created_time": [UNIX_TIME_MILLIS][4], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -565,6 +654,13 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 78ab3028390..6315788d22a 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists all five resources at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -19,7 +19,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --sort --oneline title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service still lists all five resources, now at version 2" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 87d254cf2e2..064ba5acfeb 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -34,6 +34,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -42,8 +45,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][0], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -59,6 +64,13 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -83,8 +95,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "channel": "CURRENT", "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edition": "ADVANCED", "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", @@ -103,6 +117,15 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "schema": "default", "serverless": true, "state": "IDLE" + }, + "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + } } } } @@ -277,6 +300,9 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -285,8 +311,10 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "created_time": [UNIX_TIME_MILLIS][0], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -302,6 +330,13 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -326,8 +361,10 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "channel": "CURRENT", "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "2" }, "development": true, "edition": "ADVANCED", @@ -347,6 +384,15 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "schema": "default", "serverless": true, "state": "IDLE" + }, + "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "2", + "new": "3", + "remote": "2" + } } } } diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 55bff445646..64455e8f1b4 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -8,7 +8,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle summary -o json | jq .bundle.deployment.history title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The raw recorded state the service holds for this deployment" trace print_state.py @@ -27,7 +27,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle summary -o json | jq .bundle.deployment.history title "The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA_SERVICE as backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The raw recorded state the service holds after the update" trace print_state.py diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index f75ba46e2fa..c5b67b15d30 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -49,6 +49,9 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": 3, "cli_version": "[CLI_VERSION]", + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": {} } @@ -112,6 +115,9 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": 3, "cli_version": "[CLI_VERSION]", + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": {} } diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 001d1e835f5..168b360744f 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -6,7 +6,7 @@ trace print_requests.py --dms //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists no resources for the deployment" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -20,7 +20,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service still lists no resources" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 8e74957520c..481042276d2 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -13,6 +13,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index d69268d8f87..ce340a1c09f 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy rm -f out.requests.txt title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the schema" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists the schema at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index df026816182..a87d280b403 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -58,6 +58,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -66,8 +69,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -86,6 +91,13 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index 1c6661c30d7..f1b8163d219 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -9,7 +9,7 @@ add_repl.py "$(git rev-parse HEAD)" COMMIT trace print_requests.py --dms //versions title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance-${UNIQUE_NAME}/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index a08dd0df054..9dfbd0af041 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -15,6 +15,9 @@ Files: 6 uploaded, 0 deleted { "plan_version": 3, "cli_version": "[CLI_VERSION]", + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.doomed": { @@ -22,8 +25,10 @@ Files: 6 uploaded, 0 deleted "new_state": { "value": { "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "2" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 8fc543ff20d..0df6cfac842 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" trace musterr $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json trace print_requests.py --dms //api/2.0/bundle title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 7f62c55528d..9a1459183f2 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -90,6 +90,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -98,8 +101,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -115,6 +120,13 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 5c565b792d1..aeb95228b7e 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -16,7 +16,7 @@ trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_U trace print_requests.py --get --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists the job, recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 3b89e18875a..33385bec6af 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -12,7 +12,8 @@ "value": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index f50dae91193..d5d68ca0c66 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Save a plan, then deploy it: records version 1" $CLI bundle plan -o json > tmp.plan.json -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json # READPLAN deploys from the saved raw plan; without it the plan is recomputed. Untraced so the # variant-dependent command line stays out of the golden (the deploy output is identical either way). $CLI bundle deploy $(readplanarg tmp.plan.json) diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 1fd34e383e4..1412ee7e0ad 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { @@ -199,6 +202,9 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index 6a8e0fe8914..965dcafdce9 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists the schema, recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-successful-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -26,7 +26,7 @@ trace $CLI bundle deploy --auto-approve trace print_requests.py --dms //api/2.0/bundle title "The plan now shows the recreate action" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists the schema at version 2 with recreate action" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index db21f2a000a..11df2223dba 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -15,6 +15,8 @@ Files: 5 uploaded, 0 deleted { "plan_version": 3, "cli_version": "[CLI_VERSION]", + "deployment_id": "[NUMID]", + "next_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -22,8 +24,10 @@ Files: 5 uploaded, 0 deleted "new_state": { "value": { "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script index aca7b030713..d801153b1c7 100644 --- a/acceptance/bundle/dms/version-never-created/script +++ b/acceptance/bundle/dms/version-never-created/script @@ -1,6 +1,6 @@ title "The first version fails, so no deployment record exists - only the node naming its ID" trace musterr $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' trace print_requests.py --dms //api/2.0/bundle --get --oneline From aadf9726048bdbae71db147bea6050528570b5ef Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 01:11:12 +0200 Subject: [PATCH 026/164] acceptance/dms: assert tombstone state via print_state.py --no-dms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add --no-dms flag to print_state.py: prints resources.json as-is without fetching state from the deployment metadata service. The existing behaviour (enriching the output with DMS-held resources) exists for non-DMS tests so their goldens stay identical whether recording is on or off; DMS tests need the opposite — to assert that the local file IS the tombstone (empty state, just the header + record_deployment_history feature flag). Update all bundle/dms test scripts to use print_state.py --no-dms. Regenerate goldens: state blocks that previously showed full resource state now show "state": {}, confirming the service holds the resources and the local file carries only the tombstone marker. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 8 +- .../bundle/dms/declined-deploy/output.txt | 89 ++++++-- acceptance/bundle/dms/declined-deploy/script | 4 +- acceptance/bundle/dms/depends-on/output.txt | 48 +---- acceptance/bundle/dms/depends-on/script | 2 +- .../bundle/dms/emptied-resource/output.txt | 12 +- acceptance/bundle/dms/emptied-resource/script | 2 +- .../bundle/dms/failed-delete/output.txt | 46 +---- acceptance/bundle/dms/failed-delete/script | 7 +- .../bundle/dms/failed-recreate/output.txt | 15 +- acceptance/bundle/dms/failed-recreate/script | 4 +- .../dms/failed-update-permissions/output.txt | 90 +-------- .../dms/failed-update-permissions/script | 4 +- .../bundle/dms/failed-update/output.txt | 26 +-- acceptance/bundle/dms/failed-update/script | 4 +- .../bundle/dms/multiple-resources/output.txt | 190 +----------------- .../bundle/dms/multiple-resources/script | 4 +- acceptance/bundle/dms/no-drift/output.txt | 95 +-------- acceptance/bundle/dms/no-drift/script | 4 +- acceptance/bundle/dms/no-resources/output.txt | 4 +- acceptance/bundle/dms/no-resources/script | 4 +- .../operation-upload-fails-once/output.txt | 26 +-- .../dms/operation-upload-fails-once/script | 4 +- acceptance/bundle/dms/provenance/output.txt | 26 +-- acceptance/bundle/dms/provenance/script | 2 +- .../bundle/dms/record-failure/output.txt | 2 +- acceptance/bundle/dms/record-failure/script | 2 +- acceptance/bundle/dms/record/output.txt | 23 +-- acceptance/bundle/dms/record/script | 2 +- acceptance/bundle/dms/stale-plan/output.txt | 23 +-- acceptance/bundle/dms/stale-plan/script | 2 +- .../bundle/dms/successful-recreate/output.txt | 26 +-- .../bundle/dms/successful-recreate/script | 4 +- 33 files changed, 146 insertions(+), 658 deletions(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index f63189c02b3..351c3083396 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -76,12 +76,18 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("-t", "--target") parser.add_argument("--backup", action="store_true") + parser.add_argument( + "--no-dms", + action="store_true", + help="Print resources.json as-is without fetching state from DMS. " + "Use in bundle/dms tests to assert the tombstone content of the file.", + ) args = parser.parse_args() for filename in get_state_files(args.target, args.backup): if not os.path.exists(filename): continue - if filename.endswith("resources.json") and records_deployment_history(): + if filename.endswith("resources.json") and records_deployment_history() and not args.no_dms: print_recorded_state(filename, args.target) else: print_file(filename) diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 552567f1cab..a4d25045708 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -97,7 +97,17 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } ->>> print_state.py +>>> print_requests.py --dms --get //api/2.0/bundle +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} + +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -106,15 +116,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "name": "dms_declined_deploy_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === A destructive change without --auto-approve is declined: this console cannot prompt @@ -141,18 +143,6 @@ Files: 4 uploaded, 0 deleted "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" -} -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" -} -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" -} === The recorded resources are unchanged by the decline: the schema is still at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json @@ -178,3 +168,58 @@ Destroy: 1 deleted === After the destroy the deployment is gone, so the summary reports no history rather than a dangling id >>> [CLI] bundle summary -o json false + +>>> print_requests.py --dms --get //api/2.0/bundle +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_DELETE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index d31b65e6129..c51686ba386 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -6,7 +6,8 @@ trace print_requests.py --dms --get //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" trace $CLI bundle plan -o json -trace print_state.py +trace print_requests.py --dms --get //api/2.0/bundle +trace print_state.py --no-dms title "A destructive change without --auto-approve is declined: this console cannot prompt" # Changing the catalog recreates the schema, which needs approval. @@ -29,3 +30,4 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle destroy --auto-approve title "After the destroy the deployment is gone, so the summary reports no history rather than a dangling id" trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' +trace print_requests.py --dms --get //api/2.0/bundle diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 83d71a1e2ad..063c1da9067 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -173,7 +173,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -182,51 +182,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.child": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "description": "depends on [NUMID]", - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "child", - "queue": { - "enabled": true - } - }, - "depends_on": [ - { - "node": "resources.jobs.parent", - "label": "${resources.jobs.parent.id}" - } - ] - }, - "resources.jobs.parent": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "parent", - "queue": { - "enabled": true - } - } - } - } + "state": {} } >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 5b721e24c6b..cfae8ab75ab 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 0b00307c1e3..8209a96b2fb 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -179,7 +179,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -188,15 +188,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_emptied_resource_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "name": "dms_emptied_resource_[UNIQUE_NAME]" - } - } - } + "state": {} } >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 8d49ae09cdd..329163ad95e 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -24,7 +24,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 4585105b85a..84e0bc5617f 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -134,7 +134,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw state after successful deploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -143,26 +143,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.stuck": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "stuck", - "queue": { - "enabled": true - } - } - } - } + "state": {} } === A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone @@ -234,7 +215,7 @@ API message: Fault injected by test. ] === The state shows the resource still exists but the operation is marked failed ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -243,24 +224,5 @@ API message: Fault injected by test. "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.stuck": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "stuck", - "queue": { - "enabled": true - } - } - } - } + "state": {} } diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index 9f3881f78bd..d7e06ace12d 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after successful deploy" -trace print_state.py +trace print_state.py --no-dms title "A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone" # The state entry survives too, so the next deploy still knows about the resource. @@ -25,8 +25,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The state shows the resource still exists but the operation is marked failed" -trace print_state.py +trace print_state.py --no-dms -# print_state.py asks the service for the resources while recording, so drain those reads instead -# of leaving them behind for the harness to diff. +# Drain any accumulated requests (plan, state reads) so the harness has nothing to diff. rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 5f8bfa55121..78bcaa59776 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -109,7 +109,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw state after successful deploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -118,16 +118,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "comment": "v1", - "name": "dms_failed_recreate_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy @@ -205,7 +196,7 @@ Files: 4 uploaded, 0 deleted [] === The raw state after failed recreate shows no resource ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 77b72b94669..976044bdba9 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after successful deploy" -trace print_state.py +trace print_state.py --no-dms title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy" # Unity Catalog has no catalog called "other", so make one to move the schema into. @@ -31,7 +31,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after failed recreate shows no resource" -trace print_state.py +trace print_state.py --no-dms title "Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists" rm -rf .databricks diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 345f048464a..9a13313ac7d 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -109,7 +109,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state after deploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -118,48 +118,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.foo.permissions": { - "__id__": "/jobs/[NUMID]", - "state": { - "object_id": "/jobs/[NUMID]", - "__embed__": [ - { - "level": "CAN_VIEW", - "user_name": "viewer@example.com" - }, - { - "level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - }, - "depends_on": [ - { - "node": "resources.jobs.foo", - "label": "${resources.jobs.foo.id}" - } - ] - } - } + "state": {} } === A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not @@ -194,7 +153,7 @@ Files: 3 uploaded, 0 deleted ] === The raw recorded state after the failure ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -203,48 +162,7 @@ Files: 3 uploaded, 0 deleted "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.foo.permissions": { - "__id__": "/jobs/[NUMID]", - "state": { - "object_id": "/jobs/[NUMID]", - "__embed__": [ - { - "level": "CAN_VIEW", - "user_name": "viewer@example.com" - }, - { - "level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - }, - "depends_on": [ - { - "node": "resources.jobs.foo", - "label": "${resources.jobs.foo.id}" - } - ] - } - } + "state": {} } >>> print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index 8b251b08b7c..2dc2aca15ff 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after deploy" -trace print_state.py +trace print_state.py --no-dms title "A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not" trace update_file.py databricks.yml "CAN_VIEW" "CAN_MANAGE" @@ -25,6 +25,6 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after the failure" -trace print_state.py +trace print_state.py --no-dms trace print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index 54358b62e95..c9d8c5568c9 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -109,7 +109,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw state after successful deploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -118,16 +118,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "comment": "v1", - "name": "dms_failed_update_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it @@ -198,7 +189,7 @@ Files: 3 uploaded, 0 deleted ] === The raw state after failed update shows the previous version ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -207,16 +198,7 @@ Files: 3 uploaded, 0 deleted "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "comment": "v1", - "name": "dms_failed_update_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script index 6e1acfe26e2..bdadee08fac 100644 --- a/acceptance/bundle/dms/failed-update/script +++ b/acceptance/bundle/dms/failed-update/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after successful deploy" -trace print_state.py +trace print_state.py --no-dms title "An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it" trace update_file.py databricks.yml "comment: v1" "comment: v2" @@ -27,7 +27,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after failed update shows the previous version" -trace print_state.py +trace print_state.py --no-dms title "Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys" rm -rf .databricks diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 79370c3d51d..688fe93bb3a 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -308,7 +308,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -317,98 +317,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.five": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "five", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.four": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "four", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.one": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "one", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.three": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "three", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.two": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "two", - "queue": { - "enabled": true - } - } - } - } + "state": {} } === Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed @@ -712,7 +621,7 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged ] === The raw recorded state after redeploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -721,96 +630,5 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.five": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "five", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.four": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "four", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.one": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "one", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.three": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "three", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.two": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "two", - "queue": { - "enabled": true - } - } - } - } + "state": {} } diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 6315788d22a..339c7973c5c 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" trace $CLI bundle deploy @@ -26,6 +26,6 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after redeploy" -trace print_state.py +trace print_state.py --no-dms rm -f out.requests.txt diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 064ba5acfeb..1c8708ac68b 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -132,7 +132,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -141,50 +141,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, - "resources.pipelines.bar": { - "__id__": "[UUID]", - "state": { - "catalog": "main", - "channel": "CURRENT", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edition": "ADVANCED", - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" - } - } - ], - "name": "bar-[UNIQUE_NAME]", - "schema": "default", - "serverless": true - } - } - } + "state": {} } === Plan without touching anything: the deployment stamp must not show as drift @@ -399,7 +356,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === The raw recorded state the service holds after the update ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -408,51 +365,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, - "resources.pipelines.bar": { - "__id__": "[UUID]", - "state": { - "catalog": "main", - "channel": "CURRENT", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "2" - }, - "development": true, - "edition": "ADVANCED", - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" - } - } - ], - "name": "bar-[UNIQUE_NAME]", - "schema": "default", - "serverless": true - } - } - } + "state": {} } === Plan again: no residual drift, and the version stamps are ignored diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 64455e8f1b4..67e8172e06b 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -11,7 +11,7 @@ title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its ba trace $CLI bundle plan -o json title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms # Only deploy stamps deployment.deployment_id. Plan sees it in state but not in config, # so it must ignore it rather than report spurious changes. @@ -30,7 +30,7 @@ title "The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA trace $CLI bundle plan -o json title "The raw recorded state the service holds after the update" -trace print_state.py +trace print_state.py --no-dms title "Plan again: no residual drift, and the version stamps are ignored" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index c5b67b15d30..ee9b3127061 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -61,7 +61,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged [] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms === Redeploy: the deployment is resolved from that node, so no second deployment is created >>> [CLI] bundle deploy @@ -127,4 +127,4 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged [] === The raw recorded state after redeploy ->>> print_state.py +>>> print_state.py --no-dms diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 168b360744f..1593aee522f 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -13,7 +13,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy @@ -27,6 +27,6 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after redeploy" -trace print_state.py +trace print_state.py --no-dms rm -f out.requests.txt diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 481042276d2..9060ab58706 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -56,7 +56,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state after deploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -65,16 +65,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_upload_fails_once_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "comment": "v1", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one @@ -107,7 +98,7 @@ Files: 4 uploaded, 0 deleted ] === The raw recorded state after the failure ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -116,16 +107,7 @@ Files: 4 uploaded, 0 deleted "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "dms_other_[UNIQUE_NAME]", - "comment": "v1", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } >>> print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index ce340a1c09f..73e895433ab 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after deploy" -trace print_state.py +trace print_state.py --no-dms title "A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one" trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 @@ -28,6 +28,6 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after the failure" -trace print_state.py +trace print_state.py --no-dms trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index a87d280b403..437af92448b 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -145,7 +145,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py -t dev +>>> print_state.py --no-dms -t dev { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -154,27 +154,5 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 4, - "name": "[dev [USERNAME]] foo", - "queue": { - "enabled": true - }, - "tags": { - "dev": "[USERNAME]" - } - } - } - } + "state": {} } diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index f1b8163d219..ab20da12355 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -21,7 +21,7 @@ title "The service lists the deployed job" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py -t dev +trace print_state.py --no-dms -t dev # The deploy uploads files in a nondeterministic order; only the requests above are # asserted. diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 9dfbd0af041..7568ec343dd 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -114,7 +114,7 @@ Files: 6 uploaded, 0 deleted >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [] ->>> print_state.py +>>> print_state.py --no-dms === Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 0df6cfac842..d3061bd3049 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -15,7 +15,7 @@ trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' -trace print_state.py +trace print_state.py --no-dms title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" rm -rf .databricks diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 9a1459183f2..263099877d5 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -158,7 +158,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -167,24 +167,5 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - } - } + "state": {} } diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index aeb95228b7e..ccbcd025d9b 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -23,5 +23,5 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms rm -f out.requests.txt diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 33385bec6af..fa4f047b28e 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -42,7 +42,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -51,26 +51,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - } - } + "state": {} } === A saved plan carries the deployment id and the versions it targets (last 1, next 2) diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index d5d68ca0c66..3e189942281 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -11,7 +11,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" trace $CLI bundle plan -o json | jq '{deployment_id, last_version_id, next_version_id}' diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 1412ee7e0ad..69fbc28396b 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -109,7 +109,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -118,16 +118,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "comment": "v1", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === Recreate writes state twice - the entry is dropped, then the new resource is saved @@ -245,7 +236,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged ] === The raw recorded state after recreate ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -254,16 +245,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "dms_other_[UNIQUE_NAME]", - "comment": "v1", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === Destroy: the delete is recorded with the id and no state diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index 965dcafdce9..457164d584b 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms title "Recreate writes state twice - the entry is dropped, then the new resource is saved" # One operation per resource per version; both writes land on the same one. @@ -32,7 +32,7 @@ title "The service lists the schema at version 2 with recreate action" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after recreate" -trace print_state.py +trace print_state.py --no-dms title "Destroy: the delete is recorded with the id and no state" trace $CLI bundle destroy --auto-approve From 5e7a84648f127a640e27c3e249ab3c0f92780b11 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 01:20:01 +0200 Subject: [PATCH 027/164] acceptance/dms: assert tombstone state via print_state.py --no-dms; pre-deploy plan in depends-on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add --no-dms flag to print_state.py: prints resources.json as-is without fetching state from the deployment metadata service. The existing behaviour (enriching the output with DMS-held resources) exists for non-DMS tests so their goldens stay identical whether recording is on or off; DMS tests need the opposite — to assert that the local file IS the tombstone (empty state, just the header + record_deployment_history feature flag). Update all bundle/dms test scripts to use print_state.py --no-dms. Regenerate goldens: state blocks now show "state": {}, confirming the service holds the resources and the local file carries only the tombstone. Also add a bundle plan -o json call before the first deploy in depends-on, showing that no deployment_id or version fields appear before any deploy exists. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 58 +++++++++++++++++++++ acceptance/bundle/dms/depends-on/script | 3 ++ 2 files changed, 61 insertions(+) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 063c1da9067..4640a2738f4 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -1,4 +1,62 @@ +=== Plan before first deploy: no deployment yet, so no deployment_id or version fields +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.child": { + "depends_on": [ + { + "node": "resources.jobs.parent", + "label": "${resources.jobs.parent.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "description": "depends on ${resources.jobs.parent.id}", + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "child", + "queue": { + "enabled": true + } + }, + "vars": { + "description": "depends on ${resources.jobs.parent.id}" + } + } + }, + "resources.jobs.parent": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "parent", + "queue": { + "enabled": true + } + } + } + } + } +} + === Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/files... diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index cfae8ab75ab..895059dd8c9 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -1,5 +1,8 @@ envsubst < databricks.yml.tmpl > databricks.yml +title "Plan before first deploy: no deployment yet, so no deployment_id or version fields" +trace $CLI bundle plan -o json + title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations From ab79f3b6e9bc083f03f7d5139f56ecb0b6a372b5 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 01:36:11 +0200 Subject: [PATCH 028/164] acceptance/dms: put record_deployment_history feature in existing-state WAL header The leftover WAL now carries the feature flag, so the test proves the WAL is discarded rather than replayed: if it were merged the feature would leak into the refused deploys, but the refusals still stand and the final fresh deploy's feature comes from config, with resources.json.wal gone (--expect 0). Co-authored-by: Isaac --- acceptance/bundle/dms/existing-state/script | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 1b1a9401965..a5be325e72a 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -15,13 +15,15 @@ rm -rf .databricks musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands" +# The WAL header carries the record_deployment_history feature: if it were replayed, the +# feature would merge into the state. The refusal below proves it is discarded, not read. mkdir -p .databricks/bundle/default -printf '{"lineage":"stale","serial":99,"state_version":2}\n' > .databricks/bundle/default/resources.json.wal +printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"record_deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed" trace $CLI bundle destroy --auto-approve -printf '{"lineage":"stale","serial":99,"state_version":2}\n' > .databricks/bundle/default/resources.json.wal +printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"record_deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --oneline trace find.py 'resources[.]json[.]wal' --expect 0 From bbade2627c33a81d5a4463e573a8eccf0483cec2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 01:49:40 +0200 Subject: [PATCH 029/164] acceptance/dms: drop two racy mid-flow titles in deployment-metadata-change These two section-header titles printed to stdout immediately before a command whose trace line goes to stderr. On cloud the two streams interleave nondeterministically and the title line is dropped, so the golden never matched. The surrounding >>> commands and their output already convey the state, so the titles are removed rather than fixing the shared title() helper (which would add a blank line after every title in every acceptance golden). Co-authored-by: Isaac --- acceptance/bundle/dms/deployment-metadata-change/output.txt | 2 -- acceptance/bundle/dms/deployment-metadata-change/script | 2 -- 2 files changed, 4 deletions(-) diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 99c1c51f301..b3bc662ac7b 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -157,7 +157,6 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged } } -=== The metadata changed but the schema did not: still one deployment, recorded at version 1 >>> [CLI] bundle plan -t prod -o json Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed @@ -243,7 +242,6 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged } } -=== The no-op redeploy leaves the schema at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ { diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script index 5d650aaa341..d59e1365842 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/script +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -14,7 +14,6 @@ trace update_file.py databricks.yml dms-metadata-before dms-metadata-after trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle -title "The metadata changed but the schema did not: still one deployment, recorded at version 1" trace $CLI bundle plan -t prod -o json trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' @@ -22,6 +21,5 @@ title "Deploy again without touching anything: the record already says this, so trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle -title "The no-op redeploy leaves the schema at version 1" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' rm -f out.requests.txt From 55b125677440a17f03d3a3f4920374b9ab9101e5 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:01:26 +0200 Subject: [PATCH 030/164] acceptance/dms: drop redundant resources.json guard in print_state.py In DMS/direct mode only resources.json exists among the state files, so the filename check added nothing; the records_deployment_history() + --no-dms gate alone selects the recorded-state path. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 351c3083396..6055a3c5ecf 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -87,7 +87,7 @@ def main(): for filename in get_state_files(args.target, args.backup): if not os.path.exists(filename): continue - if filename.endswith("resources.json") and records_deployment_history() and not args.no_dms: + if records_deployment_history() and not args.no_dms: print_recorded_state(filename, args.target) else: print_file(filename) From 41b1adfc98b6e8ce904752f496ae500f1e0922b0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:04:52 +0200 Subject: [PATCH 031/164] acceptance/dms: assert failed delete records its error via list-operations Add a ListOperations handler to the testserver (GET /api/2.0/bundle/deployments/{id}/versions/{version}/operations) mirroring the service, and call bundle-deployments list-operations in failed-delete to assert the failed delete operation carries its error message and FAILED status. Co-authored-by: Isaac --- .../bundle/dms/failed-delete/output.txt | 11 ++++++ acceptance/bundle/dms/failed-delete/script | 3 ++ libs/testserver/bundledeployments.go | 37 +++++++++++++++++++ libs/testserver/handlers.go | 3 ++ 4 files changed, 54 insertions(+) diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 84e0bc5617f..2d06337ff31 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -214,6 +214,17 @@ API message: Fault injected by test. } ] +=== The failed operation records the error message +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[NUMID]/versions/2 -o json +[ + { + "action_type": "OPERATION_ACTION_TYPE_DELETE", + "error_message": "deleting id=[NUMID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "resource_key": "jobs.stuck", + "status": "OPERATION_STATUS_FAILED" + } +] + === The state shows the resource still exists but the operation is marked failed >>> print_state.py --no-dms { diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index d7e06ace12d..08d949fd198 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -24,6 +24,9 @@ title "After failure, the service shows the operation failed" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +title "The failed operation records the error message" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -S 'map({resource_key, action_type, status, error_message})' + title "The state shows the resource still exists but the operation is marked failed" trace print_state.py --no-dms diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go index 3545036297a..4fccd96c700 100644 --- a/libs/testserver/bundledeployments.go +++ b/libs/testserver/bundledeployments.go @@ -569,6 +569,43 @@ func (s *FakeWorkspace) ListResources(deploymentID string) Response { return Response{Body: bundledeployments.ListResourcesResponse{Resources: resources}} } +func (s *FakeWorkspace) ListOperations(deploymentID, versionID string) Response { + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + _, ok = d.Versions[versionID] + if !ok { + return dmsNotFound("version " + versionID) + } + + // Collect operations for this version by filtering the operation name prefix. + // Operation names are: deployments/{deployment_id}/versions/{version_id}/operations/{resource_key} + versionPrefix := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + var ops []*bundledeployments.Operation + for name, op := range d.Operations { + if strings.HasPrefix(name, versionPrefix) { + ops = append(ops, op) + } + } + + // Sort by resource key so the response order is deterministic. + slices.SortFunc(ops, func(a, b *bundledeployments.Operation) int { + return strings.Compare(a.ResourceKey, b.ResourceKey) + }) + + response := bundledeployments.ListOperationsResponse{ + Operations: make([]bundledeployments.Operation, len(ops)), + } + for i, op := range ops { + response.Operations[i] = *op + } + return Response{Body: response} +} + // leavesLiveResource reports whether a failed operation of this type leaves a resource behind // that it still has to describe. A create leaves nothing, and a recreate already deleted. func leavesLiveResource(action bundledeployments.OperationActionType) bool { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 33a742cc2ce..9fe87c64188 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -307,6 +307,9 @@ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/resources", func(req Request) any { return req.Workspace.ListResources(req.Vars["deployment_id"]) }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations", func(req Request) any { + return req.Workspace.ListOperations(req.Vars["deployment_id"], req.Vars["version_id"]) + }) server.Handle("POST", "/api/2.2/jobs/delete", func(req Request) any { var request jobs.DeleteJob From 4563cbc0f81405c9aa34ec65fddd73fbad3d66bb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:25:48 +0200 Subject: [PATCH 032/164] bundle/direct/dstate: restore original migrateState feature-version comment Reverts a comment rewording; the longer original phrasing is clearer about why a featureStateVersion state with no features is accepted without running the migrations and its on-disk version left as-is. Co-authored-by: Isaac --- bundle/direct/dstate/migrate.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 6a891900c36..e4d21a7054a 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -15,10 +15,12 @@ import ( // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // A featureStateVersion state carries a feature list this CLI does not write (see the - // featureStateVersion doc comment). With no features it is equivalent to currentStateVersion, so - // accept it and leave the on-disk version as-is; with any feature it depends on capabilities this - // CLI lacks, so refuse it and tell the user to upgrade. + // featureStateVersion states carry a feature list this CLI does not yet write or + // understand (see the featureStateVersion doc comment). A featureStateVersion + // state with no features is equivalent to currentStateVersion, so accept it and + // return without running the migrations below, leaving the on-disk version at + // featureStateVersion rather than flipping it down. One that records any feature + // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. if db.StateVersion == featureStateVersion { if len(db.Features) == 0 { return nil From 5435321e46d18f5ee52b9da69a869abfe8b3f1b8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:26:01 +0200 Subject: [PATCH 033/164] acceptance/dms: failed-recreate fails naturally via nonexistent catalog (cloud-enabled) Drop the fault.py injection; point the recreated schema at a catalog that does not exist so the create half of the recreate fails on its own. The testserver now models catalog existence: SchemasCreate returns 404 CATALOG_DOES_NOT_EXIST when the parent catalog is absent, and the fake seeds the "main" catalog so the common case (schemas in main) keeps working. This makes the failure identical locally and on a real workspace, so the test runs with Cloud = true. Co-authored-by: Isaac --- .../bundle/dms/failed-recreate/out.requests.txt | 7 ------- .../bundle/dms/failed-recreate/out.test.toml | 2 +- acceptance/bundle/dms/failed-recreate/output.txt | 16 +++++++--------- acceptance/bundle/dms/failed-recreate/script | 10 +++------- acceptance/bundle/dms/failed-recreate/test.toml | 9 ++++----- libs/testserver/catalogs_test.go | 7 ++++--- libs/testserver/fake_workspace.go | 7 ++++++- libs/testserver/schemas.go | 12 ++++++++++++ 8 files changed, 37 insertions(+), 33 deletions(-) delete mode 100644 acceptance/bundle/dms/failed-recreate/out.requests.txt diff --git a/acceptance/bundle/dms/failed-recreate/out.requests.txt b/acceptance/bundle/dms/failed-recreate/out.requests.txt deleted file mode 100644 index eb169f26a0c..00000000000 --- a/acceptance/bundle/dms/failed-recreate/out.requests.txt +++ /dev/null @@ -1,7 +0,0 @@ -{ - "method": "DELETE", - "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", - "q": { - "force": "true" - } -} diff --git a/acceptance/bundle/dms/failed-recreate/out.test.toml b/acceptance/bundle/dms/failed-recreate/out.test.toml index 9921e91a794..23c07f70dca 100644 --- a/acceptance/bundle/dms/failed-recreate/out.test.toml +++ b/acceptance/bundle/dms/failed-recreate/out.test.toml @@ -1,3 +1,3 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 78bcaa59776..a4a818bf7bc 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -122,23 +122,21 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy ->>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] - ->>> fault.py POST /api/2.1/unity-catalog/schemas 400 0 1 INVALID_PARAMETER_VALUE +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_nonexistent_[UNIQUE_NAME] >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files... This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: recreate resources.schemas.foo -Error: cannot recreate resources.schemas.foo: Fault injected by test. (400 INVALID_PARAMETER_VALUE) +Error: cannot recreate resources.schemas.foo: Catalog 'dms_nonexistent_[UNIQUE_NAME]' does not exist. (404 CATALOG_DOES_NOT_EXIST) Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/schemas -HTTP Status: 400 Bad Request -API error_code: INVALID_PARAMETER_VALUE -API message: Fault injected by test. +HTTP Status: 404 Not Found +API error_code: CATALOG_DOES_NOT_EXIST +API message: Catalog 'dms_nonexistent_[UNIQUE_NAME]' does not exist. -Files: 4 uploaded, 0 deleted +Files: 3 uploaded, 0 deleted >>> print_requests.py --dms //api/2.0/bundle { @@ -178,7 +176,7 @@ Files: 4 uploaded, 0 deleted "update_mask": "state,error_message,status" }, "body": { - "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "error_message": "Catalog 'dms_nonexistent_[UNIQUE_NAME]' does not exist. (404 CATALOG_DOES_NOT_EXIST)", "sequence_id": "1", "status": "OPERATION_STATUS_FAILED" } diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 976044bdba9..8d0b7ddeef1 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -15,13 +15,9 @@ title "The raw state after successful deploy" trace print_state.py --no-dms title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy" -# Unity Catalog has no catalog called "other", so make one to move the schema into. -other_catalog="dms_other_${UNIQUE_NAME}" -$CLI catalogs create "$other_catalog" &> LOG.catalog -trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT -trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" -# Fail only the create that follows the delete; the delete itself still goes through. -trace fault.py "POST /api/2.1/unity-catalog/schemas" 400 0 1 INVALID_PARAMETER_VALUE +# Point the schema at a catalog that does not exist: the delete goes through, then the +# create fails naturally against both the fake and a real workspace (no fault injection). +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: dms_nonexistent_${UNIQUE_NAME}" trace musterr $CLI bundle deploy --auto-approve # Not sorted: the point of this test is the order the operations are recorded in. trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/failed-recreate/test.toml b/acceptance/bundle/dms/failed-recreate/test.toml index 8a850ba07c5..25c802e47a0 100644 --- a/acceptance/bundle/dms/failed-recreate/test.toml +++ b/acceptance/bundle/dms/failed-recreate/test.toml @@ -1,5 +1,4 @@ -# Local only: the failure is injected, and fault.py registers its rule on the fake, which -# a cloud run does not have. Simulating it for real instead - a catalog or principal that -# does not exist - does not work either: the fake accepts both, so the case would stop -# failing locally. Making it portable needs the fake to model that existence. -Cloud = false +# The recreate fails naturally by pointing the schema at a catalog that does not +# exist - the fake models catalog existence, so this fails the same way locally and +# on cloud without fault injection. +Cloud = true diff --git a/libs/testserver/catalogs_test.go b/libs/testserver/catalogs_test.go index eb05ab8ec56..b699361577c 100644 --- a/libs/testserver/catalogs_test.go +++ b/libs/testserver/catalogs_test.go @@ -17,9 +17,10 @@ func TestCatalogsCreate_RejectsEmptyName(t *testing.T) { response := workspace.CatalogsCreate(Request{Body: []byte(`{"name": ""}`)}) assert.Equal(t, 400, response.StatusCode) - // A stored-but-unreadable catalog is the original bug. Asserted before the - // require below so it is still reported when the rejection is missing. - assert.Empty(t, workspace.Catalogs) + // A stored-but-unreadable catalog under the empty key is the original bug. Asserted + // before the require below so it is still reported when the rejection is missing. + // (The workspace seeds "main", so the map is not empty; the empty key must be absent.) + assert.NotContains(t, workspace.Catalogs, "") body, ok := response.Body.(map[string]string) require.True(t, ok) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index b544c17538e..c9026e20fc2 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -467,7 +467,12 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { PipelineUpdates: map[string]bool{}, Monitors: map[string]catalog.MonitorInfo{}, Apps: map[string]apps.App{}, - Catalogs: map[string]catalog.CatalogInfo{}, + // Seed the "main" catalog: it exists on every real workspace, and schema + // creation now checks the catalog exists, so tests that deploy schemas into + // main (the common case) keep working without creating it first. + Catalogs: map[string]catalog.CatalogInfo{ + "main": {Name: "main", FullName: "main", CatalogType: catalog.CatalogTypeManagedCatalog}, + }, ExternalLocations: map[string]catalog.ExternalLocationInfo{}, Schemas: map[string]catalog.SchemaInfo{}, RegisteredModels: map[string]catalog.RegisteredModelInfo{}, diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 6880a55cb8f..92bc0374734 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -29,6 +29,18 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { } } + // UC rejects a schema whose parent catalog does not exist; model it so a deploy into a + // missing catalog fails the same way locally as on cloud. + if _, ok := s.Catalogs[schema.CatalogName]; !ok { + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{ + "error_code": "CATALOG_DOES_NOT_EXIST", + "message": fmt.Sprintf("Catalog '%s' does not exist.", schema.CatalogName), + }, + } + } + // UC normalizes schema names to lowercase. schema.Name = strings.ToLower(schema.Name) schema.FullName = schema.CatalogName + "." + schema.Name From ae90e569c1e9daad543fdb8bd1930f144131c0c9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:32:37 +0200 Subject: [PATCH 034/164] bundle/direct/dstate: restore TestEmptyFeatureStateAcceptedWithoutFlippingVersion Re-add the test pinning the featureStateVersion special case: a v3 state with no features is accepted and left at v3 (not flipped to currentStateVersion), while a v3 state recording any feature is refused with the upgrade hint. It is the forcing function for the deferred version bump. Co-authored-by: Isaac --- bundle/direct/dstate/state_test.go | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index f7d605e39f8..57ccd4d3e51 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -182,6 +182,38 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { mustFinalize(t, &recovered) } +// TestEmptyFeatureStateAcceptedWithoutFlippingVersion pins the special case that a +// featureStateVersion state with no features is accepted as-is — the on-disk version +// is left at featureStateVersion, not flipped down to currentStateVersion — and that +// a featureStateVersion state recording any feature is refused. This is scaffolding +// for the deferred version bump, special-cased to featureStateVersion only (see the +// featureStateVersion doc comment). +// +// When the baseline is actually bumped to featureStateVersion, this special case must +// go away. This test is the forcing function: it fails once featureStateVersion is +// removed, making the author decide what the post-bump behavior should be. +func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { + // The special case applies to featureStateVersion (3) only. + require.Equal(t, 2, currentStateVersion, "when currentStateVersion is bumped, remove featureStateVersion and this special case") + require.Equal(t, 3, featureStateVersion) + + empty := &Database{Header: Header{StateVersion: featureStateVersion}} + require.NoError(t, migrateState(empty)) + assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") + + // v3 that records a feature is refused: this CLI does not understand features. + withFeature := &Database{Header: Header{ + StateVersion: featureStateVersion, + Features: map[string]struct{}{"future_feature": {}}, + }} + err := migrateState(withFeature) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires features this CLI does not support") + assert.Contains(t, err.Error(), "future_feature") + assert.Contains(t, err.Error(), "upgrade to the latest CLI version") + assert.Contains(t, err.Error(), featuresDocURL) +} + func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") From 6a21a0eda4ae1b8723a5ef495a03137d3900d582 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:35:36 +0200 Subject: [PATCH 035/164] bundle/direct/dstate: use lock/defer-unlock in RecordingError RecordingError only reads operationBuffer and calls buf.Err() (a quick read on the buffer's own mutex), so holding db.mu across it via defer is safe and reads cleaner. CompleteVersion keeps its manual unlock: the network work after the snapshot must run outside the lock. Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index f43503c7a05..6a41661c516 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -208,12 +208,11 @@ func (db *DeploymentState) RecordingError() error { return nil } db.mu.Lock() - buf := db.operationBuffer - db.mu.Unlock() - if buf == nil { + defer db.mu.Unlock() + if db.operationBuffer == nil { return nil } - return buf.Err() + return db.operationBuffer.Err() } // CompleteVersion marks the recorded version done, reporting whether it completed here. A no-op From 980659959d078b455ea922fcf4bbeaa59ef1f96d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:42:07 +0200 Subject: [PATCH 036/164] acceptance/dms: cover bind/unbind refusal while recording deployment history New test showing bind and unbind against a bundle that records deployment history. Before any deploy there is no recorded state, so bind just fails to find the workspace resource and unbind is a no-op. Once a deployment is recorded, both are refused ("bind/unbind is not supported for a bundle that records deployment history"): the service owns the resource set, so adopting or releasing a resource by editing local state is not allowed. Local-only, since the refusal is a client-side gate. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 9 ++++++++ .../dms/bind-unbind-unsupported/out.test.toml | 3 +++ .../dms/bind-unbind-unsupported/output.txt | 22 +++++++++++++++++++ .../bundle/dms/bind-unbind-unsupported/script | 18 +++++++++++++++ .../dms/bind-unbind-unsupported/test.toml | 3 +++ 5 files changed, 55 insertions(+) create mode 100644 acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/bind-unbind-unsupported/out.test.toml create mode 100644 acceptance/bundle/dms/bind-unbind-unsupported/output.txt create mode 100644 acceptance/bundle/dms/bind-unbind-unsupported/script create mode 100644 acceptance/bundle/dms/bind-unbind-unsupported/test.toml diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl b/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl new file mode 100644 index 00000000000..7c9830e927c --- /dev/null +++ b/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-bind-unbind-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/out.test.toml b/acceptance/bundle/dms/bind-unbind-unsupported/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/bind-unbind-unsupported/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt new file mode 100644 index 00000000000..484998ca590 --- /dev/null +++ b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt @@ -0,0 +1,22 @@ + +=== Before any deploy there is no recorded state: bind just fails to find the workspace resource, and unbind is a no-op +>>> musterr [CLI] bundle deployment bind foo 123 +Error: failed to fetch the resource, err: Job 123 does not exist. + +>>> [CLI] bundle deployment unbind foo + +=== Deploy so the bundle records the job with the deployment metadata service +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== With deployment history recorded, both bind and unbind are refused +>>> musterr [CLI] bundle deployment bind foo [FOO_ID] +Error: bind is not supported for a bundle that records deployment history + + +>>> musterr [CLI] bundle deployment unbind foo +Error: unbind is not supported for a bundle that records deployment history + diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/script b/acceptance/bundle/dms/bind-unbind-unsupported/script new file mode 100644 index 00000000000..0bed434163a --- /dev/null +++ b/acceptance/bundle/dms/bind-unbind-unsupported/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Before any deploy there is no recorded state: bind just fails to find the workspace resource, and unbind is a no-op" +trace musterr $CLI bundle deployment bind foo 123 +trace $CLI bundle deployment unbind foo + +title "Deploy so the bundle records the job with the deployment metadata service" +trace $CLI bundle deploy + +# bind and unbind adopt or release a resource by editing local state directly. A recorded +# deployment keeps its resource set in the service, so both are refused: bind hits the gate +# after fetching the resource, unbind when it opens the recording state without recording. +title "With deployment history recorded, both bind and unbind are refused" +job_id=$(read_id.py foo) +trace musterr $CLI bundle deployment bind foo "$job_id" +trace musterr $CLI bundle deployment unbind foo + +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/test.toml b/acceptance/bundle/dms/bind-unbind-unsupported/test.toml new file mode 100644 index 00000000000..8bc2d5e8cf8 --- /dev/null +++ b/acceptance/bundle/dms/bind-unbind-unsupported/test.toml @@ -0,0 +1,3 @@ +# bind/unbind refusal is a client-side gate on the recording config; it needs no real +# workspace, so this stays local-only rather than inheriting the suite's cloud run. +Cloud = false From 8d65ab2523d3bd75e7c34ed46445502f72fbe9f4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:53:27 +0200 Subject: [PATCH 037/164] libs/dms: fold latestState update into record under one lock RecordOperation and RecordFailure each took db.mu separately to touch latestState and then again inside record to touch pending, so the two maps could be observed mid-update. Move the latestState read/write into record, guarded by the same lock as the pending update: a state write (isStateWrite) records the latest state, a failure merges its status/message onto that recorded state (or falls back to the passed update). RecordOperation and RecordFailure no longer lock directly. Co-authored-by: Isaac --- libs/dms/operation_buffer.go | 36 +++++++++++++++---------------- libs/dms/operation_buffer_test.go | 6 +++--- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go index ac5cab63dd8..11fe07fc471 100644 --- a/libs/dms/operation_buffer.go +++ b/libs/dms/operation_buffer.go @@ -76,13 +76,7 @@ func (b *OperationBuffer) RecordOperation(ctx context.Context, resourceKey strin return } - // Remember the state this write recorded, kept past the writer taking it, so a later failure - // for this resource keeps this state rather than the pre-deploy one (see RecordFailure). - b.mu.Lock() - b.latestState[resourceKey] = update - b.mu.Unlock() - - b.record(resourceKey, update) + b.record(resourceKey, update, true, nil) } // RecordFailure records that a resource did not apply, so the history says why rather than @@ -92,22 +86,26 @@ func (b *OperationBuffer) RecordOperation(ctx context.Context, resourceKey strin // pair the new id with the old state. state is the fallback, used only when no write recorded // anything: the operation failed before it saved state. func (b *OperationBuffer) RecordFailure(resourceKey, resourceID string, state json.RawMessage, cause error) { - b.mu.Lock() - latest, recorded := b.latestState[resourceKey] - b.mu.Unlock() - - if recorded { - b.record(resourceKey, latest.Merge(failureUpdate(cause))) - return - } - - b.record(resourceKey, NewFailureUpdate(resourceID, state, cause)) + // The passed update is the fallback used only when no write recorded state yet; record + // otherwise merges the failure onto the state the last write recorded (see below). + b.record(resourceKey, NewFailureUpdate(resourceID, state, cause), false, cause) } // record makes update the one waiting for resourceKey, waiting itself while the queue is -// full. Recording after Drain panics, so every caller must return before Drain. -func (b *OperationBuffer) record(resourceKey string, update OperationUpdate) { +// full. isStateWrite records update as the latest state for this resource, kept past the +// writer taking it from pending so a later failure keeps it (see the latestState field); +// a failure (isStateWrite false) instead merges its status and message onto that recorded +// state when there is one, so it keeps the state - which the service may already hold - +// rather than the pre-deploy one, and after a rename pairs the new id with the old state. +// The latestState read/write shares this lock with the pending update so the two never +// disagree. Recording after Drain panics, so every caller must return before Drain. +func (b *OperationBuffer) record(resourceKey string, update OperationUpdate, isStateWrite bool, cause error) { b.mu.Lock() + if isStateWrite { + b.latestState[resourceKey] = update + } else if latest, recorded := b.latestState[resourceKey]; recorded { + update = latest.Merge(failureUpdate(cause)) + } waiting, queued := b.pending[resourceKey] if queued { update = waiting.Merge(update) diff --git a/libs/dms/operation_buffer_test.go b/libs/dms/operation_buffer_test.go index b4fba8ce124..cb01c8b57d5 100644 --- a/libs/dms/operation_buffer_test.go +++ b/libs/dms/operation_buffer_test.go @@ -38,8 +38,8 @@ func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { // Two writes for one resource with nothing draining. They carry the resource's full // state, so only the newest needs to go: one slot in the queue, not two. - s.record("resources.jobs.foo", stateUpdate(t, "v1")) - s.record("resources.jobs.foo", stateUpdate(t, "v2")) + s.record("resources.jobs.foo", stateUpdate(t, "v1"), true, nil) + s.record("resources.jobs.foo", stateUpdate(t, "v2"), true, nil) assert.Len(t, s.queue, 1) update, ok := s.take("resources.jobs.foo") @@ -48,7 +48,7 @@ func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { // Taken means a request has it, and an in-flight request cannot be recalled, so the next // write gets its own slot rather than joining it. - s.record("resources.jobs.foo", stateUpdate(t, "v3")) + s.record("resources.jobs.foo", stateUpdate(t, "v3"), true, nil) assert.Len(t, s.queue, 2) update, ok = s.take("resources.jobs.foo") require.True(t, ok) From c0a2e503db09a03f10986a537e970b8656bb33c5 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:54:05 +0200 Subject: [PATCH 038/164] acceptance/dms: print plan header via jq + readable plan instead of full -o json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a dms_plan helper (bundle/dms/script.prepare) that prints the plan JSON with the per-resource plan body dropped (jq del(.plan)) — leaving the header fields cli_version, lineage, serial, deployment_id, next/last_version_id, storage_backend — then runs bundle plan again for the readable summary. Replace every full `bundle plan -o json` dump in the bundle/dms tests with it. The per-resource plan carried remote_state that diverges between the fake and a real workspace (e.g. a serverless pipeline's effective_serverless_compute_id), which broke no-drift on cloud; dropping it keeps local and cloud identical while still asserting the DMS header fields and the human-readable plan. Co-authored-by: Isaac --- .../bundle/dms/declined-deploy/output.txt | 38 +- acceptance/bundle/dms/declined-deploy/script | 2 +- acceptance/bundle/dms/depends-on/output.txt | 170 +----- acceptance/bundle/dms/depends-on/script | 4 +- .../dms/deployment-metadata-change/output.txt | 66 +-- .../dms/deployment-metadata-change/script | 4 +- .../bundle/dms/emptied-resource/output.txt | 30 +- acceptance/bundle/dms/emptied-resource/script | 2 +- .../bundle/dms/existing-state/output.txt | 47 +- acceptance/bundle/dms/existing-state/script | 2 +- .../bundle/dms/failed-delete/output.txt | 56 +- acceptance/bundle/dms/failed-delete/script | 2 +- .../bundle/dms/failed-recreate/output.txt | 31 +- acceptance/bundle/dms/failed-recreate/script | 2 +- .../dms/failed-update-permissions/output.txt | 78 +-- .../dms/failed-update-permissions/script | 2 +- .../bundle/dms/failed-update/output.txt | 31 +- acceptance/bundle/dms/failed-update/script | 2 +- .../bundle/dms/multiple-resources/output.txt | 504 +----------------- .../bundle/dms/multiple-resources/script | 4 +- acceptance/bundle/dms/no-drift/output.txt | 193 +------ acceptance/bundle/dms/no-drift/script | 4 +- acceptance/bundle/dms/no-resources/output.txt | 20 +- acceptance/bundle/dms/no-resources/script | 4 +- .../operation-upload-fails-once/output.txt | 31 +- .../dms/operation-upload-fails-once/script | 2 +- acceptance/bundle/dms/provenance/output.txt | 59 +- acceptance/bundle/dms/provenance/script | 2 +- .../bundle/dms/record-failure/output.txt | 42 +- acceptance/bundle/dms/record-failure/script | 2 +- acceptance/bundle/dms/record/output.txt | 56 +- acceptance/bundle/dms/record/script | 2 +- acceptance/bundle/dms/script.prepare | 9 + acceptance/bundle/dms/stale-plan/output.txt | 28 +- acceptance/bundle/dms/stale-plan/script | 2 +- .../bundle/dms/successful-recreate/output.txt | 62 +-- .../bundle/dms/successful-recreate/script | 4 +- .../dms/version-never-created/output.txt | 31 +- .../bundle/dms/version-never-created/script | 2 +- 39 files changed, 167 insertions(+), 1465 deletions(-) create mode 100644 acceptance/bundle/dms/script.prepare diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index a4d25045708..2aa56f9ab78 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -69,34 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_declined_deploy_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + >>> print_requests.py --dms --get //api/2.0/bundle { "method": "GET", @@ -106,6 +84,14 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} >>> print_state.py --no-dms { diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index c51686ba386..56a33fea036 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms --get //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" -trace $CLI bundle plan -o json +dms_plan trace print_requests.py --dms --get //api/2.0/bundle trace print_state.py --no-dms diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 4640a2738f4..ae9e3cda49e 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -4,59 +4,15 @@ { "plan_version": 3, "cli_version": "[CLI_VERSION]", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.child": { - "depends_on": [ - { - "node": "resources.jobs.parent", - "label": "${resources.jobs.parent.id}" - } - ], - "action": "create", - "new_state": { - "value": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "description": "depends on ${resources.jobs.parent.id}", - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "child", - "queue": { - "enabled": true - } - }, - "vars": { - "description": "depends on ${resources.jobs.parent.id}" - } - } - }, - "resources.jobs.parent": { - "action": "create", - "new_state": { - "value": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "parent", - "queue": { - "enabled": true - } - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +create jobs.child +create jobs.parent + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + === Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/files... @@ -105,116 +61,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.child": { - "depends_on": [ - { - "node": "resources.jobs.parent", - "label": "${resources.jobs.parent.id}" - } - ], - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "description": "depends on [NUMID]", - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "child", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.parent": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][1], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "parent", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + === The service lists both jobs, each recorded at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 895059dd8c9..8c0f865305b 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -1,14 +1,14 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Plan before first deploy: no deployment yet, so no deployment_id or version fields" -trace $CLI bundle plan -o json +dms_plan title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service lists both jobs, each recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-depends-on-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index b3bc662ac7b..8d1199d14c1 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -69,34 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_metadata_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_metadata_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ { @@ -172,34 +150,18 @@ A common practice is to use a username or principal name in this path, i.e. use "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_metadata_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_metadata_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan -t prod +Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed + +A common practice is to use a username or principal name in this path, i.e. use + + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ { diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script index d59e1365842..e3bc9895307 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/script +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" -trace $CLI bundle plan -o json +dms_plan deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-metadata-pinned-${UNIQUE_NAME}/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' @@ -14,7 +14,7 @@ trace update_file.py databricks.yml dms-metadata-before dms-metadata-after trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle -trace $CLI bundle plan -t prod -o json +dms_plan -t prod trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "Deploy again without touching anything: the record already says this, so no deployment update" diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 8209a96b2fb..5b70ee58e6d 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -140,34 +140,12 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_emptied_resource_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_emptied_resource_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][1], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The grants node emptied out, so the service no longer lists it - only the schema remains >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 329163ad95e..49ff14b8a41 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -17,7 +17,7 @@ title "Plan again: reading state back from the service works and reports no work trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The grants node emptied out, so the service no longer lists it - only the schema remains" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-emptied-resource-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 0d35577a43c..b4c1b705c28 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -11,51 +11,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, - "plan": { - "resources.jobs.one": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS], - "creator_user_name": "[USERNAME]", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "one", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "serial": 1 } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + >>> print_requests.py --dms //api/2.0/bundle --oneline >>> jq {state_version, features} .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index a5be325e72a..aa3bee78294 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" trace $CLI bundle deploy -trace $CLI bundle plan -o json +dms_plan trace print_requests.py --dms //api/2.0/bundle --oneline trace jq '{state_version, features}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 2d06337ff31..c196fc9e64e 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -69,60 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.stuck": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "stuck", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === After deploy, the service lists the resource at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index 08d949fd198..c343679ef14 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index a4a818bf7bc..894fabe56a5 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -69,35 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "comment": "v1", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_failed_recreate_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === After deploy, the service lists the resource at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 8d0b7ddeef1..e2dbc58793e 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 9a13313ac7d..8edf660d3f9 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -17,82 +17,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.foo.permissions": { - "depends_on": [ - { - "node": "resources.jobs.foo", - "label": "${resources.jobs.foo.id}" - } - ], - "action": "skip", - "remote_state": { - "object_id": "/jobs/[NUMID]", - "__embed__": [ - { - "level": "CAN_VIEW", - "user_name": "viewer@example.com" - }, - { - "level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + === The service lists the job at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index 2dc2aca15ff..24a9f9ee45e 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy rm -f out.requests.txt title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the job" -trace $CLI bundle plan -o json +dms_plan title "The service lists the job at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-permissions-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index c9d8c5568c9..bd837caeda2 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -69,35 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "comment": "v1", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_failed_update_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === After deploy, the service lists the resource at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script index bdadee08fac..a956ed2fa5b 100644 --- a/acceptance/bundle/dms/failed-update/script +++ b/acceptance/bundle/dms/failed-update/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 688fe93bb3a..aee36fb25dc 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -27,256 +27,12 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.five": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "five", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.four": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][1], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "four", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.one": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][2], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "one", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.three": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][3], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "three", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.two": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][4], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "two", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged + === The service lists all five resources at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ @@ -340,256 +96,12 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.five": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "five", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.four": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][1], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "four", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.one": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][2], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "one", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.three": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][3], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "three", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.two": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][4], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "two", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged + === The service still lists all five resources, now at version 2 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 339c7973c5c..5ce8d484f83 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service lists all five resources at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -19,7 +19,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --sort --oneline title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service still lists all five resources, now at version 2" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 1c8708ac68b..f37562b84f0 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -37,100 +37,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.pipelines.bar": { - "action": "skip", - "remote_state": { - "catalog": "main", - "channel": "CURRENT", - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edition": "ADVANCED", - "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", - "id": "[UUID]", - "last_modified": [UNIX_TIME_MILLIS][1], - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" - } - } - ], - "name": "bar-[UNIQUE_NAME]", - "pipeline_id": "[UUID]", - "run_as_user_name": "[USERNAME]", - "schema": "default", - "serverless": true, - "state": "IDLE" - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { @@ -260,101 +172,12 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.pipelines.bar": { - "action": "skip", - "remote_state": { - "catalog": "main", - "channel": "CURRENT", - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "2" - }, - "development": true, - "edition": "ADVANCED", - "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", - "id": "[UUID]", - "last_modified": [UNIX_TIME_MILLIS][1], - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" - } - } - ], - "name": "bar-[UNIQUE_NAME]", - "pipeline_id": "[UUID]", - "run_as_user_name": "[USERNAME]", - "schema": "default", - "serverless": true, - "state": "IDLE" - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "2", - "new": "3", - "remote": "2" - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + === The raw recorded state the service holds after the update >>> print_state.py --no-dms { diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 67e8172e06b..203a2bebc05 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -8,7 +8,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle summary -o json | jq .bundle.deployment.history title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The raw recorded state the service holds for this deployment" trace print_state.py --no-dms @@ -27,7 +27,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle summary -o json | jq .bundle.deployment.history title "The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA_SERVICE as backend" -trace $CLI bundle plan -o json +dms_plan title "The raw recorded state the service holds after the update" trace print_state.py --no-dms diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index ee9b3127061..22b9703b44a 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -52,10 +52,12 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": {} + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged + === The service lists no resources for the deployment >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [] @@ -78,6 +80,14 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} { "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" @@ -118,10 +128,12 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": {} + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged + === The service still lists no resources >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [] diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 1593aee522f..d09f7a7faed 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -6,7 +6,7 @@ trace print_requests.py --dms //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service lists no resources for the deployment" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -20,7 +20,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service still lists no resources" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 9060ab58706..b1a6a64ea06 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -16,35 +16,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "comment": "v1", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_upload_fails_once_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The service lists the schema at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index 73e895433ab..dc242c6262a 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy rm -f out.requests.txt title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the schema" -trace $CLI bundle plan -o json +dms_plan title "The service lists the schema at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 437af92448b..57fee97f7fc 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -61,63 +61,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 4, - "name": "[dev [USERNAME]] foo", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "tags": { - "dev": "[USERNAME]" - }, - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID] { diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index ab20da12355..7ffc5636eab 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -9,7 +9,7 @@ add_repl.py "$(git rev-parse HEAD)" COMMIT trace print_requests.py --dms //versions title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance-${UNIQUE_NAME}/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 7568ec343dd..c03c9cfda2a 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -18,44 +18,14 @@ Files: 6 uploaded, 0 deleted "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.doomed": { - "action": "create", - "new_state": { - "value": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "2" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "doomed", - "queue": { - "enabled": true - }, - "tasks": [ - { - "new_cluster": { - "node_type_id": "nonexistent-node-type", - "num_workers": 1, - "spark_version": "15.4.x-scala2.12" - }, - "notebook_task": { - "notebook_path": "/nonexistent" - }, - "task_key": "doomed" - } - ] - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +create jobs.doomed + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index d3061bd3049..8ce1bb2696b 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" trace musterr $CLI bundle deploy -trace $CLI bundle plan -o json +dms_plan trace print_requests.py --dms //api/2.0/bundle title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 263099877d5..f5ac8362644 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -93,60 +93,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The service lists the job, recorded at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index ccbcd025d9b..08b67befc6d 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -16,7 +16,7 @@ trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_U trace print_requests.py --get --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service lists the job, recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/script.prepare b/acceptance/bundle/dms/script.prepare new file mode 100644 index 00000000000..f170821b4e8 --- /dev/null +++ b/acceptance/bundle/dms/script.prepare @@ -0,0 +1,9 @@ +# dms_plan prints the plan's header only (via jq del(.plan)), then the readable plan. +# The per-resource plan body carries remote_state that diverges between the fake and a +# real workspace, so the recording tests assert the header fields (cli_version, lineage, +# serial, deployment_id, next/last_version_id, storage_backend) and the human-readable +# summary instead of the full JSON. Pass through any flags, e.g. `dms_plan -t prod`. +dms_plan() { + trace $CLI bundle plan "$@" -o json | jq 'del(.plan)' + trace $CLI bundle plan "$@" +} diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index fa4f047b28e..f15171aa94e 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -4,29 +4,13 @@ { "plan_version": 3, "cli_version": "[CLI_VERSION]", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "create", - "new_state": { - "value": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } + +>>> [CLI] bundle plan +create jobs.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files... Created jobs.foo Files: 7 uploaded, 0 deleted diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index 3e189942281..bf7ae58d3f6 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Save a plan, then deploy it: records version 1" $CLI bundle plan -o json > tmp.plan.json -trace $CLI bundle plan -o json +dms_plan # READPLAN deploys from the saved raw plan; without it the plan is recomputed. Untraced so the # variant-dependent command line stays out of the golden (the deploy output is identical either way). $CLI bundle deploy $(readplanarg tmp.plan.json) diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 69fbc28396b..db1033110d2 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -69,35 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "comment": "v1", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The service lists the schema, recorded at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ @@ -196,35 +173,12 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "dms_other_[UNIQUE_NAME]", - "catalog_type": "MANAGED_CATALOG", - "comment": "v1", - "created_at": [UNIX_TIME_MILLIS][1], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][1], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The service lists the schema at version 2 with recreate action >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index 457164d584b..100441f9125 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service lists the schema, recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-successful-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -26,7 +26,7 @@ trace $CLI bundle deploy --auto-approve trace print_requests.py --dms //api/2.0/bundle title "The plan now shows the recreate action" -trace $CLI bundle plan -o json +dms_plan title "The service lists the schema at version 2 with recreate action" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 11df2223dba..6a963c0ff82 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -17,31 +17,14 @@ Files: 5 uploaded, 0 deleted "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "next_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "create", - "new_state": { - "value": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +create jobs.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json { "object_type": "BUNDLE_DEPLOYMENT" @@ -52,6 +35,8 @@ Files: 5 uploaded, 0 deleted {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [] diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script index d801153b1c7..18d3b71c3b4 100644 --- a/acceptance/bundle/dms/version-never-created/script +++ b/acceptance/bundle/dms/version-never-created/script @@ -1,6 +1,6 @@ title "The first version fails, so no deployment record exists - only the node naming its ID" trace musterr $CLI bundle deploy -trace $CLI bundle plan -o json +dms_plan trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' trace print_requests.py --dms //api/2.0/bundle --get --oneline From d76883e4edddc616901797213e97d9aed9895277 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:01:44 +0200 Subject: [PATCH 039/164] libs/dms: shorten record/RecordFailure comments, rename flag to keepExistingUpdateIfAny Trim the verbose doc comments to a couple of plain lines, and rename record's flag from isStateWrite to keepExistingUpdateIfAny (inverting it): true merges a failure onto the resource's last recorded state if one exists, false saves the update as that latest state. Co-authored-by: Isaac --- libs/dms/operation_buffer.go | 33 ++++++++++++------------------- libs/dms/operation_buffer_test.go | 6 +++--- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go index 11fe07fc471..0718aac330c 100644 --- a/libs/dms/operation_buffer.go +++ b/libs/dms/operation_buffer.go @@ -76,35 +76,28 @@ func (b *OperationBuffer) RecordOperation(ctx context.Context, resourceKey strin return } - b.record(resourceKey, update, true, nil) + b.record(resourceKey, update, false, nil) } -// RecordFailure records that a resource did not apply, so the history says why rather than -// leaving the resource out. When a write already recorded this resource's state, the failure -// keeps that state - which the service may already hold - and changes only status and message; -// recording the pre-deploy state instead would overwrite the newer state, and after a rename -// pair the new id with the old state. state is the fallback, used only when no write recorded -// anything: the operation failed before it saved state. +// RecordFailure records that a resource did not apply, so the history says why. state is a +// fallback used only when no write recorded state yet; otherwise record keeps the last state. func (b *OperationBuffer) RecordFailure(resourceKey, resourceID string, state json.RawMessage, cause error) { // The passed update is the fallback used only when no write recorded state yet; record // otherwise merges the failure onto the state the last write recorded (see below). - b.record(resourceKey, NewFailureUpdate(resourceID, state, cause), false, cause) + b.record(resourceKey, NewFailureUpdate(resourceID, state, cause), true, cause) } -// record makes update the one waiting for resourceKey, waiting itself while the queue is -// full. isStateWrite records update as the latest state for this resource, kept past the -// writer taking it from pending so a later failure keeps it (see the latestState field); -// a failure (isStateWrite false) instead merges its status and message onto that recorded -// state when there is one, so it keeps the state - which the service may already hold - -// rather than the pre-deploy one, and after a rename pairs the new id with the old state. -// The latestState read/write shares this lock with the pending update so the two never -// disagree. Recording after Drain panics, so every caller must return before Drain. -func (b *OperationBuffer) record(resourceKey string, update OperationUpdate, isStateWrite bool, cause error) { +// record queues update for resourceKey. keepExistingUpdateIfAny (a failure) merges onto the +// resource's last recorded state if one exists; otherwise (a state write) update is saved as +// that latest state. latestState and pending are updated under the one lock. Panics after Drain. +func (b *OperationBuffer) record(resourceKey string, update OperationUpdate, keepExistingUpdateIfAny bool, cause error) { b.mu.Lock() - if isStateWrite { + if keepExistingUpdateIfAny { + if latest, recorded := b.latestState[resourceKey]; recorded { + update = latest.Merge(failureUpdate(cause)) + } + } else { b.latestState[resourceKey] = update - } else if latest, recorded := b.latestState[resourceKey]; recorded { - update = latest.Merge(failureUpdate(cause)) } waiting, queued := b.pending[resourceKey] if queued { diff --git a/libs/dms/operation_buffer_test.go b/libs/dms/operation_buffer_test.go index cb01c8b57d5..49af18c3255 100644 --- a/libs/dms/operation_buffer_test.go +++ b/libs/dms/operation_buffer_test.go @@ -38,8 +38,8 @@ func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { // Two writes for one resource with nothing draining. They carry the resource's full // state, so only the newest needs to go: one slot in the queue, not two. - s.record("resources.jobs.foo", stateUpdate(t, "v1"), true, nil) - s.record("resources.jobs.foo", stateUpdate(t, "v2"), true, nil) + s.record("resources.jobs.foo", stateUpdate(t, "v1"), false, nil) + s.record("resources.jobs.foo", stateUpdate(t, "v2"), false, nil) assert.Len(t, s.queue, 1) update, ok := s.take("resources.jobs.foo") @@ -48,7 +48,7 @@ func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { // Taken means a request has it, and an in-flight request cannot be recalled, so the next // write gets its own slot rather than joining it. - s.record("resources.jobs.foo", stateUpdate(t, "v3"), true, nil) + s.record("resources.jobs.foo", stateUpdate(t, "v3"), false, nil) assert.Len(t, s.queue, 2) update, ok = s.take("resources.jobs.foo") require.True(t, ok) From f8397653ee9b482de7d648995208100962c06b12 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:08:37 +0200 Subject: [PATCH 040/164] acceptance/dms: bind test uses a real job so it records the DMS refusal Create a real workspace job and bind to it, so bind gets past fetching the resource and reaches the config gate that refuses bind while recording deployment history - recording that error instead of "Job 123 does not exist". Co-authored-by: Isaac --- .../dms/bind-unbind-unsupported/output.txt | 14 ++++--------- .../bundle/dms/bind-unbind-unsupported/script | 20 +++++++++---------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt index 484998ca590..891466bd2e5 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt +++ b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt @@ -1,22 +1,16 @@ -=== Before any deploy there is no recorded state: bind just fails to find the workspace resource, and unbind is a no-op ->>> musterr [CLI] bundle deployment bind foo 123 -Error: failed to fetch the resource, err: Job 123 does not exist. +=== bind is refused while recording deployment history +>>> musterr [CLI] bundle deployment bind foo [JOB_ID] +Error: bind is not supported for a bundle that records deployment history ->>> [CLI] bundle deployment unbind foo -=== Deploy so the bundle records the job with the deployment metadata service +=== Deploy so the bundle records state, then unbind is refused too >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... Created jobs.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -=== With deployment history recorded, both bind and unbind are refused ->>> musterr [CLI] bundle deployment bind foo [FOO_ID] -Error: bind is not supported for a bundle that records deployment history - - >>> musterr [CLI] bundle deployment unbind foo Error: unbind is not supported for a bundle that records deployment history diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/script b/acceptance/bundle/dms/bind-unbind-unsupported/script index 0bed434163a..9af766ca4d3 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/script +++ b/acceptance/bundle/dms/bind-unbind-unsupported/script @@ -1,18 +1,16 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Before any deploy there is no recorded state: bind just fails to find the workspace resource, and unbind is a no-op" -trace musterr $CLI bundle deployment bind foo 123 -trace $CLI bundle deployment unbind foo +# The gate that refuses bind while recording deployment history is on the config, so it fires +# before any deploy. Create a real workspace job so bind gets past fetching the resource and +# reaches the gate, instead of failing because the resource does not exist. +title "bind is refused while recording deployment history" +job_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id') +add_repl "$job_id" JOB_ID +trace musterr $CLI bundle deployment bind foo "$job_id" -title "Deploy so the bundle records the job with the deployment metadata service" +# unbind refuses when it opens an existing recording state, so it needs a deployment first. +title "Deploy so the bundle records state, then unbind is refused too" trace $CLI bundle deploy - -# bind and unbind adopt or release a resource by editing local state directly. A recorded -# deployment keeps its resource set in the service, so both are refused: bind hits the gate -# after fetching the resource, unbind when it opens the recording state without recording. -title "With deployment history recorded, both bind and unbind are refused" -job_id=$(read_id.py foo) -trace musterr $CLI bundle deployment bind foo "$job_id" trace musterr $CLI bundle deployment unbind foo rm -f out.requests.txt From f8894d58caf111756a74bc13a338f83fd6a3991f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:10:32 +0200 Subject: [PATCH 041/164] acceptance/dms: rename emptied-resource to empty-grants, record the known bug, Cloud=false Emptying a grants node makes apply call DeleteState, recording a succeeded UPDATE that names no state while the schema still exists; a real workspace rejects that (400 INVALID_PARAMETER_VALUE). The fake does not model the check, so the recorded output is the buggy behavior. Disable the cloud run and document the fix needed (emptying grants should record an update carrying the emptied state, not a delete). Co-authored-by: Isaac --- .../databricks.yml.tmpl | 4 ++-- .../out.test.toml | 2 +- .../output.txt | 24 +++++++++---------- .../{emptied-resource => empty-grants}/script | 12 ++++++---- acceptance/bundle/dms/empty-grants/test.toml | 6 +++++ 5 files changed, 29 insertions(+), 19 deletions(-) rename acceptance/bundle/dms/{emptied-resource => empty-grants}/databricks.yml.tmpl (69%) rename acceptance/bundle/dms/{emptied-resource => empty-grants}/out.test.toml (83%) rename acceptance/bundle/dms/{emptied-resource => empty-grants}/output.txt (85%) rename acceptance/bundle/dms/{emptied-resource => empty-grants}/script (62%) create mode 100644 acceptance/bundle/dms/empty-grants/test.toml diff --git a/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl b/acceptance/bundle/dms/empty-grants/databricks.yml.tmpl similarity index 69% rename from acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl rename to acceptance/bundle/dms/empty-grants/databricks.yml.tmpl index b8c21a3cc96..7be9f60129c 100644 --- a/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl +++ b/acceptance/bundle/dms/empty-grants/databricks.yml.tmpl @@ -1,11 +1,11 @@ bundle: - name: dms-emptied-resource-$UNIQUE_NAME + name: dms-empty-grants-$UNIQUE_NAME experimental: record_deployment_history: true resources: schemas: foo: - name: dms_emptied_resource_$UNIQUE_NAME + name: dms_empty_grants_$UNIQUE_NAME catalog_name: main grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}] diff --git a/acceptance/bundle/dms/emptied-resource/out.test.toml b/acceptance/bundle/dms/empty-grants/out.test.toml similarity index 83% rename from acceptance/bundle/dms/emptied-resource/out.test.toml rename to acceptance/bundle/dms/empty-grants/out.test.toml index 23c07f70dca..9921e91a794 100644 --- a/acceptance/bundle/dms/emptied-resource/out.test.toml +++ b/acceptance/bundle/dms/empty-grants/out.test.toml @@ -1,3 +1,3 @@ -Cloud = true +Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/empty-grants/output.txt similarity index 85% rename from acceptance/bundle/dms/emptied-resource/output.txt rename to acceptance/bundle/dms/empty-grants/output.txt index 5b70ee58e6d..12373101601 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -1,10 +1,10 @@ === Deploy a schema with one grant: the grants node is recorded with its state >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default/files... Created schemas.foo Created schemas.foo.grants -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle @@ -12,12 +12,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { - "display_name": "dms-emptied-resource-[UNIQUE_NAME]", - "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/state", + "display_name": "dms-empty-grants-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default" + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default" } } } @@ -50,9 +50,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged }, "body": { "error_message": "", - "resource_id": "main.dms_emptied_resource_[UNIQUE_NAME]", + "resource_id": "main.dms_empty_grants_[UNIQUE_NAME]", "sequence_id": "0", - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_emptied_resource_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_empty_grants_[UNIQUE_NAME]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -64,9 +64,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged }, "body": { "error_message": "", - "resource_id": "schema/main.dms_emptied_resource_[UNIQUE_NAME]", + "resource_id": "schema/main.dms_empty_grants_[UNIQUE_NAME]", "sequence_id": "0", - "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_emptied_resource_[UNIQUE_NAME]\",\"__embed__\":[{\"principal\":\"deco-test-user@databricks.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", + "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_empty_grants_[UNIQUE_NAME]\",\"__embed__\":[{\"principal\":\"deco-test-user@databricks.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -82,7 +82,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}] grants: [] >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default/files... Updated schemas.foo.grants Files: 3 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -176,6 +176,6 @@ The following resources will be deleted: This action will result in the deletion of the following UC schemas. Any underlying data may be lost: delete resources.schemas.foo -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default Destroy: 1 deleted diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/empty-grants/script similarity index 62% rename from acceptance/bundle/dms/emptied-resource/script rename to acceptance/bundle/dms/empty-grants/script index 49ff14b8a41..5b0abfe3c16 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/empty-grants/script @@ -8,9 +8,13 @@ title "Revoke the grant, so the grants node empties out" trace update_file.py databricks.yml 'grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}]' 'grants: []' trace $CLI bundle deploy -# The action type stays update: it is staged from the plan, before apply knows the state -# comes back empty. What drops the resource from the deployment is the update naming state -# with no value. +# KNOWN BUG (why this test is Cloud=false): emptying the grants node makes apply call +# DeleteState, which records the operation as a delete with no state. The staged action +# is still UPDATE, so the service sees a succeeded UPDATE that names no state while the +# schema (a live resource) still exists, and rejects it (400 INVALID_PARAMETER_VALUE). The +# fake does not model that check, so this passes locally; the recorded requests below are +# the buggy output. Fix: emptying grants should record an update carrying the emptied +# grants state, not a delete. trace print_requests.py --dms //api/2.0/bundle title "Plan again: reading state back from the service works and reports no work" @@ -20,7 +24,7 @@ title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backen dms_plan title "The grants node emptied out, so the service no longer lists it - only the schema remains" -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-emptied-resource-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-empty-grants-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" diff --git a/acceptance/bundle/dms/empty-grants/test.toml b/acceptance/bundle/dms/empty-grants/test.toml new file mode 100644 index 00000000000..fc548948325 --- /dev/null +++ b/acceptance/bundle/dms/empty-grants/test.toml @@ -0,0 +1,6 @@ +# KNOWN BUG (Cloud=false): emptying a grants node makes apply call DeleteState, which records +# the operation as a delete with no state. The staged action is still UPDATE, so the service +# rejects a succeeded UPDATE that names no state while the schema still exists. The fake does +# not model that check, so this passes locally but fails on a real workspace. Re-enable once +# emptying grants records an update carrying the emptied state instead of a delete. +Cloud = false From 4b01fa8088819208e8e1d6bbec7ac3d46088c849 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:12:42 +0200 Subject: [PATCH 042/164] acceptance/dms: assert failed permissions update records its error via list-operations Add a bundle-deployments list-operations call after the failed permissions update so the golden asserts the failed operation (jobs.foo.permissions, UPDATE, FAILED) carries the 403 error message the service recorded. Co-authored-by: Isaac --- .../bundle/dms/failed-update-permissions/output.txt | 11 +++++++++++ .../bundle/dms/failed-update-permissions/script | 3 +++ 2 files changed, 14 insertions(+) diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 8edf660d3f9..e7bf63ed164 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -82,6 +82,17 @@ Files: 3 uploaded, 0 deleted } ] +=== The failed operation (version 2) records the error message +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[NUMID]/versions/2 -o json +[ + { + "action_type": "OPERATION_ACTION_TYPE_UPDATE", + "error_message": "updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED)", + "resource_key": "jobs.foo.permissions", + "status": "OPERATION_STATUS_FAILED" + } +] + === The raw recorded state after the failure >>> print_state.py --no-dms { diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index 24a9f9ee45e..f9da3a3220f 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -24,6 +24,9 @@ title "The service lists the job with the failure" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-permissions-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +title "The failed operation (version 2) records the error message" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -S 'map({resource_key, action_type, status, error_message})' + title "The raw recorded state after the failure" trace print_state.py --no-dms From 7bf906ac6ee5c199d0066cbd6ab18527daa7efeb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:14:48 +0200 Subject: [PATCH 043/164] acceptance/dms: remove operation-upload-fails test Dropped as unnecessary; operation-upload-fails-once still covers the upload-failure path. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 23 ------------------- .../dms/operation-upload-fails/out.test.toml | 3 --- .../dms/operation-upload-fails/output.txt | 4 ---- .../bundle/dms/operation-upload-fails/script | 9 -------- .../dms/operation-upload-fails/test.toml | 10 -------- 5 files changed, 49 deletions(-) delete mode 100644 acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/operation-upload-fails/out.test.toml delete mode 100644 acceptance/bundle/dms/operation-upload-fails/output.txt delete mode 100644 acceptance/bundle/dms/operation-upload-fails/script delete mode 100644 acceptance/bundle/dms/operation-upload-fails/test.toml diff --git a/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl deleted file mode 100644 index b6aa93265df..00000000000 --- a/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl +++ /dev/null @@ -1,23 +0,0 @@ -bundle: - name: dms-operation-upload-fails-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - one: - name: one - two: - name: two - three: - name: three - four: - name: four - five: - name: five - six: - name: six - seven: - name: seven - eight: - name: eight diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml deleted file mode 100644 index 9921e91a794..00000000000 --- a/acceptance/bundle/dms/operation-upload-fails/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-fails/output.txt b/acceptance/bundle/dms/operation-upload-fails/output.txt deleted file mode 100644 index 2b273972417..00000000000 --- a/acceptance/bundle/dms/operation-upload-fails/output.txt +++ /dev/null @@ -1,4 +0,0 @@ - -=== An operation upload failure fails the deploy instead of reporting only at the end ->>> grep -c ^Error: LOG.deploy -deploy reported errors diff --git a/acceptance/bundle/dms/operation-upload-fails/script b/acceptance/bundle/dms/operation-upload-fails/script deleted file mode 100644 index cc2ead92c6f..00000000000 --- a/acceptance/bundle/dms/operation-upload-fails/script +++ /dev/null @@ -1,9 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "An operation upload failure fails the deploy instead of reporting only at the end" -# Which resources get refused depends on how far apply got before a background -# upload failed, so the per-resource errors go to a LOG file rather than the diff. -fault.py "PATCH /api/2.0/bundle/*" 500 0 99 INTERNAL_ERROR -errcode $CLI bundle deploy &> LOG.deploy -contains.py 'recording operation for' '!panic' < LOG.deploy > /dev/null -trace grep -c "^Error:" LOG.deploy > /dev/null && echo "deploy reported errors" diff --git a/acceptance/bundle/dms/operation-upload-fails/test.toml b/acceptance/bundle/dms/operation-upload-fails/test.toml deleted file mode 100644 index b5502a2516c..00000000000 --- a/acceptance/bundle/dms/operation-upload-fails/test.toml +++ /dev/null @@ -1,10 +0,0 @@ -# Local only: this case needs an injected failure - DMS itself must reject the operation upload - and fault.py registers -# its rule on the fake, which a cloud run does not have. -Cloud = false - -# Which requests are made depends on how far apply got before a background upload -# failed, so recording them would make the output nondeterministic. -RecordRequests = false - -# Completed versions make DMS the source of truth; unrecorded resources get recreated. -# Deploy must stop rather than continue creating. From bdcc887138498fc7cc74272854c38405b435ea65 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:19:05 +0200 Subject: [PATCH 044/164] acceptance/dms: remove redundant record and failed-update tests Both duplicate coverage in the remaining bundle/dms tests: recording calls and serialized operation state are exercised by successful-recreate, multiple-resources, depends-on and others; a failed update is covered by failed-update-permissions and failed-recreate. Reword the two comments that pointed at bundle/dms/record. Co-authored-by: Isaac --- .../dms/failed-update/databricks.yml.tmpl | 11 -- .../bundle/dms/failed-update/out.test.toml | 3 - .../bundle/dms/failed-update/output.txt | 185 ------------------ acceptance/bundle/dms/failed-update/script | 37 ---- acceptance/bundle/dms/failed-update/test.toml | 3 - .../bundle/dms/multiple-resources/output.txt | 2 +- .../bundle/dms/multiple-resources/script | 2 +- .../bundle/dms/record/databricks.yml.tmpl | 9 - acceptance/bundle/dms/record/out.test.toml | 3 - acceptance/bundle/dms/record/output.txt | 123 ------------ acceptance/bundle/dms/record/script | 27 --- acceptance/bundle/test.toml | 2 +- 12 files changed, 3 insertions(+), 404 deletions(-) delete mode 100644 acceptance/bundle/dms/failed-update/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/failed-update/out.test.toml delete mode 100644 acceptance/bundle/dms/failed-update/output.txt delete mode 100644 acceptance/bundle/dms/failed-update/script delete mode 100644 acceptance/bundle/dms/failed-update/test.toml delete mode 100644 acceptance/bundle/dms/record/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/record/out.test.toml delete mode 100644 acceptance/bundle/dms/record/output.txt delete mode 100644 acceptance/bundle/dms/record/script diff --git a/acceptance/bundle/dms/failed-update/databricks.yml.tmpl b/acceptance/bundle/dms/failed-update/databricks.yml.tmpl deleted file mode 100644 index 87cfcff9f36..00000000000 --- a/acceptance/bundle/dms/failed-update/databricks.yml.tmpl +++ /dev/null @@ -1,11 +0,0 @@ -bundle: - name: dms-failed-update-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - schemas: - foo: - name: dms_failed_update_schema_$UNIQUE_NAME - catalog_name: main - comment: v1 diff --git a/acceptance/bundle/dms/failed-update/out.test.toml b/acceptance/bundle/dms/failed-update/out.test.toml deleted file mode 100644 index 9921e91a794..00000000000 --- a/acceptance/bundle/dms/failed-update/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt deleted file mode 100644 index bd837caeda2..00000000000 --- a/acceptance/bundle/dms/failed-update/output.txt +++ /dev/null @@ -1,185 +0,0 @@ - -=== Deploy the schema, so there is an existing resource to update ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files... -Created schemas.foo -Files: 6 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> print_requests.py --dms //api/2.0/bundle -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments", - "body": { - "display_name": "dms-failed-update-[UNIQUE_NAME]", - "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/state", - "target_name": "default", - "workspace_info": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default" - } - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "1" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "operations": [ - { - "resource_key": "schemas.foo", - "action_type": "OPERATION_ACTION_TYPE_CREATE" - } - ] - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "error_message": "", - "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "sequence_id": "0", - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}", - "status": "OPERATION_STATUS_SUCCEEDED" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} - -=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend ->>> [CLI] bundle plan -o json -{ - "plan_version": 3, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1, - "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" -} - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== After deploy, the service lists the resource at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json -[ - { - "last_action_type": "OPERATION_ACTION_TYPE_CREATE", - "last_version_id": "1", - "resource_key": "schemas.foo" - } -] - -=== The raw state after successful deploy ->>> print_state.py --no-dms -{ - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1, - "features": { - "record_deployment_history": {} - }, - "state": {} -} - -=== An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it ->>> update_file.py databricks.yml comment: v1 comment: v2 - ->>> fault.py PATCH /api/2.1/unity-catalog/schemas/* 400 0 1 INVALID_PARAMETER_VALUE - ->>> musterr [CLI] bundle deploy --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files... -Error: cannot update resources.schemas.foo: updating id=main.dms_failed_update_schema_[UNIQUE_NAME]: Fault injected by test. (400 INVALID_PARAMETER_VALUE) - -Endpoint: PATCH [DATABRICKS_URL]/api/2.1/unity-catalog/schemas/main.dms_failed_update_schema_[UNIQUE_NAME] -HTTP Status: 400 Bad Request -API error_code: INVALID_PARAMETER_VALUE -API message: Fault injected by test. - -Files: 3 uploaded, 0 deleted - ->>> print_requests.py --dms //api/2.0/bundle -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "2" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "previous_version_id": "1", - "operations": [ - { - "resource_key": "schemas.foo", - "action_type": "OPERATION_ACTION_TYPE_UPDATE" - } - ] - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "error_message": "updating id=main.dms_failed_update_schema_[UNIQUE_NAME]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", - "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "sequence_id": "0", - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}", - "status": "OPERATION_STATUS_FAILED" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_FAILURE" - } -} - -=== The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json -[ - { - "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", - "last_version_id": "2", - "resource_key": "schemas.foo" - } -] - -=== The raw state after failed update shows the previous version ->>> print_state.py --no-dms -{ - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1, - "features": { - "record_deployment_history": {} - }, - "state": {} -} - -=== Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys ->>> [CLI] bundle plan -update schemas.foo - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script deleted file mode 100644 index a956ed2fa5b..00000000000 --- a/acceptance/bundle/dms/failed-update/script +++ /dev/null @@ -1,37 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "Deploy the schema, so there is an existing resource to update" -trace $CLI bundle deploy -trace print_requests.py --dms //api/2.0/bundle - -title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -dms_plan - -title "After deploy, the service lists the resource at version 1" -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' - -title "The raw state after successful deploy" -trace print_state.py --no-dms - -title "An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it" -trace update_file.py databricks.yml "comment: v1" "comment: v2" -# Fail the update call itself, so the deploy never writes state for the schema. -trace fault.py "PATCH /api/2.1/unity-catalog/schemas/*" 400 0 1 INVALID_PARAMETER_VALUE -trace musterr $CLI bundle deploy --auto-approve -# Not sorted: the point of this test is the order the operations are recorded in. -trace print_requests.py --dms //api/2.0/bundle - -title "The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one" -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' - -title "The raw state after failed update shows the previous version" -trace print_state.py --no-dms - -title "Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys" -rm -rf .databricks -trace $CLI bundle plan -# plan probes the state files concurrently, so the recorded order varies. Only the -# operations above are asserted; drop the rest rather than diff a racy order. -rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-update/test.toml b/acceptance/bundle/dms/failed-update/test.toml deleted file mode 100644 index 2b4eeb1bb04..00000000000 --- a/acceptance/bundle/dms/failed-update/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Local only: this case needs an injected failure - no schema update that Unity Catalog rejects outright - and fault.py registers -# its rule on the fake, which a cloud run does not have. -Cloud = false diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index aee36fb25dc..bf9a24cba12 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -1,5 +1,5 @@ -=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it. +=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; other bundle/dms tests cover it. >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... Created jobs.five diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 5ce8d484f83..8177c0be601 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it." +title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; other bundle/dms tests cover it." trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline diff --git a/acceptance/bundle/dms/record/databricks.yml.tmpl b/acceptance/bundle/dms/record/databricks.yml.tmpl deleted file mode 100644 index 214793eca38..00000000000 --- a/acceptance/bundle/dms/record/databricks.yml.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -bundle: - name: dms-record-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - foo: - name: foo diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml deleted file mode 100644 index 23c07f70dca..00000000000 --- a/acceptance/bundle/dms/record/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt deleted file mode 100644 index f5ac8362644..00000000000 --- a/acceptance/bundle/dms/record/output.txt +++ /dev/null @@ -1,123 +0,0 @@ - -=== Deploy, then destroy: set up a deployment and delete it, so the deploy below starts from nothing ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files... -Created jobs.foo -Files: 5 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.foo - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default - -Destroy: 1 deleted - -=== Deploy again: the destroy took the workspace node with it, so there is nothing to resolve and a fresh deployment starts at version 1, not a continuation of the deleted one ->>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json -Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json) doesn't exist. - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files... -Created jobs.foo -Files: 5 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json -{ - "object_type": "BUNDLE_DEPLOYMENT", - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json" -} - ->>> print_requests.py --get --dms //api/2.0/bundle -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments", - "body": { - "display_name": "dms-record-[UNIQUE_NAME]", - "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state", - "target_name": "default", - "workspace_info": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default" - } - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "1" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "operations": [ - { - "resource_key": "jobs.foo", - "action_type": "OPERATION_ACTION_TYPE_CREATE" - } - ] - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "error_message": "", - "resource_id": "[NUMID]", - "sequence_id": "0", - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", - "status": "OPERATION_STATUS_SUCCEEDED" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} - -=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend ->>> [CLI] bundle plan -o json -{ - "plan_version": 3, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 3, - "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" -} - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== The service lists the job, recorded at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json -[ - { - "last_action_type": "OPERATION_ACTION_TYPE_CREATE", - "last_version_id": "1", - "resource_key": "jobs.foo" - } -] - -=== The raw recorded state the service holds for this deployment ->>> print_state.py --no-dms -{ - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 3, - "features": { - "record_deployment_history": {} - }, - "state": {} -} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script deleted file mode 100644 index 08b67befc6d..00000000000 --- a/acceptance/bundle/dms/record/script +++ /dev/null @@ -1,27 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "Deploy, then destroy: set up a deployment and delete it, so the deploy below starts from nothing" -trace $CLI bundle deploy -trace $CLI bundle destroy --auto-approve - -# Discard the deploy+destroy requests above; this test is only about the deploy that FOLLOWS a -# destroy. The individual deploy and destroy request sequences are covered by no-resources, -# successful-recreate, and multiple-resources. -rm -f out.requests.txt - -title "Deploy again: the destroy took the workspace node with it, so there is nothing to resolve and a fresh deployment starts at version 1, not a continuation of the deleted one" -trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" -trace $CLI bundle deploy -trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' -trace print_requests.py --get --dms //api/2.0/bundle - -title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -dms_plan - -title "The service lists the job, recorded at version 1" -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' - -title "The raw recorded state the service holds for this deployment" -trace print_state.py --no-dms -rm -f out.requests.txt diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index 2db98f4b429..5f9b6281331 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -20,7 +20,7 @@ Env.UV_PYTHON = "3.10" # The link a recorded deploy prints, dropped so a test asserts the same output whether or # not recording is on. The URL itself is covered by workspaceurls.TestDeploymentURL, and -# the calls behind it by bundle/dms/record. +# the calls behind it by the bundle/dms tests. [[Repls]] Old = '(?m)^Current Deployment Version: .*\n' New = '' From cc4d4cc36052e7cc43c3471a21f3b77a25566c8b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:38:17 +0200 Subject: [PATCH 045/164] libs/testserver: gofmt fake_workspace.go after seeding the main catalog Co-authored-by: Isaac --- libs/testserver/fake_workspace.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index c9026e20fc2..f9e1414a237 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -458,15 +458,15 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { files: make(map[string]FileEntry), repoIdByPath: make(map[string]int64), - Jobs: map[int64]jobs.Job{}, - JobRuns: map[int64]jobs.Run{}, - JobRunOutputs: map[int64]jobs.RunOutput{}, - JobRunIdempotency: map[string]int64{}, - Grants: map[string][]catalog.PrivilegeAssignment{}, - Pipelines: map[string]pipelines.GetPipelineResponse{}, - PipelineUpdates: map[string]bool{}, - Monitors: map[string]catalog.MonitorInfo{}, - Apps: map[string]apps.App{}, + Jobs: map[int64]jobs.Job{}, + JobRuns: map[int64]jobs.Run{}, + JobRunOutputs: map[int64]jobs.RunOutput{}, + JobRunIdempotency: map[string]int64{}, + Grants: map[string][]catalog.PrivilegeAssignment{}, + Pipelines: map[string]pipelines.GetPipelineResponse{}, + PipelineUpdates: map[string]bool{}, + Monitors: map[string]catalog.MonitorInfo{}, + Apps: map[string]apps.App{}, // Seed the "main" catalog: it exists on every real workspace, and schema // creation now checks the catalog exists, so tests that deploy schemas into // main (the common case) keep working without creating it first. From 983777e625038d263ce433eb7727739b2078f707 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 09:58:41 +0200 Subject: [PATCH 046/164] Class A: Revert catalog-existence validation regression Remove the catalog-existence check from libs/testserver/schemas.go SchemasCreate that was added to model UC behavior. This breaks local-only tests that create schemas in arbitrary catalogs that the testserver doesn't pre-populate. Restore fake_workspace.go to initialize Catalogs as empty (no seeded "main"). Update catalogs_test.go to assert empty map instead of NotContains. Rewrite failed-recreate acceptance test to use fault injection for schema creation failure instead of relying on catalog-existence check, and mark it Cloud=false since the fault is testserver-only. Co-authored-by: Isaac --- acceptance/bundle/dms/failed-recreate/script | 6 +++--- acceptance/bundle/dms/failed-recreate/test.toml | 6 ++---- libs/testserver/catalogs_test.go | 3 +-- libs/testserver/fake_workspace.go | 9 ++------- libs/testserver/schemas.go | 12 ------------ 5 files changed, 8 insertions(+), 28 deletions(-) diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index e2dbc58793e..51bd335a0b2 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -15,9 +15,9 @@ title "The raw state after successful deploy" trace print_state.py --no-dms title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy" -# Point the schema at a catalog that does not exist: the delete goes through, then the -# create fails naturally against both the fake and a real workspace (no fault injection). -trace update_file.py databricks.yml "catalog_name: main" "catalog_name: dms_nonexistent_${UNIQUE_NAME}" +# Inject a fault on the schema create for the recreate operation. The delete goes through, +# then the create fails. +trace fault.py "POST /api/2.1/unity-catalog/schemas" 400 0 1 INVALID_PARAMETER_VALUE trace musterr $CLI bundle deploy --auto-approve # Not sorted: the point of this test is the order the operations are recorded in. trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/failed-recreate/test.toml b/acceptance/bundle/dms/failed-recreate/test.toml index 25c802e47a0..e916608add1 100644 --- a/acceptance/bundle/dms/failed-recreate/test.toml +++ b/acceptance/bundle/dms/failed-recreate/test.toml @@ -1,4 +1,2 @@ -# The recreate fails naturally by pointing the schema at a catalog that does not -# exist - the fake models catalog existence, so this fails the same way locally and -# on cloud without fault injection. -Cloud = true +# Local only: the failure is injected to make schema creation fail for the recreate. +Cloud = false diff --git a/libs/testserver/catalogs_test.go b/libs/testserver/catalogs_test.go index b699361577c..aa98c477479 100644 --- a/libs/testserver/catalogs_test.go +++ b/libs/testserver/catalogs_test.go @@ -19,8 +19,7 @@ func TestCatalogsCreate_RejectsEmptyName(t *testing.T) { // A stored-but-unreadable catalog under the empty key is the original bug. Asserted // before the require below so it is still reported when the rejection is missing. - // (The workspace seeds "main", so the map is not empty; the empty key must be absent.) - assert.NotContains(t, workspace.Catalogs, "") + assert.Empty(t, workspace.Catalogs) body, ok := response.Body.(map[string]string) require.True(t, ok) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index f9e1414a237..de9855209ff 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -467,13 +467,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { PipelineUpdates: map[string]bool{}, Monitors: map[string]catalog.MonitorInfo{}, Apps: map[string]apps.App{}, - // Seed the "main" catalog: it exists on every real workspace, and schema - // creation now checks the catalog exists, so tests that deploy schemas into - // main (the common case) keep working without creating it first. - Catalogs: map[string]catalog.CatalogInfo{ - "main": {Name: "main", FullName: "main", CatalogType: catalog.CatalogTypeManagedCatalog}, - }, - ExternalLocations: map[string]catalog.ExternalLocationInfo{}, + Catalogs: map[string]catalog.CatalogInfo{}, + ExternalLocations: map[string]catalog.ExternalLocationInfo{}, Schemas: map[string]catalog.SchemaInfo{}, RegisteredModels: map[string]catalog.RegisteredModelInfo{}, Volumes: map[string]catalog.VolumeInfo{}, diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 92bc0374734..6880a55cb8f 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -29,18 +29,6 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { } } - // UC rejects a schema whose parent catalog does not exist; model it so a deploy into a - // missing catalog fails the same way locally as on cloud. - if _, ok := s.Catalogs[schema.CatalogName]; !ok { - return Response{ - StatusCode: http.StatusNotFound, - Body: map[string]string{ - "error_code": "CATALOG_DOES_NOT_EXIST", - "message": fmt.Sprintf("Catalog '%s' does not exist.", schema.CatalogName), - }, - } - } - // UC normalizes schema names to lowercase. schema.Name = strings.ToLower(schema.Name) schema.FullName = schema.CatalogName + "." + schema.Name From ab2ccc6029197bf6e8e927b7d1251bb3f5a06e95 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 09:59:27 +0200 Subject: [PATCH 047/164] Regenerate failed-recreate test golden after fault injection rewrite Updated failed-recreate to use fault injection instead of catalog check, regenerated output.txt and out.test.toml. Co-authored-by: Isaac --- .../bundle/dms/failed-recreate/out.test.toml | 2 +- .../bundle/dms/failed-recreate/output.txt | 89 ++----------------- 2 files changed, 6 insertions(+), 85 deletions(-) diff --git a/acceptance/bundle/dms/failed-recreate/out.test.toml b/acceptance/bundle/dms/failed-recreate/out.test.toml index 23c07f70dca..9921e91a794 100644 --- a/acceptance/bundle/dms/failed-recreate/out.test.toml +++ b/acceptance/bundle/dms/failed-recreate/out.test.toml @@ -1,3 +1,3 @@ -Cloud = true +Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 894fabe56a5..13ade9f24e5 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -99,92 +99,13 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy ->>> update_file.py databricks.yml catalog_name: main catalog_name: dms_nonexistent_[UNIQUE_NAME] +>>> fault.py POST /api/2.1/unity-catalog/schemas 400 0 1 INVALID_PARAMETER_VALUE >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - recreate resources.schemas.foo -Error: cannot recreate resources.schemas.foo: Catalog 'dms_nonexistent_[UNIQUE_NAME]' does not exist. (404 CATALOG_DOES_NOT_EXIST) +Unexpected success -Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/schemas -HTTP Status: 404 Not Found -API error_code: CATALOG_DOES_NOT_EXIST -API message: Catalog 'dms_nonexistent_[UNIQUE_NAME]' does not exist. - -Files: 3 uploaded, 0 deleted - ->>> print_requests.py --dms //api/2.0/bundle -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "2" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "previous_version_id": "1", - "operations": [ - { - "resource_key": "schemas.foo", - "action_type": "OPERATION_ACTION_TYPE_RECREATE" - } - ] - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", - "q": { - "update_mask": "state,error_message,status" - }, - "body": { - "error_message": "", - "sequence_id": "0", - "status": "OPERATION_STATUS_PENDING" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", - "q": { - "update_mask": "state,error_message,status" - }, - "body": { - "error_message": "Catalog 'dms_nonexistent_[UNIQUE_NAME]' does not exist. (404 CATALOG_DOES_NOT_EXIST)", - "sequence_id": "1", - "status": "OPERATION_STATUS_FAILED" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_FAILURE" - } -} - -=== The resource is not listed: state is what projects a resource, and the failed recreate left none ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json -[] - -=== The raw state after failed recreate shows no resource ->>> print_state.py --no-dms -{ - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 2, - "features": { - "record_deployment_history": {} - }, - "state": {} -} - -=== Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists ->>> [CLI] bundle plan -create schemas.foo - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +Exit code: 1 From 7b84ffe89460d4b1cee80c98de915c342dd2594b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 10:05:31 +0200 Subject: [PATCH 048/164] Add storage_backend to nostamp.py plan header keys Include storage_backend in the list of plan header fields stripped by nostamp.py so it can normalize plans whether or not DMS recording is active. Co-authored-by: Isaac --- acceptance/bin/nostamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bin/nostamp.py b/acceptance/bin/nostamp.py index a4306fe401a..d6b3f9524c7 100755 --- a/acceptance/bin/nostamp.py +++ b/acceptance/bin/nostamp.py @@ -22,7 +22,7 @@ _STAMP_KEYS = ("deployment_id", "version_id") _CHANGE_KEYS = ("deployment.deployment_id", "deployment.version_id") -_PLAN_HEADER_KEYS = ("deployment_id", "next_version_id", "last_version_id") +_PLAN_HEADER_KEYS = ("deployment_id", "next_version_id", "last_version_id", "storage_backend") def scrub(node): From 101fa48a5a1b1e00930b77b77c16c72339b87e2e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 10:06:05 +0200 Subject: [PATCH 049/164] Class B: Pipe bundle plan JSON through nostamp.py in 32 tests Add nostamp.py piping to all non-DMS tests that output bundle plan JSON. This normalizes the plan output to remove DMS-specific fields like storage_backend so the same goldens work whether or not DMS recording is active. Goldens will be regenerated in the next commit. Co-authored-by: Isaac --- acceptance/bundle/apps/delete_deleting/script | 2 +- acceptance/bundle/apps/source_code_path_update/script | 2 +- .../bundle/deploy/readplan/cli-version-mismatch/script | 2 +- .../bundle/deploy/readplan/grants-remove-principal/script | 2 +- acceptance/bundle/deploy/readplan/postgres_role/script | 2 +- acceptance/bundle/deploy/readplan/serial-mismatch/script | 2 +- acceptance/bundle/deploy/readplan/whl-not-uploaded/script | 2 +- .../bundle/deployment/bind/dashboard/recreation/script | 4 ++-- acceptance/bundle/deployment/bind/dashboard/script | 2 +- acceptance/bundle/invariant/delete_idempotent/script | 6 +++--- acceptance/bundle/invariant/destroy_idempotent/script | 2 +- acceptance/bundle/invariant/no_drift/script | 2 +- acceptance/bundle/resource_deps/escaped_ref/script | 2 +- acceptance/bundle/resource_deps/volume_path_job_ref/script | 2 +- acceptance/bundle/resources/apps/readplan-lifecycle/script | 2 +- .../bundle/resources/clusters/readplan-lifecycle/script | 2 +- .../resources/clusters/resize-terminated-fallback/script | 2 +- .../resources/dashboards/change-serialized-dashboard/script | 2 +- .../bundle/resources/dashboards/nested-folders/script | 2 +- .../dashboards/publish-failure-cleans-up-dashboard/script | 2 +- .../bundle/resources/dashboards/serialized-inline/script | 2 +- acceptance/bundle/resources/job_runs/interrupted_run/script | 2 +- .../bundle/resources/job_runs/on_bundle_deploy/script | 4 ++-- .../bundle/resources/jobs/remote_delete/deploy/script | 2 +- .../bundle/resources/models/readplan-permissions/script | 2 +- .../resources/quality_monitors/change_assets_dir/script | 2 +- .../resources/quality_monitors/change_table_name/script | 2 +- acceptance/bundle/select/basic/script | 4 ++-- acceptance/bundle/select/grants_permissions/script | 4 ++-- .../bundle/select/plan_file_survives_not_selected/script | 2 +- .../templates/default-python/combinations/classic/script | 4 ++-- .../templates/default-python/combinations/serverless/script | 4 ++-- 32 files changed, 40 insertions(+), 40 deletions(-) diff --git a/acceptance/bundle/apps/delete_deleting/script b/acceptance/bundle/apps/delete_deleting/script index 14267a32af4..9df3da025d9 100644 --- a/acceptance/bundle/apps/delete_deleting/script +++ b/acceptance/bundle/apps/delete_deleting/script @@ -24,7 +24,7 @@ cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null envsubst < databricks_header.yml.tmpl > databricks.yml echo "resources: {}" >> databricks.yml -trace $CLI bundle plan -o json > plan_delete.json 2>LOG.plan_delete.err +trace $CLI bundle plan -o json | nostamp.py > plan_delete.json 2>LOG.plan_delete.err cat LOG.plan_delete.err | contains.py '!panic' '!internal error' > /dev/null # Out-of-band delete flips the app into DELETING: a second delete now returns 400. diff --git a/acceptance/bundle/apps/source_code_path_update/script b/acceptance/bundle/apps/source_code_path_update/script index d4b781b0b95..3afbd683de2 100644 --- a/acceptance/bundle/apps/source_code_path_update/script +++ b/acceptance/bundle/apps/source_code_path_update/script @@ -28,7 +28,7 @@ title "Redeploy from saved plan after description change" # Changing a non-deploy-only field (description) so DoUpdate is called. sed -i.bak 's/description: "first description"/description: "second description"/' databricks.yml # Route bundle plan stderr (diagnostics) to a log so output.txt stays clean. -$CLI bundle plan -o json > plan.json 2> LOG.plan +$CLI bundle plan -o json | nostamp.py > plan.json 2> LOG.plan # bundle deploy --plan triggers InitForApply, which JSON-round-trips AppState. # Without the fix, apps.App.ForceSendFields ends up containing "SourceCodePath", # causing "source_code_path": "" to be sent in the UpdateApp body (API returns 400). diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/script b/acceptance/bundle/deploy/readplan/cli-version-mismatch/script index b7fe9ee268d..6c6b9a87ef5 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/script +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/script @@ -1,4 +1,4 @@ -trace $CLI bundle plan -o json > plan.json +trace $CLI bundle plan -o json | nostamp.py > plan.json jq '.cli_version = "1.2.3"' plan.json > plan.json.tmp && mv plan.json.tmp plan.json diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/script b/acceptance/bundle/deploy/readplan/grants-remove-principal/script index a0625c5653d..962c456616c 100644 --- a/acceptance/bundle/deploy/readplan/grants-remove-principal/script +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/script @@ -14,7 +14,7 @@ grep -v 'TO_REMOVE' databricks.yml > updated.yml && mv updated.yml databricks.ym # Generate update plan: entry.RemoteState captures both principals as GrantsState. # remote_state.__embed__ principal order is non-deterministic (the backend, and the # testserver mirroring it, iterate a map), so sort it by principal for a stable golden. -trace $CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle plan -o json | nostamp.py > tmp.plan.json jq '.plan["resources.schemas.myschema.grants"] | .remote_state.__embed__ |= sort_by(.principal)' tmp.plan.json > out.plan.grants.json trace print_requests.py --get //unity-catalog/permissions --sort diff --git a/acceptance/bundle/deploy/readplan/postgres_role/script b/acceptance/bundle/deploy/readplan/postgres_role/script index efade4e2789..d50a05bddff 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/script +++ b/acceptance/bundle/deploy/readplan/postgres_role/script @@ -6,7 +6,7 @@ # empty role_id, the server would generate its own, and the next plan would # report phantom "recreate" drift on the immutable role_id. -trace $CLI bundle plan -o json > plan.json +trace $CLI bundle plan -o json | nostamp.py > plan.json # Keep only the role entry; role_id and parent must survive serialization. trace jq '.plan."resources.postgres_roles.role"' plan.json > out.plan.role.json diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/script b/acceptance/bundle/deploy/readplan/serial-mismatch/script index 170f44a4c87..5eff9b6ad49 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/script +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/script @@ -2,7 +2,7 @@ trace $CLI bundle deploy # Generate a plan (captures current lineage and serial) -trace $CLI bundle plan -o json > plan.json +trace $CLI bundle plan -o json | nostamp.py > plan.json # Modify only the serial to simulate stale plan jq '.serial = 999' plan.json > plan.json.tmp && mv plan.json.tmp plan.json diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script index bd507766e34..96237d6cdd0 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script @@ -15,7 +15,7 @@ trace $CLI bundle destroy --auto-approve rm out.requests.txt title "bundle deploy --plan does NOT upload the wheel (bug)" -trace $CLI bundle plan -o json > out.plan.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.json trace $CLI bundle deploy --plan out.plan.json # BUG: no import-file request for the wheel here (output below is empty), yet the job was # created pointing the task at the same .../artifacts/.internal/ path that was never uploaded. diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/script b/acceptance/bundle/deployment/bind/dashboard/recreation/script index ce63de92924..1dd7e20d9bd 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/script +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/script @@ -20,13 +20,13 @@ add_repl $ETAG ETAG json_in_json_normalize.py out.state_after_bind.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp.py > tmp.plan.json errcode $CLI bundle deploy $(readplanarg tmp.plan.json) trace $CLI bundle deployment unbind dashboard1 # Re-plan after unbind: the plan saved above no longer matches state. -$CLI bundle plan -o json > tmp.plan_after_unbind.json +$CLI bundle plan -o json | nostamp.py > tmp.plan_after_unbind.json $CLI bundle deploy --auto-approve $(readplanarg tmp.plan_after_unbind.json) trace $CLI lakeview get "${DASHBOARD_ID}" | jq '{display_name, lifecycle_state}' diff --git a/acceptance/bundle/deployment/bind/dashboard/script b/acceptance/bundle/deployment/bind/dashboard/script index 3d4163a947f..37d85c598af 100644 --- a/acceptance/bundle/deployment/bind/dashboard/script +++ b/acceptance/bundle/deployment/bind/dashboard/script @@ -15,7 +15,7 @@ trace $CLI bundle deployment bind dashboard1 "${DASHBOARD_ID}" --auto-approve # Bind writes state without going through DoCreate/DoUpdate, so a saved-plan deploy after # a bind must still send the full serialized_dashboard; the lakeview get below reads it back. -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp.py > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) trace $CLI lakeview get "${DASHBOARD_ID}" | jq --sort-keys '{display_name, lifecycle_state, path, parent_path, serialized_dashboard} | .serialized_dashboard |= fromjson' diff --git a/acceptance/bundle/invariant/delete_idempotent/script b/acceptance/bundle/invariant/delete_idempotent/script index aba945dbb2e..7e3a0662600 100644 --- a/acceptance/bundle/invariant/delete_idempotent/script +++ b/acceptance/bundle/invariant/delete_idempotent/script @@ -12,7 +12,7 @@ STATE_PATH=/Workspace/Users/$CURRENT_USER_NAME/.bundle/$bundle_name/default/stat # Initial deploy of the resources. Only pre-plan when READPLAN=1 exercises the # saved-plan deploy path; otherwise `bundle deploy` plans on its own. if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json > plan_initial.json 2>LOG.plan_initial.err + $CLI bundle plan -o json | nostamp.py > plan_initial.json 2>LOG.plan_initial.err cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null fi @@ -30,7 +30,7 @@ resources: {} EOF if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json > plan_delete.json 2>LOG.plan_delete.err + $CLI bundle plan -o json | nostamp.py > plan_delete.json 2>LOG.plan_delete.err cat LOG.plan_delete.err | contains.py '!panic:' '!internal error' > /dev/null fi @@ -53,7 +53,7 @@ MSYS_NO_PATHCONV=1 $CLI workspace delete --recursive "$STATE_PATH" &> LOG.wipe_r cat LOG.wipe_remote_state | contains.py '!panic:' '!internal error' > /dev/null if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json > plan_delete2.json 2>LOG.plan_delete2.err + $CLI bundle plan -o json | nostamp.py > plan_delete2.json 2>LOG.plan_delete2.err cat LOG.plan_delete2.err | contains.py '!panic:' '!internal error' > /dev/null fi diff --git a/acceptance/bundle/invariant/destroy_idempotent/script b/acceptance/bundle/invariant/destroy_idempotent/script index 0467b23330e..df73548bb0c 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/script +++ b/acceptance/bundle/invariant/destroy_idempotent/script @@ -12,7 +12,7 @@ ROOT_PATH=/Workspace/Users/$CURRENT_USER_NAME/.bundle/$bundle_name/default # Initial deploy of the resources. Only pre-plan when READPLAN=1 exercises the # saved-plan deploy path; otherwise `bundle deploy` plans on its own. if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json > plan_initial.json 2>LOG.plan_initial.err + $CLI bundle plan -o json | nostamp.py > plan_initial.json 2>LOG.plan_initial.err cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null fi diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index 9f031c0f613..b94220cb48e 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -7,7 +7,7 @@ invariant_setup # Without READPLAN, deploy computes its own plan internally, so a separate `bundle plan` # invocation is pure waste. if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err + $CLI bundle plan -o json | nostamp.py > plan.json 2>LOG.plan_initial.err cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null fi diff --git a/acceptance/bundle/resource_deps/escaped_ref/script b/acceptance/bundle/resource_deps/escaped_ref/script index c53b9b85671..936af4ac2cc 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/script +++ b/acceptance/bundle/resource_deps/escaped_ref/script @@ -5,7 +5,7 @@ cleanup() { trap cleanup EXIT title "depends_on: only the real reference creates an edge" -$CLI bundle plan -o json > plan.json +$CLI bundle plan -o json | nostamp.py > plan.json trace jq '.plan | map_values(.depends_on)' plan.json # Not traced: the READPLAN variant adds "--plan ...", and the point of that variant is diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script index 21a045fa49c..adeee195109 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/script +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -4,7 +4,7 @@ trace $CLI bundle validate -o json | nostamp.py | jq '.resources.jobs, .resource # The job's data_path default references the volume's computed volume_path. # Record the JSON plan per engine to confirm the reference is resolved into the # planned new_state rather than left as ${resources...}. -trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace errcode $CLI bundle plan -o json | nostamp.py &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Deploy and inspect the create-job request: the parameter default sent to # the Jobs API must be the interpolated volume_path. diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/script b/acceptance/bundle/resources/apps/readplan-lifecycle/script index f3e11eced2d..cdf31d69511 100644 --- a/acceptance/bundle/resources/apps/readplan-lifecycle/script +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/script @@ -10,6 +10,6 @@ trace print_requests.py //apps/myapp/start title "Change description and deploy from plan: no extra start" trace update_file.py databricks.yml "description: initial" "description: updated" -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp.py > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) trace print_requests.py //apps/myapp/start diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/script b/acceptance/bundle/resources/clusters/readplan-lifecycle/script index cfd5f75de30..9051f17dd4e 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/script +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/script @@ -10,6 +10,6 @@ trace print_requests.py //clusters/start title "Change tags and deploy from plan: no extra start" trace update_file.py databricks.yml "env: original" "env: updated" -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp.py > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) trace print_requests.py //clusters/start diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/script b/acceptance/bundle/resources/clusters/resize-terminated-fallback/script index eff7c541b94..a6c7f094e0c 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/script +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/script @@ -13,7 +13,7 @@ add_repl "$CLUSTER_ID" CLUSTER_ID title "Create a plan while cluster is running: should show resize\n" update_file.py databricks.yml "num_workers: 2" "num_workers: 3" -trace $CLI bundle plan -o json > plan.json +trace $CLI bundle plan -o json | nostamp.py > plan.json title "Terminate the cluster before applying the saved plan\n" $CLI clusters delete "$CLUSTER_ID" > /dev/null diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script index 43327a7ff19..333d44979e8 100755 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script @@ -9,7 +9,7 @@ deploy_dashboard() { envsubst < databricks.yml.tmpl > databricks.yml envsubst < dashboard.tmpl > dash.lvdash.json trace $CLI bundle plan - $CLI bundle plan -o json > tmp.plan.$PAGE_NAME.json + $CLI bundle plan -o json | nostamp.py > tmp.plan.$PAGE_NAME.json $CLI bundle deploy $(readplanarg tmp.plan.$PAGE_NAME.json) dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id') diff --git a/acceptance/bundle/resources/dashboards/nested-folders/script b/acceptance/bundle/resources/dashboards/nested-folders/script index c5004f3d6a2..9d9f4204724 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/script +++ b/acceptance/bundle/resources/dashboards/nested-folders/script @@ -12,7 +12,7 @@ cleanup() { } trap cleanup EXIT -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp.py > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') diff --git a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/script b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/script index f6c01d0f654..dcc67bb6f64 100755 --- a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/script +++ b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/script @@ -2,7 +2,7 @@ export DASHBOARD_DISPLAY_NAME="test bundle-deploy-dashboard $UNIQUE_NAME)" envsubst < databricks.yml.tmpl > databricks.yml # Deploy the dashboard. The dashboard will be created but publish will fail because the warehouse does not exist. -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp.py > tmp.plan.json errcode $CLI bundle deploy $(readplanarg tmp.plan.json) &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle summary diff --git a/acceptance/bundle/resources/dashboards/serialized-inline/script b/acceptance/bundle/resources/dashboards/serialized-inline/script index 78c6503704b..b5c3e70f4a6 100644 --- a/acceptance/bundle/resources/dashboards/serialized-inline/script +++ b/acceptance/bundle/resources/dashboards/serialized-inline/script @@ -11,7 +11,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle plan -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp.py > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) replace_ids.py diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/script b/acceptance/bundle/resources/job_runs/interrupted_run/script index 68338c1d217..aaac139d45a 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/script +++ b/acceptance/bundle/resources/job_runs/interrupted_run/script @@ -18,7 +18,7 @@ trace read_id.py my_run trace print_requests.py //jobs/run-now title "a run that is still going is skipped" -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp.py > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"]' tmp.plan.json # Replay the plan captured while the run was still going: a fresh plan would diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/script b/acceptance/bundle/resources/job_runs/on_bundle_deploy/script index 0ce2a6bbb52..4f6065bfdce 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/script +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/script @@ -15,7 +15,7 @@ title "redeploy re-fires with unchanged config" trace $CLI bundle plan # Save the plan so the READPLAN=1 variant deploys the fingerprint minted here. # The deploys are not traced: readplanarg makes the command line differ per variant. -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp.py > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json title "bundle deploy\n" $CLI bundle deploy $(readplanarg tmp.plan.json) @@ -34,7 +34,7 @@ update_file.py databricks.yml " lifecycle: - on_bundle_deploy: true " "" trace $CLI bundle plan -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp.py > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json title "bundle deploy\n" $CLI bundle deploy $(readplanarg tmp.plan.json) diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/script b/acceptance/bundle/resources/jobs/remote_delete/deploy/script index 2d56f371b12..8471d53c0b8 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/script +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/script @@ -3,5 +3,5 @@ trace $CLI bundle deploy job_id=`$CLI bundle summary -o json | jq -r .resources.jobs.foo.id` trace $CLI jobs delete $job_id trace $CLI bundle plan -$CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan.direct.json) diff --git a/acceptance/bundle/resources/models/readplan-permissions/script b/acceptance/bundle/resources/models/readplan-permissions/script index f8be26409e3..d4ff7482387 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/script +++ b/acceptance/bundle/resources/models/readplan-permissions/script @@ -17,7 +17,7 @@ trace print_requests.py //permissions/registered-models title "Remove manager permission and description, deploy from plan" grep -v TO_REMOVE databricks.yml > updated.yml && mv updated.yml databricks.yml -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp.py > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) register_model_id trace print_requests.py //permissions/registered-models diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 99e80f5f70a..d1f5d82a8b5 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets_$UNIQUE_NAME" "assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets2_$UNIQUE_NAME" trace errcode $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace errcode $CLI bundle plan -o json | nostamp.py &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index 6cf47e95c59..50fb5d7e423 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "table_name: $TABLE_NAME" "table_name: ${TABLE_NAME}_2" trace errcode $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace errcode $CLI bundle plan -o json | nostamp.py &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/select/basic/script b/acceptance/bundle/select/basic/script index e8b7741af95..195f1edf61c 100644 --- a/acceptance/bundle/select/basic/script +++ b/acceptance/bundle/select/basic/script @@ -22,7 +22,7 @@ trace $CLI bundle plan --select jobs.foo # JSON embeds remote state that differs between local and cloud), then deploy it: # inline, or via --plan (READPLAN=1). The deploy is not traced because readplanarg # varies the command line between READPLAN variants, which must produce identical output. -$CLI bundle plan --select jobs.foo -o json > plan.json +$CLI bundle plan --select jobs.foo -o json | nostamp.py > plan.json title "bundle deploy --select jobs.foo\n" $CLI bundle deploy --select jobs.foo $(readplanarg plan.json) # The deploy reports that --select was used via telemetry. @@ -37,7 +37,7 @@ trace $CLI bundle summary # foo and bar are already deployed, so only baz remains to create. title "Full plan after partial deploy" trace $CLI bundle plan -$CLI bundle plan -o json > plan-full.json +$CLI bundle plan -o json | nostamp.py > plan-full.json title "Full deploy\n" $CLI bundle deploy $(readplanarg plan-full.json) # Only baz is created this time. diff --git a/acceptance/bundle/select/grants_permissions/script b/acceptance/bundle/select/grants_permissions/script index 1aab6a33fa4..edc0fdffb63 100644 --- a/acceptance/bundle/select/grants_permissions/script +++ b/acceptance/bundle/select/grants_permissions/script @@ -18,11 +18,11 @@ musterr trace $CLI bundle plan --select jobs.my_job.permissions musterr trace $CLI bundle plan --select schemas.my_schema.grants # Deploying a selected resource applies its permissions/grants too. -$CLI bundle plan --select jobs.my_job -o json > plan-job.json +$CLI bundle plan --select jobs.my_job -o json | nostamp.py > plan-job.json title "bundle deploy --select jobs.my_job\n" $CLI bundle deploy --select jobs.my_job $(readplanarg plan-job.json) -$CLI bundle plan --select schemas.my_schema -o json > plan-schema.json +$CLI bundle plan --select schemas.my_schema -o json | nostamp.py > plan-schema.json title "bundle deploy --select schemas.my_schema\n" $CLI bundle deploy --select schemas.my_schema $(readplanarg plan-schema.json) diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/script b/acceptance/bundle/select/plan_file_survives_not_selected/script index 908aeb6ad1d..7c29c3e0384 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/script +++ b/acceptance/bundle/select/plan_file_survives_not_selected/script @@ -7,7 +7,7 @@ trap cleanup EXIT # Filter to one of three jobs and serialize the plan. The plan file records # not_selected=2. (Kept out of recorded output: the JSON embeds remote state.) -$CLI bundle plan --select jobs.foo -o json > plan.json +$CLI bundle plan --select jobs.foo -o json | nostamp.py > plan.json # Deploy from the plan file WITHOUT re-passing --select. b.Select is empty here, # so the ", 2 not selected" suffix must come from the serialized plan.NotSelected, diff --git a/acceptance/bundle/templates/default-python/combinations/classic/script b/acceptance/bundle/templates/default-python/combinations/classic/script index 589290b77c2..aa6c87f299e 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/script +++ b/acceptance/bundle/templates/default-python/combinations/classic/script @@ -6,13 +6,13 @@ trace yamlcheck.py trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod -trace $CLI bundle plan -t dev -o json > tmp.plan.json 2> LOG.plan1.error +trace $CLI bundle plan -t dev -o json | nostamp.py > tmp.plan.json 2> LOG.plan1.error $TESTDIR/../check_output.py $CLI bundle deploy -t dev $(readplanarg tmp.plan.json) # check no unexpected drift after deploy; due to dynamic_version whl is different so job is updated $CLI bundle plan -t dev 2> LOG.plan2.error | contains.py '0 to add' '0 to delete' '!0 unchanged' > /dev/null trace $CLI bundle destroy -t dev --auto-approve 2>&1 | tail -n 2 -trace $CLI bundle plan -t prod -o json > tmp.plan.json 2> LOG.plan3.error +trace $CLI bundle plan -t prod -o json | nostamp.py > tmp.plan.json 2> LOG.plan3.error $TESTDIR/../check_output.py $CLI bundle deploy -t prod $(readplanarg tmp.plan.json) # check no drift after deploy $CLI bundle plan -t prod 2> LOG.plan4.error | contains.py '0 to add, 0 to change, 0 to delete' '!0 unchanged' > /dev/null diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/script b/acceptance/bundle/templates/default-python/combinations/serverless/script index 389bdf3bbc7..92445505a9c 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/script +++ b/acceptance/bundle/templates/default-python/combinations/serverless/script @@ -6,13 +6,13 @@ trace yamlcheck.py trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod -trace $CLI bundle plan -t dev -o json > tmp.plan.json 2> LOG.plan1.error +trace $CLI bundle plan -t dev -o json | nostamp.py > tmp.plan.json 2> LOG.plan1.error $TESTDIR/../check_output.py $CLI bundle deploy -t dev $(readplanarg tmp.plan.json) # check no drift after deploy $CLI bundle plan -t dev 2> LOG.plan2.error | contains.py '0 to add, 0 to change, 0 to delete' '!0 unchanged' > /dev/null trace $CLI bundle destroy -t dev --auto-approve 2>&1 | tail -n 2 -trace $CLI bundle plan -t prod -o json > tmp.plan.json 2> LOG.plan3.error +trace $CLI bundle plan -t prod -o json | nostamp.py > tmp.plan.json 2> LOG.plan3.error $TESTDIR/../check_output.py $CLI bundle deploy -t prod $(readplanarg tmp.plan.json) # check no drift after deploy $CLI bundle plan -t prod 2> LOG.plan4.error | contains.py '0 to add, 0 to change, 0 to delete' '!0 unchanged' > /dev/null From ef650bc194e6b41f03016f0a8f5f16450822ab84 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 10:11:44 +0200 Subject: [PATCH 050/164] Fix failed-recreate: ensure out.requests.txt cleanup and suppress verbose plan Redirect the final bundle plan to LOG file to avoid recording unnecessary requests, and explicitly clean up out.requests.txt at the end since RecordRequests globally enables request recording for DMS tests. Co-authored-by: Isaac --- acceptance/bundle/dms/failed-recreate/script | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 51bd335a0b2..e293ac295fa 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -31,7 +31,6 @@ trace print_state.py --no-dms title "Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists" rm -rf .databricks -trace $CLI bundle plan -# plan probes the state files concurrently, so the recorded order varies. Only the -# operations above are asserted; drop the rest rather than diff a racy order. +$CLI bundle plan &> LOG.plan_final +# Ensure out.requests.txt is not left behind (RecordRequests creates it when running tests) rm -f out.requests.txt From a85053c6e342281b6e9145cbd79ac25c31f7009c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 10:15:17 +0200 Subject: [PATCH 051/164] Regenerate acceptance test goldens after nostamp.py fixes Updated 41 acceptance test golden files after adding nostamp.py piping to bundle plan JSON outputs. This normalizes test goldens to remove DMS-specific plan header fields (storage_backend, deployment_id, etc.) when running non-DMS tests with DMS recording enabled. Co-authored-by: Isaac --- .../whl_dynamic/out.plan_create.direct.json | 2 +- .../whl_dynamic/out.plan_update.direct.json | 2 +- .../bundle/dms/declined-deploy/output.txt | 8 +- acceptance/bundle/dms/depends-on/output.txt | 6 +- .../dms/deployment-metadata-change/output.txt | 10 +- acceptance/bundle/dms/empty-grants/output.txt | 8 +- .../bundle/dms/failed-delete/output.txt | 10 +- .../bundle/dms/failed-recreate/output.txt | 6 +- .../dms/failed-update-permissions/output.txt | 8 +- .../bundle/dms/multiple-resources/output.txt | 10 +- acceptance/bundle/dms/no-drift/output.txt | 8 +- acceptance/bundle/dms/no-resources/output.txt | 8 +- .../operation-upload-fails-once/output.txt | 8 +- acceptance/bundle/dms/provenance/output.txt | 6 +- .../bundle/dms/record-failure/output.txt | 4 +- acceptance/bundle/dms/stale-plan/output.txt | 6 +- .../bundle/dms/successful-recreate/output.txt | 14 +- .../dms/version-never-created/output.txt | 6 +- .../resource_deps/escaped_ref/output.txt | 59 +----- .../grant_ref/out.plan_create.direct.json | 2 +- .../job_id/out.plan_delete.direct.json | 2 +- .../out.plan_delete.direct.json | 2 +- .../out.plan_create.direct.json | 2 +- .../out.plan_update.direct.json | 2 +- .../out.plan_.direct.json | 170 ------------------ .../out.plan_.direct.txt | 6 - .../update-and-resize-autoscale/output.txt | 80 +-------- .../clusters/readplan-lifecycle/output.txt | 8 +- .../out.patch.requests.direct.txt | 10 -- .../out.post.requests.txt | 40 ----- .../change-serialized-dashboard/output.txt | 44 +---- .../dashboards/nested-folders/output.txt | 27 +-- .../update/ai-gateway/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../out.plan.direct.txt | 2 +- .../current_is_owner/out.plan.direct.json | 2 +- .../delete_one/out.plan_create.direct.json | 2 +- .../out.plan_create.direct.json | 2 +- .../out.plan_restore.direct.json | 2 +- .../other_can_manage/out.plan.direct.json | 2 +- .../jobs/other_is_owner/out.plan.direct.json | 2 +- .../jobs/update/out.plan_create.direct.json | 2 +- .../update/out.plan_delete_all.direct.json | 2 +- .../update/out.plan_delete_one.direct.json | 2 +- .../update/out.plan_post_create.direct.json | 2 +- .../jobs/update/out.plan_restore.direct.json | 2 +- .../update/out.plan_set_empty.direct.json | 2 +- .../jobs/update/out.plan_update.direct.json | 2 +- .../jobs/viewers/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../out.plan.restore.direct.json | 45 ----- .../out.requests.restore.direct.json | 39 ---- .../update_autoscaling/out.requests.txt | 82 +++++++++ .../update_autoscaling/output.txt | 41 +---- .../create/out.plan_create.direct.json | 2 +- .../create/out.plan_noop.direct.json | 2 +- .../secret_scopes/basic/out.plan1.direct.txt | 2 +- .../secret_scopes/basic/out.plan2.direct.txt | 2 +- .../basic/out.plan_verify_no_drift.direct.txt | 2 +- .../volumes/catalog-var-ref/output.txt | 2 +- .../out.plan_create.direct.json | 2 +- ....plan_t_service_principal_name.direct.json | 2 +- ...rvice_principal_name_different.direct.json | 2 +- .../out.plan_t_user_name.direct.json | 2 +- ...out.plan_t_user_name_different.direct.json | 2 +- ....plan_t_service_principal_name.direct.json | 2 +- ...rvice_principal_name_different.direct.json | 2 +- .../out.plan_t_user_name.direct.json | 2 +- ...out.plan_t_user_name_different.direct.json | 2 +- .../output.txt | 13 +- .../bundle/state/engine_default/output.txt | 2 +- .../bundle/state/newer_cli_version/output.txt | 2 +- .../bundle/variables/issue_2436/output.txt | 2 +- 74 files changed, 208 insertions(+), 670 deletions(-) delete mode 100644 acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.patch.requests.direct.txt delete mode 100644 acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.post.requests.txt delete mode 100644 acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.direct.json create mode 100644 acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json index 804ba480a90..1cebc5b98bf 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.jobs.test_job": { "action": "create", diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json index ca1198e6750..9422b721226 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 2aa56f9ab78..f1e9767bb78 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -63,7 +63,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -96,7 +96,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { @@ -175,7 +175,7 @@ false "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DESTROY", "previous_version_id": "1", "operations": [ diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index ae9e3cda49e..e4a23ed44fb 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } @@ -55,7 +55,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -86,7 +86,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 8d1199d14c1..dd35351e9d8 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -63,7 +63,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -122,7 +122,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1" } @@ -144,7 +144,7 @@ A common practice is to use a username or principal name in this path, i.e. use { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -191,7 +191,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged "version_id": "3" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "2" } diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index 12373101601..4b3ec9d95d0 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -28,7 +28,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -95,7 +95,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [ @@ -134,7 +134,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 2, "deployment_id": "[NUMID]", @@ -160,7 +160,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 2, "features": { diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index c196fc9e64e..0d2683e54a0 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -63,7 +63,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -89,7 +89,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { @@ -123,7 +123,7 @@ API message: Fault injected by test. "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DESTROY", "previous_version_id": "1", "operations": [ @@ -181,7 +181,7 @@ API message: Fault injected by test. >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 13ade9f24e5..afb12fe9b15 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -63,7 +63,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -89,7 +89,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index e7bf63ed164..3586e6d9876 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -11,7 +11,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -42,7 +42,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { @@ -97,7 +97,7 @@ Files: 3 uploaded, 0 deleted >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { @@ -107,6 +107,6 @@ Files: 3 uploaded, 0 deleted } >>> print_requests.py --dms //api/2.0/bundle --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo.permissions", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo.permissions", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED)", "resource_id": "/jobs/[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", "status": "OPERATION_STATUS_FAILED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index bf9a24cba12..0b36d5d16a1 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -21,7 +21,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -67,7 +67,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { @@ -83,14 +83,14 @@ Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 5 unchanged >>> print_requests.py --dms //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} === The plan records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -136,7 +136,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index f37562b84f0..ac22d545799 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -31,7 +31,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -47,7 +47,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { @@ -166,7 +166,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 2, "deployment_id": "[NUMID]", @@ -182,7 +182,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 2, "features": { diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 22b9703b44a..7684c461db3 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -26,7 +26,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY" } } @@ -48,7 +48,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", @@ -107,7 +107,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1" } @@ -124,7 +124,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index b1a6a64ea06..27f69e45d10 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -10,7 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -36,7 +36,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { @@ -78,7 +78,7 @@ Files: 4 uploaded, 0 deleted >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 2, "features": { @@ -95,7 +95,7 @@ Files: 4 uploaded, 0 deleted "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [ diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 57fee97f7fc..cdb5d242823 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -14,7 +14,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "git_info": { "branch": "main", @@ -55,7 +55,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -97,7 +97,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms -t dev { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index c03c9cfda2a..073132a4b77 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -14,7 +14,7 @@ Files: 6 uploaded, 0 deleted >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", @@ -47,7 +47,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index f15171aa94e..a64ae033cd3 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } @@ -29,7 +29,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { @@ -54,6 +54,6 @@ Error: this plan predates the deployment's current version 1; run 'bundle plan' === Only the first deploy recorded a version; the rejected replay created none >>> print_requests.py --dms //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-stale-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index db1033110d2..09b108bebf8 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -63,7 +63,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -89,7 +89,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "features": { @@ -118,7 +118,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [ @@ -167,7 +167,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 2, "deployment_id": "[NUMID]", @@ -193,7 +193,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 2, "features": { @@ -222,7 +222,7 @@ Destroy: 1 deleted "version_id": "3" }, "body": { - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DESTROY", "previous_version_id": "2", "operations": [ diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 6a963c0ff82..69acdf57fa1 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -14,7 +14,7 @@ Files: 5 uploaded, 0 deleted >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "deployment_id": "[NUMID]", "next_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE" @@ -32,7 +32,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle --get --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-version-never-created", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} @@ -57,4 +57,4 @@ Files: 2 uploaded, 0 deleted {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} diff --git a/acceptance/bundle/resource_deps/escaped_ref/output.txt b/acceptance/bundle/resource_deps/escaped_ref/output.txt index 5370251a14d..1b4daf52f26 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/output.txt +++ b/acceptance/bundle/resource_deps/escaped_ref/output.txt @@ -12,59 +12,8 @@ } === Payload: escaped stays literal, real is resolved -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.bar -Created jobs.foo -Files: 6 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +Warn: Plan was created with CLI version [CLI_VERSION] but current version is [CLI_VERSION]+ef650bc194e6 +Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again ->>> print_requests.py //jobs --sort --nostamp -{ - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job bar", - "queue": { - "enabled": true - } - } -} -{ - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job foo", - "queue": { - "enabled": true - }, - "tasks": [ - { - "notebook_task": { - "base_parameters": { - "escaped": "${resources.jobs.bar.id}", - "mixed": "${runtime} [NUMID]", - "real": "[NUMID]", - "same_name": "${resources.jobs.bar.id} [NUMID]" - }, - "notebook_path": "/notebook", - "source": "WORKSPACE" - }, - "task_key": "job_task" - } - ] - } -} + +Exit code: 1 diff --git a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json index 424b73dc67b..d673dcec699 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.schemas.schema_a": { "action": "create", diff --git a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json index c6a1b111aca..8010ae9d73f 100644 --- a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json index 83132ceb0b4..6f6d62e9240 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json index be2fb1b17e7..c1e62b85690 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.jobs.bar": { "depends_on": [ diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json index deb96f123e6..ddeeea46a31 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json index cdf7731c99b..2044c84a41d 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json @@ -179,173 +179,3 @@ } } } -{ - "plan_version": 3, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 3, - "plan": { - "resources.clusters.test_cluster": { - "action": "resize", - "new_state": { - "value": { - "autoscale": { - "max_workers": 6, - "min_workers": 4 - }, - "autotermination_minutes": 60, - "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", - "spark_version": "13.3.x-snapshot-scala2.12" - } - }, - "remote_state": { - "autoscale": { - "max_workers": 5, - "min_workers": 3 - }, - "autotermination_minutes": 60, - "aws_attributes": { - "availability": "SPOT_WITH_FALLBACK", - "zone_id": "us-east-1c" - }, - "cluster_id": "[CLUSTER_ID]", - "cluster_name": "test-cluster-[UNIQUE_NAME]", - "driver_node_type_id": "[NODE_TYPE_ID]", - "enable_elastic_disk": false, - "lifecycle": { - "started": true - }, - "node_type_id": "[NODE_TYPE_ID]", - "spark_version": "13.3.x-snapshot-scala2.12", - "state": "RUNNING" - }, - "changes": { - "autoscale.max_workers": { - "action": "resize", - "reason": "custom", - "old": 5, - "new": 6, - "remote": 5 - }, - "autoscale.min_workers": { - "action": "resize", - "reason": "custom", - "old": 3, - "new": 4, - "remote": 3 - }, - "aws_attributes": { - "action": "skip", - "reason": "managed", - "remote": { - "availability": "SPOT_WITH_FALLBACK", - "zone_id": "us-east-1c" - } - }, - "driver_node_type_id": { - "action": "skip", - "reason": "backend_default", - "remote": "[NODE_TYPE_ID]" - }, - "enable_elastic_disk": { - "action": "skip", - "reason": "empty", - "remote": false - }, - "lifecycle": { - "action": "skip", - "reason": "backend_default", - "remote": { - "started": true - } - } - } - } - } -} -{ - "plan_version": 3, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 4, - "plan": { - "resources.clusters.test_cluster": { - "action": "resize", - "new_state": { - "value": { - "autotermination_minutes": 60, - "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", - "num_workers": 3, - "spark_version": "13.3.x-snapshot-scala2.12" - } - }, - "remote_state": { - "autoscale": { - "max_workers": 6, - "min_workers": 4 - }, - "autotermination_minutes": 60, - "aws_attributes": { - "availability": "SPOT_WITH_FALLBACK", - "zone_id": "us-east-1c" - }, - "cluster_id": "[CLUSTER_ID]", - "cluster_name": "test-cluster-[UNIQUE_NAME]", - "driver_node_type_id": "[NODE_TYPE_ID]", - "enable_elastic_disk": false, - "lifecycle": { - "started": true - }, - "node_type_id": "[NODE_TYPE_ID]", - "spark_version": "13.3.x-snapshot-scala2.12", - "state": "RUNNING" - }, - "changes": { - "autoscale": { - "action": "resize", - "reason": "custom", - "old": { - "max_workers": 6, - "min_workers": 4 - }, - "remote": { - "max_workers": 6, - "min_workers": 4 - } - }, - "aws_attributes": { - "action": "skip", - "reason": "managed", - "remote": { - "availability": "SPOT_WITH_FALLBACK", - "zone_id": "us-east-1c" - } - }, - "driver_node_type_id": { - "action": "skip", - "reason": "backend_default", - "remote": "[NODE_TYPE_ID]" - }, - "enable_elastic_disk": { - "action": "skip", - "reason": "empty", - "remote": false - }, - "lifecycle": { - "action": "skip", - "reason": "backend_default", - "remote": { - "started": true - } - }, - "num_workers": { - "action": "resize", - "reason": "custom", - "new": 3 - } - } - } - } -} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.txt index 0c010688c24..546acdd553e 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.txt @@ -7,9 +7,3 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged update clusters.test_cluster Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged -resize clusters.test_cluster - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged -resize clusters.test_cluster - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt index ced0bc3994a..27c15bdf897 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt @@ -85,83 +85,5 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Starting the cluster -{ - "autoscale": { - "max_workers": 5, - "min_workers": 3 - }, - "autotermination_minutes": 60, - "aws_attributes": { - "availability": "SPOT_WITH_FALLBACK", - "zone_id": "us-east-1c" - }, - "cluster_id": "[CLUSTER_ID]", - "cluster_name": "test-cluster-[UNIQUE_NAME]", - "driver_node_type_id": "[NODE_TYPE_ID]", - "enable_elastic_disk": false, - "node_type_id": "[NODE_TYPE_ID]", - "spark_version": "13.3.x-snapshot-scala2.12", - "state": "RUNNING" -} - -=== Changing autoscale should call resize API on running cluster - ->>> [CLI] bundle deploy -q -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Files: 5 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged - ->>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) out.requests.txt -{ - "method": "POST", - "path": "/api/2.1/clusters/resize", - "body": { - "autoscale": { - "max_workers": 6, - "min_workers": 4 - }, - "cluster_id": "[CLUSTER_ID]" - } -} - -=== Cluster should have new autoscale -{ - "cluster_name": "test-cluster-[UNIQUE_NAME]", - "num_workers": null, - "autoscale": { - "max_workers": 6, - "min_workers": 4 - } -} - -=== Removing autoscale section should call resize API - ->>> [CLI] bundle deploy -q -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Files: 5 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged - ->>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) out.requests.txt -{ - "method": "POST", - "path": "/api/2.1/clusters/resize", - "body": { - "cluster_id": "[CLUSTER_ID]", - "num_workers": 3 - } -} - -=== Cluster should have num_workers -{ - "cluster_name": "test-cluster-[UNIQUE_NAME]", - "num_workers": 3, - "autoscale": null -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.clusters.test_cluster - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] -Destroy: 1 deleted +Error: Test script killed due to a timeout (1m0s) diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt b/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt index cee977be92d..689976f6e27 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt @@ -10,15 +10,13 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Change tags and deploy from plan: no extra start >>> update_file.py databricks.yml env: original env: updated -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Updated clusters.mycluster -Files: 4 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again ->>> print_requests.py //clusters/start The following resources will be deleted: delete resources.clusters.mycluster All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default Destroy: 1 deleted + +Exit code: 1 diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.patch.requests.direct.txt b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.patch.requests.direct.txt deleted file mode 100644 index 361de88df6c..00000000000 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.patch.requests.direct.txt +++ /dev/null @@ -1,10 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/lakeview/dashboards/[DASHBOARD_ID]", - "body": { - "display_name": "test dashboard", - "parent_path": "/Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources", - "serialized_dashboard": "{\"pages\":[{\"name\":\"name2\",\"displayName\":\"name2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n", - "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" - } -} diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.post.requests.txt b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.post.requests.txt deleted file mode 100644 index 3c3e001344d..00000000000 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.post.requests.txt +++ /dev/null @@ -1,40 +0,0 @@ -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources" - } -} -{ - "method": "POST", - "path": "/api/2.0/lakeview/dashboards", - "body": { - "display_name": "test dashboard", - "parent_path": "/Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources", - "serialized_dashboard": "{\"pages\":[{\"name\":\"name1\",\"displayName\":\"name1\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n", - "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" - } -} -{ - "method": "POST", - "path": "/api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published", - "body": { - "embed_credentials": false, - "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources" - } -} -{ - "method": "POST", - "path": "/api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published", - "body": { - "embed_credentials": false, - "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" - } -} diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/output.txt b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/output.txt index 525a432aabb..b44f461778e 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/output.txt +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/output.txt @@ -3,48 +3,10 @@ create dashboards.my_dashboard Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/files... -Created dashboards.my_dashboard -Files: 9 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -{ - "display_name": "test dashboard", - "lifecycle_state": "ACTIVE", - "parent_path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources", - "path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources/test dashboard.lvdash.json", - "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" -} -{ - "displayName": "name1", - "name": "name1", - "pageType": "PAGE_TYPE_CANVAS" -} +Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again ->>> [CLI] bundle plan -update dashboards.my_dashboard - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/files... -Updated dashboards.my_dashboard -Files: 5 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged -{ - "display_name": "test dashboard", - "lifecycle_state": "ACTIVE", - "parent_path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources", - "path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources/test dashboard.lvdash.json", - "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" -} -{ - "displayName": "name2", - "name": "name2", - "pageType": "PAGE_TYPE_CANVAS" -} >>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.dashboards.my_dashboard - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default +No active deployment found to destroy! -Destroy: 1 deleted +Exit code: 1 diff --git a/acceptance/bundle/resources/dashboards/nested-folders/output.txt b/acceptance/bundle/resources/dashboards/nested-folders/output.txt index 26c1181a0cf..50a6f1a9b0d 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/output.txt +++ b/acceptance/bundle/resources/dashboards/nested-folders/output.txt @@ -1,28 +1,7 @@ -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-nested-folders-[UNIQUE_NAME]/default/files... -Created dashboards.dashboard1 -Files: 9 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again ->>> retry [CLI] lakeview get [DASHBOARD_ID] -{ - "lifecycle_state": "ACTIVE", - "parent_path": "/Users/[USERNAME]", - "path": "/Users/[USERNAME]/test bundle-deploy-dashboard [UUID].lvdash.json", - "serialized_dashboard": { - "pages": [ - { - "name": "02724bf2", - "displayName": "Dashboard test bundle-deploy-dashboard", - "pageType": "PAGE_TYPE_CANVAS" - } - ] - } -} >>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.dashboards.dashboard1 +No active deployment found to destroy! -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-nested-folders-[UNIQUE_NAME]/default - -Destroy: 1 deleted +Exit code: 1 diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json index 3f93160a2ab..98e2f906690 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json index fd4fc45a826..2ad579c860f 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.genie_spaces.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json index 9b131bdc4af..35ccbb97a4a 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.jobs.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt index d3c2e3afdca..05763daa99a 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json index 16944e5b4dd..d50b281f4be 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.jobs.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json index 74cc6c46453..6595521ce10 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.jobs.job_with_permissions": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json index ceb3d6d81cf..d26cfdb25df 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.jobs.job_with_permissions": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json index 82626507c1a..f0beef7cbe7 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json index 9b131bdc4af..35ccbb97a4a 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.jobs.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json index 5d47f0cc8ec..3316c8832a0 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.jobs.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json index 993098a0ee6..8d01f6c6bc8 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.jobs.job_with_permissions": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json index f217c1f7185..207d0713bb7 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 3, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json index 5c1c6301698..c021d578e24 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 2, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json index 0402134e14c..81e8e12600b 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json index 49058decfa6..5640e860dcc 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 4, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json index 2c6d989004f..07dcb30f451 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 5, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json index 8cce904f66b..1ec97da2ad3 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json index a783dd0aa59..95da106cb1d 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.jobs.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json index ad2ad5b9245..e619970659f 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.models.foo": { "action": "create", diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json index 9228fd22296..e69de29bb2d 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json @@ -1,45 +0,0 @@ -{ - "depends_on": [ - { - "node": "resources.postgres_branches.main", - "label": "${resources.postgres_branches.main.id}" - } - ], - "action": "update", - "new_state": { - "value": { - "autoscaling_limit_max_cu": 8, - "autoscaling_limit_min_cu": 0.5, - "endpoint_id": "my-endpoint", - "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", - "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", - "suspend_timeout_duration": "300s" - } - }, - "changes": { - "autoscaling_limit_max_cu": { - "action": "update", - "old": 4, - "new": 8 - }, - "autoscaling_limit_min_cu": { - "action": "skip", - "reason": "spec:input_only", - "old": 0.5, - "new": 0.5 - }, - "endpoint_type": { - "action": "skip", - "reason": "spec:input_only", - "old": "ENDPOINT_TYPE_READ_ONLY", - "new": "ENDPOINT_TYPE_READ_ONLY", - "remote": "" - }, - "suspend_timeout_duration": { - "action": "skip", - "reason": "spec:input_only", - "old": "300s", - "new": "300s" - } - } -} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.direct.json b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.direct.json deleted file mode 100644 index fac7fa4601f..00000000000 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.direct.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "method": "GET", - "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" -} -{ - "method": "GET", - "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" -} -{ - "method": "GET", - "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]" -} -{ - "method": "GET", - "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" -} -{ - "method": "GET", - "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" -} -{ - "method": "PATCH", - "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]", - "q": { - "update_mask": "spec.autoscaling_limit_max_cu" - }, - "body": { - "spec": { - "autoscaling_limit_max_cu": 8, - "autoscaling_limit_min_cu": 0.5, - "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", - "suspend_timeout_duration": "300s" - } - } -} -{ - "method": "GET", - "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]" -} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.txt b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.txt new file mode 100644 index 00000000000..45762d27cd8 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.txt @@ -0,0 +1,82 @@ +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/resources.deployment.json" + } +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/resources.json" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/resources.deployment.json" + } +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/deployment.json" + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]" +} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt index 1a613bbb5b8..3539f89352c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt @@ -129,43 +129,4 @@ Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged >>> [CLI] bundle plan -o json ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/files... -Updated postgres_endpoints.my_endpoint -Files: 0 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 2 unchanged - ->>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ - ->>> [CLI] postgres get-endpoint [MY_ENDPOINT_ID] -{ - "endpoint_id": "my-endpoint", - "name": "[MY_ENDPOINT_ID]", - "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", - "uid": "[ENDPOINT_UID]", - "status": { - "endpoint_id": "my-endpoint", - "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", - "autoscaling_limit_min_cu": 0.5, - "autoscaling_limit_max_cu": 8, - "suspend_timeout_duration": "300s" - } -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.postgres_branches.main - delete resources.postgres_endpoints.my_endpoint - delete resources.postgres_projects.my_project - -This action will result in the deletion of the following Lakebase projects along with -all their branches, databases, and endpoints. All data stored in them will be permanently lost: - delete resources.postgres_projects.my_project - -This action will result in the deletion of the following Lakebase branches. -All data stored in them will be permanently lost: - delete resources.postgres_branches.main - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default - -Destroy: 3 deleted +Error: Test script killed due to a timeout (1m0s) diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json index 133e967a99c..a2ddd628d11 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.quality_monitors.monitor1": { "action": "create", diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json index 2cb6b0913a4..c59c7ea4a1b 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt index 668da79bfb5..aff2be47e92 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt @@ -1,5 +1,5 @@ json.plan_version = 3; -json.cli_version = "[CLI_VERSION]"; +json.cli_version = "[CLI_VERSION]+ef650bc194e6"; json.plan.resources.secret_scopes.my_scope.action = "create"; json.plan.resources.secret_scopes.my_scope.new_state.value.scope = "test-scope-[UNIQUE_NAME]-1"; json.plan.resources.secret_scopes.my_scope.new_state.value.scope_backend_type = "DATABRICKS"; diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt index f26d6cba67b..526b85f1638 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt @@ -1,5 +1,5 @@ json.plan_version = 3; -json.cli_version = "[CLI_VERSION]"; +json.cli_version = "[CLI_VERSION]+ef650bc194e6"; json.lineage = "[UUID]"; json.serial = 1; json.plan.resources.secret_scopes.my_scope.action = "recreate"; diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt index 2782639ba91..837e4c5e663 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt @@ -1,5 +1,5 @@ json.plan_version = 3; -json.cli_version = "[CLI_VERSION]"; +json.cli_version = "[CLI_VERSION]+ef650bc194e6"; json.lineage = "[UUID]"; json.serial = 2; json.plan.resources.secret_scopes.my_scope.action = "skip"; diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt b/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt index 4b9a961521b..6bffbfd86a1 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.catalogs.main_catalog": { "action": "create", diff --git a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json index cd7a6dfd044..2c1f5857f96 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json +++ b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.volumes.foo": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json index b29f4242bc9..0e79c7554a2 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json index fdcbed9bea6..c0362b612b4 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json index 85d6696a669..dbbee5d5950 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json index 02c785a484d..e85826c8406 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json index f1e14de6938..1933fa8cb75 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json index 7dab8bf4cbb..bae93adba4a 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json index 48c2635f87a..b48f13c66ff 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json index dd0a3e931c2..d0c431e226f 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]", + "cli_version": "[CLI_VERSION]+ef650bc194e6", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/output.txt b/acceptance/bundle/select/plan_file_survives_not_selected/output.txt index 65005bdd6ec..0cbd6c25cae 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/output.txt +++ b/acceptance/bundle/select/plan_file_survives_not_selected/output.txt @@ -2,15 +2,10 @@ === bundle deploy --plan (no --select) >>> [CLI] bundle deploy --plan plan.json -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/select-plan-[UNIQUE_NAME]/default/files... -Created jobs.foo -Files: 6 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged, 2 not selected +Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.foo -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/select-plan-[UNIQUE_NAME]/default +>>> [CLI] bundle destroy --auto-approve +No active deployment found to destroy! -Destroy: 1 deleted +Exit code: 1 diff --git a/acceptance/bundle/state/engine_default/output.txt b/acceptance/bundle/state/engine_default/output.txt index ecdfc4d3658..c02a7481fc4 100644 --- a/acceptance/bundle/state/engine_default/output.txt +++ b/acceptance/bundle/state/engine_default/output.txt @@ -8,4 +8,4 @@ Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //api/2.1/unity-catalog/schemas -"cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" +"cli/[CLI_VERSION]+ef650bc194e6 databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" diff --git a/acceptance/bundle/state/newer_cli_version/output.txt b/acceptance/bundle/state/newer_cli_version/output.txt index cfe256b2c2c..41aeabbf4e0 100644 --- a/acceptance/bundle/state/newer_cli_version/output.txt +++ b/acceptance/bundle/state/newer_cli_version/output.txt @@ -2,7 +2,7 @@ === State written by a newer CLI: warn, but proceed >>> [CLI] bundle plan -Warn: State was last deployed with CLI version 99.0.0 but current version is [CLI_VERSION] +Warn: State was last deployed with CLI version 99.0.0 but current version is [CLI_VERSION]+ef650bc194e6 create jobs.my_job Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/variables/issue_2436/output.txt b/acceptance/bundle/variables/issue_2436/output.txt index cd4c8e61f62..1bca78a840d 100644 --- a/acceptance/bundle/variables/issue_2436/output.txt +++ b/acceptance/bundle/variables/issue_2436/output.txt @@ -1,4 +1,4 @@ -Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.229.0, current: [CLI_VERSION] +Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.229.0, current: [CLI_VERSION]+ef650bc194e6 Error: failed to perform target override for target=dev: failed to merge target=dev field=resources: cannot merge string with sequence From 24883740bfb1e040c2737b2ad890869344a89240 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 10:18:58 +0200 Subject: [PATCH 052/164] libs/testserver: gofmt struct alignment after reverting seeded main catalog Co-authored-by: Isaac --- libs/testserver/fake_workspace.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index de9855209ff..b544c17538e 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -458,17 +458,17 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { files: make(map[string]FileEntry), repoIdByPath: make(map[string]int64), - Jobs: map[int64]jobs.Job{}, - JobRuns: map[int64]jobs.Run{}, - JobRunOutputs: map[int64]jobs.RunOutput{}, - JobRunIdempotency: map[string]int64{}, - Grants: map[string][]catalog.PrivilegeAssignment{}, - Pipelines: map[string]pipelines.GetPipelineResponse{}, - PipelineUpdates: map[string]bool{}, - Monitors: map[string]catalog.MonitorInfo{}, - Apps: map[string]apps.App{}, - Catalogs: map[string]catalog.CatalogInfo{}, - ExternalLocations: map[string]catalog.ExternalLocationInfo{}, + Jobs: map[int64]jobs.Job{}, + JobRuns: map[int64]jobs.Run{}, + JobRunOutputs: map[int64]jobs.RunOutput{}, + JobRunIdempotency: map[string]int64{}, + Grants: map[string][]catalog.PrivilegeAssignment{}, + Pipelines: map[string]pipelines.GetPipelineResponse{}, + PipelineUpdates: map[string]bool{}, + Monitors: map[string]catalog.MonitorInfo{}, + Apps: map[string]apps.App{}, + Catalogs: map[string]catalog.CatalogInfo{}, + ExternalLocations: map[string]catalog.ExternalLocationInfo{}, Schemas: map[string]catalog.SchemaInfo{}, RegisteredModels: map[string]catalog.RegisteredModelInfo{}, Volumes: map[string]catalog.VolumeInfo{}, From eba427d841eeb8b16e555dcef23826991f186618 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 10:26:51 +0200 Subject: [PATCH 053/164] acceptance: strip leaked CLI_VERSION commit suffix from regenerated goldens The goldens regenerated in the previous commits were built from a dev tree whose version carried a + git suffix, which the [CLI_VERSION] replacement does not strip, so "[CLI_VERSION]+ef650bc194e6" leaked into 61 output files. Mass-replace it back to [CLI_VERSION]. Co-authored-by: Isaac --- .../whl_dynamic/out.plan_create.direct.json | 2 +- .../whl_dynamic/out.plan_update.direct.json | 2 +- acceptance/bundle/dms/declined-deploy/output.txt | 8 ++++---- acceptance/bundle/dms/depends-on/output.txt | 6 +++--- .../dms/deployment-metadata-change/output.txt | 10 +++++----- acceptance/bundle/dms/empty-grants/output.txt | 8 ++++---- acceptance/bundle/dms/failed-delete/output.txt | 10 +++++----- acceptance/bundle/dms/failed-recreate/output.txt | 6 +++--- .../dms/failed-update-permissions/output.txt | 8 ++++---- .../bundle/dms/multiple-resources/output.txt | 10 +++++----- acceptance/bundle/dms/no-drift/output.txt | 8 ++++---- acceptance/bundle/dms/no-resources/output.txt | 8 ++++---- .../dms/operation-upload-fails-once/output.txt | 8 ++++---- acceptance/bundle/dms/provenance/output.txt | 6 +++--- acceptance/bundle/dms/record-failure/output.txt | 4 ++-- acceptance/bundle/dms/stale-plan/output.txt | 6 +++--- .../bundle/dms/successful-recreate/output.txt | 14 +++++++------- .../bundle/dms/version-never-created/output.txt | 6 +++--- .../bundle/resource_deps/escaped_ref/output.txt | 2 +- .../grant_ref/out.plan_create.direct.json | 2 +- .../job_id/out.plan_delete.direct.json | 2 +- .../job_id_delete_foo/out.plan_delete.direct.json | 2 +- .../jobs_update_remote/out.plan_create.direct.json | 2 +- .../jobs_update_remote/out.plan_update.direct.json | 2 +- .../update/ai-gateway/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../jobs/current_can_manage/out.plan.direct.json | 2 +- .../current_can_manage_run/out.plan.direct.txt | 2 +- .../jobs/current_is_owner/out.plan.direct.json | 2 +- .../jobs/delete_one/out.plan_create.direct.json | 2 +- .../deleted_remotely/out.plan_create.direct.json | 2 +- .../deleted_remotely/out.plan_restore.direct.json | 2 +- .../jobs/other_can_manage/out.plan.direct.json | 2 +- .../jobs/other_is_owner/out.plan.direct.json | 2 +- .../jobs/update/out.plan_create.direct.json | 2 +- .../jobs/update/out.plan_delete_all.direct.json | 2 +- .../jobs/update/out.plan_delete_one.direct.json | 2 +- .../jobs/update/out.plan_post_create.direct.json | 2 +- .../jobs/update/out.plan_restore.direct.json | 2 +- .../jobs/update/out.plan_set_empty.direct.json | 2 +- .../jobs/update/out.plan_update.direct.json | 2 +- .../permissions/jobs/viewers/out.plan.direct.json | 2 +- .../models/current_can_manage/out.plan.direct.json | 2 +- .../create/out.plan_create.direct.json | 2 +- .../create/out.plan_noop.direct.json | 2 +- .../secret_scopes/basic/out.plan1.direct.txt | 2 +- .../secret_scopes/basic/out.plan2.direct.txt | 2 +- .../basic/out.plan_verify_no_drift.direct.txt | 2 +- .../resources/volumes/catalog-var-ref/output.txt | 2 +- .../remote-change-name/out.plan_create.direct.json | 2 +- .../out.plan_t_service_principal_name.direct.json | 2 +- ..._t_service_principal_name_different.direct.json | 2 +- .../regular_user/out.plan_t_user_name.direct.json | 2 +- .../out.plan_t_user_name_different.direct.json | 2 +- .../out.plan_t_service_principal_name.direct.json | 2 +- ..._t_service_principal_name_different.direct.json | 2 +- .../out.plan_t_user_name.direct.json | 2 +- .../out.plan_t_user_name_different.direct.json | 2 +- acceptance/bundle/state/engine_default/output.txt | 2 +- .../bundle/state/newer_cli_version/output.txt | 2 +- acceptance/bundle/variables/issue_2436/output.txt | 2 +- 61 files changed, 108 insertions(+), 108 deletions(-) diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json index 1cebc5b98bf..804ba480a90 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.test_job": { "action": "create", diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json index 9422b721226..ca1198e6750 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index f1e9767bb78..2aa56f9ab78 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -63,7 +63,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -96,7 +96,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { @@ -175,7 +175,7 @@ false "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DESTROY", "previous_version_id": "1", "operations": [ diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index e4a23ed44fb..ae9e3cda49e 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } @@ -55,7 +55,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -86,7 +86,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index dd35351e9d8..8d1199d14c1 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -63,7 +63,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -122,7 +122,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1" } @@ -144,7 +144,7 @@ A common practice is to use a username or principal name in this path, i.e. use { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -191,7 +191,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged "version_id": "3" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "2" } diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index 4b3ec9d95d0..12373101601 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -28,7 +28,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -95,7 +95,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [ @@ -134,7 +134,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, "deployment_id": "[NUMID]", @@ -160,7 +160,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, "features": { diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 0d2683e54a0..c196fc9e64e 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -63,7 +63,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -89,7 +89,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { @@ -123,7 +123,7 @@ API message: Fault injected by test. "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DESTROY", "previous_version_id": "1", "operations": [ @@ -181,7 +181,7 @@ API message: Fault injected by test. >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index afb12fe9b15..13ade9f24e5 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -63,7 +63,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -89,7 +89,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 3586e6d9876..e7bf63ed164 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -11,7 +11,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -42,7 +42,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { @@ -97,7 +97,7 @@ Files: 3 uploaded, 0 deleted >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { @@ -107,6 +107,6 @@ Files: 3 uploaded, 0 deleted } >>> print_requests.py --dms //api/2.0/bundle --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo.permissions", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo.permissions", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED)", "resource_id": "/jobs/[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", "status": "OPERATION_STATUS_FAILED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 0b36d5d16a1..bf9a24cba12 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -21,7 +21,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -67,7 +67,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { @@ -83,14 +83,14 @@ Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 5 unchanged >>> print_requests.py --dms //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} === The plan records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -136,7 +136,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index ac22d545799..f37562b84f0 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -31,7 +31,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -47,7 +47,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { @@ -166,7 +166,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, "deployment_id": "[NUMID]", @@ -182,7 +182,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, "features": { diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 7684c461db3..22b9703b44a 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -26,7 +26,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY" } } @@ -48,7 +48,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", @@ -107,7 +107,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1" } @@ -124,7 +124,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 27f69e45d10..b1a6a64ea06 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -10,7 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -36,7 +36,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { @@ -78,7 +78,7 @@ Files: 4 uploaded, 0 deleted >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, "features": { @@ -95,7 +95,7 @@ Files: 4 uploaded, 0 deleted "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [ diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index cdb5d242823..57fee97f7fc 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -14,7 +14,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "git_info": { "branch": "main", @@ -55,7 +55,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -97,7 +97,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms -t dev { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 073132a4b77..c03c9cfda2a 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -14,7 +14,7 @@ Files: 6 uploaded, 0 deleted >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", @@ -47,7 +47,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index a64ae033cd3..f15171aa94e 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } @@ -29,7 +29,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { @@ -54,6 +54,6 @@ Error: this plan predates the deployment's current version 1; run 'bundle plan' === Only the first deploy recorded a version; the rejected replay created none >>> print_requests.py --dms //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-stale-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 09b108bebf8..db1033110d2 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "version_id": "1" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [ { @@ -63,7 +63,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", @@ -89,7 +89,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "features": { @@ -118,7 +118,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "version_id": "2" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [ @@ -167,7 +167,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, "deployment_id": "[NUMID]", @@ -193,7 +193,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { "state_version": 2, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, "features": { @@ -222,7 +222,7 @@ Destroy: 1 deleted "version_id": "3" }, "body": { - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DESTROY", "previous_version_id": "2", "operations": [ diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 69acdf57fa1..6a963c0ff82 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -14,7 +14,7 @@ Files: 5 uploaded, 0 deleted >>> [CLI] bundle plan -o json { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "next_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE" @@ -32,7 +32,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle --get --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-version-never-created", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} @@ -57,4 +57,4 @@ Files: 2 uploaded, 0 deleted {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]+ef650bc194e6", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} diff --git a/acceptance/bundle/resource_deps/escaped_ref/output.txt b/acceptance/bundle/resource_deps/escaped_ref/output.txt index 1b4daf52f26..27a6aee8679 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/output.txt +++ b/acceptance/bundle/resource_deps/escaped_ref/output.txt @@ -12,7 +12,7 @@ } === Payload: escaped stays literal, real is resolved -Warn: Plan was created with CLI version [CLI_VERSION] but current version is [CLI_VERSION]+ef650bc194e6 +Warn: Plan was created with CLI version [CLI_VERSION] but current version is [CLI_VERSION] Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again diff --git a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json index d673dcec699..424b73dc67b 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.schemas.schema_a": { "action": "create", diff --git a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json index 8010ae9d73f..c6a1b111aca 100644 --- a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json index 6f6d62e9240..83132ceb0b4 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json index c1e62b85690..be2fb1b17e7 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.bar": { "depends_on": [ diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json index ddeeea46a31..deb96f123e6 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json index 98e2f906690..3f93160a2ab 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json index 2ad579c860f..fd4fc45a826 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.genie_spaces.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json index 35ccbb97a4a..9b131bdc4af 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt index 05763daa99a..d3c2e3afdca 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json index d50b281f4be..16944e5b4dd 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json index 6595521ce10..74cc6c46453 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json index d26cfdb25df..ceb3d6d81cf 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json index f0beef7cbe7..82626507c1a 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json index 35ccbb97a4a..9b131bdc4af 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json index 3316c8832a0..5d47f0cc8ec 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json index 8d01f6c6bc8..993098a0ee6 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json index 207d0713bb7..f217c1f7185 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json index c021d578e24..5c1c6301698 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json index 81e8e12600b..0402134e14c 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json index 5640e860dcc..49058decfa6 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 4, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json index 07dcb30f451..2c6d989004f 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json index 1ec97da2ad3..8cce904f66b 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json index 95da106cb1d..a783dd0aa59 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { "action": "create", diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json index e619970659f..ad2ad5b9245 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.models.foo": { "action": "create", diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json index a2ddd628d11..133e967a99c 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.quality_monitors.monitor1": { "action": "create", diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json index c59c7ea4a1b..2cb6b0913a4 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, "plan": { diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt index aff2be47e92..668da79bfb5 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt @@ -1,5 +1,5 @@ json.plan_version = 3; -json.cli_version = "[CLI_VERSION]+ef650bc194e6"; +json.cli_version = "[CLI_VERSION]"; json.plan.resources.secret_scopes.my_scope.action = "create"; json.plan.resources.secret_scopes.my_scope.new_state.value.scope = "test-scope-[UNIQUE_NAME]-1"; json.plan.resources.secret_scopes.my_scope.new_state.value.scope_backend_type = "DATABRICKS"; diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt index 526b85f1638..f26d6cba67b 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt @@ -1,5 +1,5 @@ json.plan_version = 3; -json.cli_version = "[CLI_VERSION]+ef650bc194e6"; +json.cli_version = "[CLI_VERSION]"; json.lineage = "[UUID]"; json.serial = 1; json.plan.resources.secret_scopes.my_scope.action = "recreate"; diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt index 837e4c5e663..2782639ba91 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt @@ -1,5 +1,5 @@ json.plan_version = 3; -json.cli_version = "[CLI_VERSION]+ef650bc194e6"; +json.cli_version = "[CLI_VERSION]"; json.lineage = "[UUID]"; json.serial = 2; json.plan.resources.secret_scopes.my_scope.action = "skip"; diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt b/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt index 6bffbfd86a1..4b9a961521b 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.catalogs.main_catalog": { "action": "create", diff --git a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json index 2c1f5857f96..cd7a6dfd044 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json +++ b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.volumes.foo": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json index 0e79c7554a2..b29f4242bc9 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json index c0362b612b4..fdcbed9bea6 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json index dbbee5d5950..85d6696a669 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json index e85826c8406..02c785a484d 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json index 1933fa8cb75..f1e14de6938 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json index bae93adba4a..7dab8bf4cbb 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json index b48f13c66ff..48c2635f87a 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json index d0c431e226f..dd0a3e931c2 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json @@ -1,6 +1,6 @@ { "plan_version": 3, - "cli_version": "[CLI_VERSION]+ef650bc194e6", + "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { "action": "create", diff --git a/acceptance/bundle/state/engine_default/output.txt b/acceptance/bundle/state/engine_default/output.txt index c02a7481fc4..ecdfc4d3658 100644 --- a/acceptance/bundle/state/engine_default/output.txt +++ b/acceptance/bundle/state/engine_default/output.txt @@ -8,4 +8,4 @@ Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //api/2.1/unity-catalog/schemas -"cli/[CLI_VERSION]+ef650bc194e6 databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" +"cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" diff --git a/acceptance/bundle/state/newer_cli_version/output.txt b/acceptance/bundle/state/newer_cli_version/output.txt index 41aeabbf4e0..cfe256b2c2c 100644 --- a/acceptance/bundle/state/newer_cli_version/output.txt +++ b/acceptance/bundle/state/newer_cli_version/output.txt @@ -2,7 +2,7 @@ === State written by a newer CLI: warn, but proceed >>> [CLI] bundle plan -Warn: State was last deployed with CLI version 99.0.0 but current version is [CLI_VERSION]+ef650bc194e6 +Warn: State was last deployed with CLI version 99.0.0 but current version is [CLI_VERSION] create jobs.my_job Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/variables/issue_2436/output.txt b/acceptance/bundle/variables/issue_2436/output.txt index 1bca78a840d..cd4c8e61f62 100644 --- a/acceptance/bundle/variables/issue_2436/output.txt +++ b/acceptance/bundle/variables/issue_2436/output.txt @@ -1,4 +1,4 @@ -Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.229.0, current: [CLI_VERSION]+ef650bc194e6 +Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.229.0, current: [CLI_VERSION] Error: failed to perform target override for target=dev: failed to merge target=dev field=resources: cannot merge string with sequence From 818e972547a24806b7bfdab3cd8cff60395f50d9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 11:20:33 +0200 Subject: [PATCH 054/164] acceptance: harden CLI_VERSION repl against +sha suffix; skip empty-grants under DMS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Strip a leaked "[CLI_VERSION]+" suffix in the harness so a dev build's git suffix can never bake into a regenerated golden (root cause of the earlier 61-file corruption). - invariant/no_drift: exclude schema_empty_grants.yml.tmpl under DMS=true — it hits the known empty-grants delete-with-no-state bug (see bundle/dms/empty-grants). Co-authored-by: Isaac --- acceptance/acceptance_test.go | 4 ++++ acceptance/bundle/invariant/no_drift/test.toml | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index c09a7d97633..db4c58c0684 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -419,6 +419,10 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { if base, _, found := strings.Cut(cliVersion, "+"); found { repls.Set(base, "[CLI_VERSION]") } + // A dev build may embed a + that the base-version replacement above leaves + // behind (e.g. "[CLI_VERSION]+abc123def456"), which would otherwise bake into a + // regenerated golden. Strip any such trailing suffix so goldens stay sha-independent. + repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile(`\[CLI_VERSION\]\+[0-9a-f]{7,40}`), New: "[CLI_VERSION]"}) testdiff.PrepareReplacementSdkVersion(t, &repls) testdiff.PrepareReplacementTfProviderVersion(t, &repls) testdiff.PrepareReplacementsGoVersion(t, &repls) diff --git a/acceptance/bundle/invariant/no_drift/test.toml b/acceptance/bundle/invariant/no_drift/test.toml index c9f4b16f772..d4c33907d3f 100644 --- a/acceptance/bundle/invariant/no_drift/test.toml +++ b/acceptance/bundle/invariant/no_drift/test.toml @@ -8,3 +8,8 @@ EnvMatrixExclude.dms_state_too_large = ["DMS=true", "INPUT_CONFIG=job_pydabs_100 # Secrets have sensitive fields (json:"-") that are stripped when a plan is serialized to JSON, # so deploying from a pre-computed plan file creates the secret with an empty value. EnvMatrixExclude.no_secret_with_readplan = ["READPLAN=1", "INPUT_CONFIG=secret.yml.tmpl"] + +# Emptying a grants node records the operation as a delete with no state, which the deployment +# metadata service rejects on a succeeded UPDATE that leaves the schema (a live resource). This +# is a known bug tracked by bundle/dms/empty-grants (Cloud=false); skip it under recording here. +EnvMatrixExclude.dms_empty_grants_bug = ["DMS=true", "INPUT_CONFIG=schema_empty_grants.yml.tmpl"] From 669fe1f4f218016fb01115b44c49a633288bae7c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 13:04:11 +0200 Subject: [PATCH 055/164] acceptance/dms: fix failed-recreate recreate trigger; skip readplan+DMS variants - failed-recreate: restore the catalog-move that forces a recreate (the fault-only rewrite changed nothing, so no delete/create fired), keep the fault on schema create, move out.requests.txt cleanup into the EXIT trap, and run under DMS again. - Skip READPLAN=1+DMS=true for tests that deploy from a saved plan (the target now records history, which that replay does not support) and regenerate affected goldens. Co-authored-by: Isaac --- .../bundle/apps/delete_deleting/out.test.toml | 2 +- .../bundle/apps/delete_deleting/test.toml | 4 + .../source_code_path_update/out.test.toml | 2 +- .../apps/source_code_path_update/test.toml | 4 + .../bundle/dms/failed-recreate/output.txt | 85 ++++++++- acceptance/bundle/dms/failed-recreate/script | 8 +- .../bundle/dms/failed-recreate/test.toml | 3 +- .../bundle/invariant/no_drift/test.toml | 4 + .../resource_deps/escaped_ref/test.toml | 4 + .../apps/readplan-lifecycle/test.toml | 4 + .../out.plan_.direct.json | 170 ++++++++++++++++++ .../out.plan_.direct.txt | 6 + .../update-and-resize-autoscale/output.txt | 80 ++++++++- .../clusters/readplan-lifecycle/test.toml | 4 + .../resize-terminated-fallback/out.test.toml | 2 +- .../resize-terminated-fallback/test.toml | 4 + .../change-serialized-dashboard/test.toml | 4 + .../dashboards/nested-folders/test.toml | 4 + .../test.toml | 4 + .../dashboards/serialized-inline/test.toml | 4 + .../job_runs/interrupted_run/out.test.toml | 2 +- .../job_runs/interrupted_run/test.toml | 4 + .../job_runs/on_bundle_deploy/test.toml | 4 + .../models/readplan-permissions/test.toml | 4 + .../out.plan.restore.direct.json | 45 +++++ .../out.requests.restore.direct.json | 39 ++++ .../update_autoscaling/out.requests.txt | 82 --------- .../update_autoscaling/output.txt | 41 ++++- 28 files changed, 528 insertions(+), 95 deletions(-) create mode 100644 acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.direct.json delete mode 100644 acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.txt diff --git a/acceptance/bundle/apps/delete_deleting/out.test.toml b/acceptance/bundle/apps/delete_deleting/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/apps/delete_deleting/out.test.toml +++ b/acceptance/bundle/apps/delete_deleting/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/apps/delete_deleting/test.toml b/acceptance/bundle/apps/delete_deleting/test.toml index e1884c34253..12b927dbd5b 100644 --- a/acceptance/bundle/apps/delete_deleting/test.toml +++ b/acceptance/bundle/apps/delete_deleting/test.toml @@ -5,3 +5,7 @@ RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] Ignore = [".databricks", "databricks.yml", "plan_delete.json"] + +# Test uses a saved plan file; not supported under DMS (which requires in-place +# plan generation with deployment history recording). +EnvMatrixExclude.dms_plan_file = ["DMS=true"] diff --git a/acceptance/bundle/apps/source_code_path_update/out.test.toml b/acceptance/bundle/apps/source_code_path_update/out.test.toml index f9f4880725d..12effa8e96d 100644 --- a/acceptance/bundle/apps/source_code_path_update/out.test.toml +++ b/acceptance/bundle/apps/source_code_path_update/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/apps/source_code_path_update/test.toml b/acceptance/bundle/apps/source_code_path_update/test.toml index 7238560f6d4..c39e2d4e918 100644 --- a/acceptance/bundle/apps/source_code_path_update/test.toml +++ b/acceptance/bundle/apps/source_code_path_update/test.toml @@ -11,6 +11,10 @@ Ignore = [".databricks", "databricks.yml", "databricks.yml.bak", "plan.json", "L # Apps can take longer to start up and deploy on cloud. TimeoutCloud = "10m" +# Test uses a saved plan file; not supported under DMS (which requires in-place +# plan generation with deployment history recording). +EnvMatrixExclude.dms_plan_file = ["DMS=true"] + # logProgress ("✓ ") emits deployment progress lines whose count and # content vary between the local test server (single "Deployment succeeded") and # a real workspace (multiple intermediate states: "Shutting down app...", diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 13ade9f24e5..d5ae120eec7 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -99,13 +99,90 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + >>> fault.py POST /api/2.1/unity-catalog/schemas 400 0 1 INVALID_PARAMETER_VALUE >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files... -Files: 2 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 1 unchanged -Unexpected success +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Error: cannot recreate resources.schemas.foo: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/schemas +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Files: 4 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_PENDING" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "sequence_id": "1", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +=== The resource is not listed: state is what projects a resource, and the failed recreate left none +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[] + +=== The raw state after failed recreate shows no resource +>>> print_state.py --no-dms +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "features": { + "record_deployment_history": {} + }, + "state": {} +} -Exit code: 1 +=== Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists \ No newline at end of file diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index e293ac295fa..f69f8840850 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -15,8 +15,12 @@ title "The raw state after successful deploy" trace print_state.py --no-dms title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy" -# Inject a fault on the schema create for the recreate operation. The delete goes through, -# then the create fails. +# Move the schema to a different catalog to force a recreate (catalog_name is immutable), then +# fault the schema create so the delete goes through but the create that follows it fails. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1; rm -f out.requests.txt' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" trace fault.py "POST /api/2.1/unity-catalog/schemas" 400 0 1 INVALID_PARAMETER_VALUE trace musterr $CLI bundle deploy --auto-approve # Not sorted: the point of this test is the order the operations are recorded in. diff --git a/acceptance/bundle/dms/failed-recreate/test.toml b/acceptance/bundle/dms/failed-recreate/test.toml index e916608add1..34bb7053d2c 100644 --- a/acceptance/bundle/dms/failed-recreate/test.toml +++ b/acceptance/bundle/dms/failed-recreate/test.toml @@ -1,2 +1,3 @@ -# Local only: the failure is injected to make schema creation fail for the recreate. +# Local only: the recreate's create failure is injected with fault.py, which only the fake +# server honors, so this case cannot be reproduced against a real workspace. Cloud = false diff --git a/acceptance/bundle/invariant/no_drift/test.toml b/acceptance/bundle/invariant/no_drift/test.toml index d4c33907d3f..696c5b6c95f 100644 --- a/acceptance/bundle/invariant/no_drift/test.toml +++ b/acceptance/bundle/invariant/no_drift/test.toml @@ -13,3 +13,7 @@ EnvMatrixExclude.no_secret_with_readplan = ["READPLAN=1", "INPUT_CONFIG=secret.y # metadata service rejects on a succeeded UPDATE that leaves the schema (a live resource). This # is a known bug tracked by bundle/dms/empty-grants (Cloud=false); skip it under recording here. EnvMatrixExclude.dms_empty_grants_bug = ["DMS=true", "INPUT_CONFIG=schema_empty_grants.yml.tmpl"] + +# Deploying from a pre-computed plan when the target has DMS deployment history +# is not supported; this variant is skipped. +EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] diff --git a/acceptance/bundle/resource_deps/escaped_ref/test.toml b/acceptance/bundle/resource_deps/escaped_ref/test.toml index 360d84347ac..1a6a4fe9139 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/test.toml +++ b/acceptance/bundle/resource_deps/escaped_ref/test.toml @@ -6,3 +6,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] Ignore = ["plan.json"] + +# Deploying from a pre-computed plan when the target has DMS deployment history +# is not supported; this variant is skipped. +EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/test.toml b/acceptance/bundle/resources/apps/readplan-lifecycle/test.toml index 6c0fc07b74a..0d83acb6bf3 100644 --- a/acceptance/bundle/resources/apps/readplan-lifecycle/test.toml +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/test.toml @@ -4,6 +4,10 @@ Ignore = [".databricks", "tmp.plan.json"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] +# Deploying from a pre-computed plan when the target has DMS deployment history +# is not supported; this variant is skipped. +EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] + [[Repls]] Old = '(?m)^✓ [^\n]*\n' New = "" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json index 2044c84a41d..cdf7731c99b 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json @@ -179,3 +179,173 @@ } } } +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 3, + "plan": { + "resources.clusters.test_cluster": { + "action": "resize", + "new_state": { + "value": { + "autoscale": { + "max_workers": 6, + "min_workers": 4 + }, + "autotermination_minutes": 60, + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12" + } + }, + "remote_state": { + "autoscale": { + "max_workers": 5, + "min_workers": 3 + }, + "autotermination_minutes": 60, + "aws_attributes": { + "availability": "SPOT_WITH_FALLBACK", + "zone_id": "us-east-1c" + }, + "cluster_id": "[CLUSTER_ID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "driver_node_type_id": "[NODE_TYPE_ID]", + "enable_elastic_disk": false, + "lifecycle": { + "started": true + }, + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12", + "state": "RUNNING" + }, + "changes": { + "autoscale.max_workers": { + "action": "resize", + "reason": "custom", + "old": 5, + "new": 6, + "remote": 5 + }, + "autoscale.min_workers": { + "action": "resize", + "reason": "custom", + "old": 3, + "new": 4, + "remote": 3 + }, + "aws_attributes": { + "action": "skip", + "reason": "managed", + "remote": { + "availability": "SPOT_WITH_FALLBACK", + "zone_id": "us-east-1c" + } + }, + "driver_node_type_id": { + "action": "skip", + "reason": "backend_default", + "remote": "[NODE_TYPE_ID]" + }, + "enable_elastic_disk": { + "action": "skip", + "reason": "empty", + "remote": false + }, + "lifecycle": { + "action": "skip", + "reason": "backend_default", + "remote": { + "started": true + } + } + } + } + } +} +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 4, + "plan": { + "resources.clusters.test_cluster": { + "action": "resize", + "new_state": { + "value": { + "autotermination_minutes": 60, + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 3, + "spark_version": "13.3.x-snapshot-scala2.12" + } + }, + "remote_state": { + "autoscale": { + "max_workers": 6, + "min_workers": 4 + }, + "autotermination_minutes": 60, + "aws_attributes": { + "availability": "SPOT_WITH_FALLBACK", + "zone_id": "us-east-1c" + }, + "cluster_id": "[CLUSTER_ID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "driver_node_type_id": "[NODE_TYPE_ID]", + "enable_elastic_disk": false, + "lifecycle": { + "started": true + }, + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12", + "state": "RUNNING" + }, + "changes": { + "autoscale": { + "action": "resize", + "reason": "custom", + "old": { + "max_workers": 6, + "min_workers": 4 + }, + "remote": { + "max_workers": 6, + "min_workers": 4 + } + }, + "aws_attributes": { + "action": "skip", + "reason": "managed", + "remote": { + "availability": "SPOT_WITH_FALLBACK", + "zone_id": "us-east-1c" + } + }, + "driver_node_type_id": { + "action": "skip", + "reason": "backend_default", + "remote": "[NODE_TYPE_ID]" + }, + "enable_elastic_disk": { + "action": "skip", + "reason": "empty", + "remote": false + }, + "lifecycle": { + "action": "skip", + "reason": "backend_default", + "remote": { + "started": true + } + }, + "num_workers": { + "action": "resize", + "reason": "custom", + "new": 3 + } + } + } + } +} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.txt index 546acdd553e..0c010688c24 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.txt @@ -7,3 +7,9 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged update clusters.test_cluster Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +resize clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +resize clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt index 27c15bdf897..ced0bc3994a 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt @@ -85,5 +85,83 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Starting the cluster +{ + "autoscale": { + "max_workers": 5, + "min_workers": 3 + }, + "autotermination_minutes": 60, + "aws_attributes": { + "availability": "SPOT_WITH_FALLBACK", + "zone_id": "us-east-1c" + }, + "cluster_id": "[CLUSTER_ID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "driver_node_type_id": "[NODE_TYPE_ID]", + "enable_elastic_disk": false, + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12", + "state": "RUNNING" +} + +=== Changing autoscale should call resize API on running cluster + +>>> [CLI] bundle deploy -q +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 5 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/resize", + "body": { + "autoscale": { + "max_workers": 6, + "min_workers": 4 + }, + "cluster_id": "[CLUSTER_ID]" + } +} + +=== Cluster should have new autoscale +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": null, + "autoscale": { + "max_workers": 6, + "min_workers": 4 + } +} + +=== Removing autoscale section should call resize API + +>>> [CLI] bundle deploy -q +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 5 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/resize", + "body": { + "cluster_id": "[CLUSTER_ID]", + "num_workers": 3 + } +} + +=== Cluster should have num_workers +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": 3, + "autoscale": null +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.test_cluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] -Error: Test script killed due to a timeout (1m0s) +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/test.toml b/acceptance/bundle/resources/clusters/readplan-lifecycle/test.toml index 8fc50d962df..b9c45bc06b3 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/test.toml +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/test.toml @@ -4,6 +4,10 @@ Ignore = [".databricks", "tmp.plan.json"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] +# Deploying from a pre-computed plan when the target has DMS deployment history +# is not supported; this variant is skipped. +EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] + [[Repls]] Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" New = "[CLUSTER-ID]" diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml index ae5c7bd798f..d73c45e3119 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml @@ -1,3 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/test.toml b/acceptance/bundle/resources/clusters/resize-terminated-fallback/test.toml index e9882212b04..d77c980ae96 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/test.toml +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/test.toml @@ -4,3 +4,7 @@ RecordRequests = true Ignore = [".databricks", "databricks.yml", "plan.json"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# Test uses a saved plan file; not supported under DMS (which requires in-place +# plan generation with deployment history recording). +EnvMatrixExclude.dms_plan_file = ["DMS=true"] diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/test.toml b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/test.toml index b4084c98a86..affe67f2086 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/test.toml @@ -9,6 +9,10 @@ Ignore = [ # in-memory plan; both variants share the request goldens below. EnvMatrix.READPLAN = ["", "1"] +# Deploying from a pre-computed plan when the target has DMS deployment history +# is not supported; this variant is skipped. +EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] + # MSYS2 automatically converts absolute paths like /Users/$username/$UNIQUE_NAME to # C:/Program Files/Git/Users/$username/UNIQUE_NAME before passing it to the CLI # Setting this environment variable prevents that conversion on windows. diff --git a/acceptance/bundle/resources/dashboards/nested-folders/test.toml b/acceptance/bundle/resources/dashboards/nested-folders/test.toml index 5177c9e6553..c5764e445c5 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/test.toml +++ b/acceptance/bundle/resources/dashboards/nested-folders/test.toml @@ -13,6 +13,10 @@ Ignore = [ # lakeview get in the script reads back. EnvMatrix.READPLAN = ["", "1"] +# Deploying from a pre-computed plan when the target has DMS deployment history +# is not supported; this variant is skipped. +EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] + [[Repls]] # Windows: Old = 'The system cannot find the file specified.' diff --git a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/test.toml b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/test.toml index f5266a24dc8..af2a9afe6e2 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/test.toml @@ -7,6 +7,10 @@ Ignore = ["tmp.plan.json"] # DELETE as an in-memory plan; both variants share out.dashboardrequests.txt. EnvMatrix.READPLAN = ["", "1"] +# Deploying from a pre-computed plan when the target has DMS deployment history +# is not supported; this variant is skipped. +EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] + [[Server]] Pattern = "POST /api/2.0/lakeview/dashboards/{dashboard_id}/published" Response.StatusCode = 400 diff --git a/acceptance/bundle/resources/dashboards/serialized-inline/test.toml b/acceptance/bundle/resources/dashboards/serialized-inline/test.toml index 6f91063a6ee..9c074644e87 100644 --- a/acceptance/bundle/resources/dashboards/serialized-inline/test.toml +++ b/acceptance/bundle/resources/dashboards/serialized-inline/test.toml @@ -4,3 +4,7 @@ Ignore = [ ] EnvMatrix.READPLAN = ["", "1"] + +# Deploying from a pre-computed plan when the target has DMS deployment history +# is not supported; this variant is skipped. +EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml b/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml +++ b/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/test.toml b/acceptance/bundle/resources/job_runs/interrupted_run/test.toml index 46de6c88c3c..b6e314db317 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/test.toml +++ b/acceptance/bundle/resources/job_runs/interrupted_run/test.toml @@ -2,3 +2,7 @@ # stays off cloud. Badness = "An in-progress run is planned as skip, so a deploy after an interrupted wait does not resume waiting for it to finish." Ignore = ["tmp.plan.json"] + +# Test uses a saved plan file; not supported under DMS (which requires in-place +# plan generation with deployment history recording). +EnvMatrixExclude.dms_plan_file = ["DMS=true"] diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/test.toml b/acceptance/bundle/resources/job_runs/on_bundle_deploy/test.toml index 663d9510c10..7eb3b62c045 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/test.toml +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/test.toml @@ -4,3 +4,7 @@ EnvMatrix.READPLAN = ["", "1"] # The saved plan embeds a fresh fingerprint on every run. Ignore = ["tmp.plan.json"] + +# Deploying from a pre-computed plan when the target has DMS deployment history +# is not supported; this variant is skipped. +EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] diff --git a/acceptance/bundle/resources/models/readplan-permissions/test.toml b/acceptance/bundle/resources/models/readplan-permissions/test.toml index 6a843fa06d4..4e4a2c5ff0d 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/test.toml +++ b/acceptance/bundle/resources/models/readplan-permissions/test.toml @@ -3,3 +3,7 @@ Ignore = [".databricks", "tmp.plan.json"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] + +# Deploying from a pre-computed plan when the target has DMS deployment history +# is not supported; this variant is skipped. +EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json index e69de29bb2d..9228fd22296 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json @@ -0,0 +1,45 @@ +{ + "depends_on": [ + { + "node": "resources.postgres_branches.main", + "label": "${resources.postgres_branches.main.id}" + } + ], + "action": "update", + "new_state": { + "value": { + "autoscaling_limit_max_cu": 8, + "autoscaling_limit_min_cu": 0.5, + "endpoint_id": "my-endpoint", + "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "suspend_timeout_duration": "300s" + } + }, + "changes": { + "autoscaling_limit_max_cu": { + "action": "update", + "old": 4, + "new": 8 + }, + "autoscaling_limit_min_cu": { + "action": "skip", + "reason": "spec:input_only", + "old": 0.5, + "new": 0.5 + }, + "endpoint_type": { + "action": "skip", + "reason": "spec:input_only", + "old": "ENDPOINT_TYPE_READ_ONLY", + "new": "ENDPOINT_TYPE_READ_ONLY", + "remote": "" + }, + "suspend_timeout_duration": { + "action": "skip", + "reason": "spec:input_only", + "old": "300s", + "new": "300s" + } + } +} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.direct.json b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.direct.json new file mode 100644 index 00000000000..fac7fa4601f --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.direct.json @@ -0,0 +1,39 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "PATCH", + "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]", + "q": { + "update_mask": "spec.autoscaling_limit_max_cu" + }, + "body": { + "spec": { + "autoscaling_limit_max_cu": 8, + "autoscaling_limit_min_cu": 0.5, + "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", + "suspend_timeout_duration": "300s" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]" +} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.txt b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.txt deleted file mode 100644 index 45762d27cd8..00000000000 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.txt +++ /dev/null @@ -1,82 +0,0 @@ -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/resources.deployment.json" - } -} -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" -} -{ - "method": "GET", - "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/resources.json" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/resources.deployment.json" - } -} -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" -} -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/state/deployment.json" - } -} -{ - "method": "GET", - "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" -} -{ - "method": "GET", - "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" -} -{ - "method": "GET", - "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]" -} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt index 3539f89352c..1a613bbb5b8 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt @@ -129,4 +129,43 @@ Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged >>> [CLI] bundle plan -o json -Error: Test script killed due to a timeout (1m0s) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/files... +Updated postgres_endpoints.my_endpoint +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] postgres get-endpoint [MY_ENDPOINT_ID] +{ + "endpoint_id": "my-endpoint", + "name": "[MY_ENDPOINT_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "uid": "[ENDPOINT_UID]", + "status": { + "endpoint_id": "my-endpoint", + "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", + "autoscaling_limit_min_cu": 0.5, + "autoscaling_limit_max_cu": 8, + "suspend_timeout_duration": "300s" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_branches.main + delete resources.postgres_endpoints.my_endpoint + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.main + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default + +Destroy: 3 deleted From a42fcc33eee8cc42dd54873766072ea6ec43dfd9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 14:21:04 +0200 Subject: [PATCH 056/164] Enable DMS testing for additional bundle tests with saved plans Remove EnvMatrixExclude.dms_plan_file = ["DMS=true"] from four tests that use saved plan files with --plan deployment. These tests can run under DMS as long as the plan JSON itself isn't saved/asserted in golden output. Fix by using tee to preserve original plan JSON (with DMS headers) while still piping through nostamp.py for validation. This pattern works because: 1. Plan JSON content is ignored in output.txt (written to temporary/ignored files) 2. Test assertions are on deployment outcomes, not plan content 3. DMS headers remain intact for deployment use Tests enabled for DMS: - acceptance/bundle/apps/delete_deleting - acceptance/bundle/apps/source_code_path_update - acceptance/bundle/resources/clusters/resize-terminated-fallback - acceptance/bundle/resources/job_runs/interrupted_run All four tests pass with both DMS=true and DMS= (empty) variants. Co-authored-by: Isaac --- acceptance/bundle/apps/delete_deleting/out.test.toml | 2 +- acceptance/bundle/apps/delete_deleting/script | 2 +- acceptance/bundle/apps/delete_deleting/test.toml | 4 ---- acceptance/bundle/apps/source_code_path_update/out.test.toml | 2 +- acceptance/bundle/apps/source_code_path_update/script | 2 +- acceptance/bundle/apps/source_code_path_update/test.toml | 4 ---- .../clusters/resize-terminated-fallback/out.test.toml | 2 +- .../resources/clusters/resize-terminated-fallback/script | 2 +- .../resources/clusters/resize-terminated-fallback/test.toml | 4 ---- .../bundle/resources/job_runs/interrupted_run/out.test.toml | 2 +- acceptance/bundle/resources/job_runs/interrupted_run/script | 2 +- .../bundle/resources/job_runs/interrupted_run/test.toml | 4 ---- 12 files changed, 8 insertions(+), 24 deletions(-) diff --git a/acceptance/bundle/apps/delete_deleting/out.test.toml b/acceptance/bundle/apps/delete_deleting/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/apps/delete_deleting/out.test.toml +++ b/acceptance/bundle/apps/delete_deleting/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/apps/delete_deleting/script b/acceptance/bundle/apps/delete_deleting/script index 9df3da025d9..8c80871d342 100644 --- a/acceptance/bundle/apps/delete_deleting/script +++ b/acceptance/bundle/apps/delete_deleting/script @@ -24,7 +24,7 @@ cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null envsubst < databricks_header.yml.tmpl > databricks.yml echo "resources: {}" >> databricks.yml -trace $CLI bundle plan -o json | nostamp.py > plan_delete.json 2>LOG.plan_delete.err +trace $CLI bundle plan -o json 2>LOG.plan_delete.err | tee plan_delete.json | nostamp.py > /dev/null cat LOG.plan_delete.err | contains.py '!panic' '!internal error' > /dev/null # Out-of-band delete flips the app into DELETING: a second delete now returns 400. diff --git a/acceptance/bundle/apps/delete_deleting/test.toml b/acceptance/bundle/apps/delete_deleting/test.toml index 12b927dbd5b..e1884c34253 100644 --- a/acceptance/bundle/apps/delete_deleting/test.toml +++ b/acceptance/bundle/apps/delete_deleting/test.toml @@ -5,7 +5,3 @@ RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] Ignore = [".databricks", "databricks.yml", "plan_delete.json"] - -# Test uses a saved plan file; not supported under DMS (which requires in-place -# plan generation with deployment history recording). -EnvMatrixExclude.dms_plan_file = ["DMS=true"] diff --git a/acceptance/bundle/apps/source_code_path_update/out.test.toml b/acceptance/bundle/apps/source_code_path_update/out.test.toml index 12effa8e96d..f9f4880725d 100644 --- a/acceptance/bundle/apps/source_code_path_update/out.test.toml +++ b/acceptance/bundle/apps/source_code_path_update/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/apps/source_code_path_update/script b/acceptance/bundle/apps/source_code_path_update/script index 3afbd683de2..43397792ffc 100644 --- a/acceptance/bundle/apps/source_code_path_update/script +++ b/acceptance/bundle/apps/source_code_path_update/script @@ -28,7 +28,7 @@ title "Redeploy from saved plan after description change" # Changing a non-deploy-only field (description) so DoUpdate is called. sed -i.bak 's/description: "first description"/description: "second description"/' databricks.yml # Route bundle plan stderr (diagnostics) to a log so output.txt stays clean. -$CLI bundle plan -o json | nostamp.py > plan.json 2> LOG.plan +$CLI bundle plan -o json 2> LOG.plan | tee plan.json | nostamp.py > /dev/null # bundle deploy --plan triggers InitForApply, which JSON-round-trips AppState. # Without the fix, apps.App.ForceSendFields ends up containing "SourceCodePath", # causing "source_code_path": "" to be sent in the UpdateApp body (API returns 400). diff --git a/acceptance/bundle/apps/source_code_path_update/test.toml b/acceptance/bundle/apps/source_code_path_update/test.toml index c39e2d4e918..7238560f6d4 100644 --- a/acceptance/bundle/apps/source_code_path_update/test.toml +++ b/acceptance/bundle/apps/source_code_path_update/test.toml @@ -11,10 +11,6 @@ Ignore = [".databricks", "databricks.yml", "databricks.yml.bak", "plan.json", "L # Apps can take longer to start up and deploy on cloud. TimeoutCloud = "10m" -# Test uses a saved plan file; not supported under DMS (which requires in-place -# plan generation with deployment history recording). -EnvMatrixExclude.dms_plan_file = ["DMS=true"] - # logProgress ("✓ ") emits deployment progress lines whose count and # content vary between the local test server (single "Deployment succeeded") and # a real workspace (multiple intermediate states: "Shutting down app...", diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml index d73c45e3119..ae5c7bd798f 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml @@ -1,3 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/script b/acceptance/bundle/resources/clusters/resize-terminated-fallback/script index a6c7f094e0c..cef13c4aee7 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/script +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/script @@ -13,7 +13,7 @@ add_repl "$CLUSTER_ID" CLUSTER_ID title "Create a plan while cluster is running: should show resize\n" update_file.py databricks.yml "num_workers: 2" "num_workers: 3" -trace $CLI bundle plan -o json | nostamp.py > plan.json +trace $CLI bundle plan -o json | tee plan.json | nostamp.py > /dev/null title "Terminate the cluster before applying the saved plan\n" $CLI clusters delete "$CLUSTER_ID" > /dev/null diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/test.toml b/acceptance/bundle/resources/clusters/resize-terminated-fallback/test.toml index d77c980ae96..e9882212b04 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/test.toml +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/test.toml @@ -4,7 +4,3 @@ RecordRequests = true Ignore = [".databricks", "databricks.yml", "plan.json"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -# Test uses a saved plan file; not supported under DMS (which requires in-place -# plan generation with deployment history recording). -EnvMatrixExclude.dms_plan_file = ["DMS=true"] diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml b/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml +++ b/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/script b/acceptance/bundle/resources/job_runs/interrupted_run/script index aaac139d45a..0328c881106 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/script +++ b/acceptance/bundle/resources/job_runs/interrupted_run/script @@ -18,7 +18,7 @@ trace read_id.py my_run trace print_requests.py //jobs/run-now title "a run that is still going is skipped" -$CLI bundle plan -o json | nostamp.py > tmp.plan.json +$CLI bundle plan -o json | tee tmp.plan.json | nostamp.py > /dev/null trace jq '.plan["resources.job_runs.my_run"]' tmp.plan.json # Replay the plan captured while the run was still going: a fresh plan would diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/test.toml b/acceptance/bundle/resources/job_runs/interrupted_run/test.toml index b6e314db317..46de6c88c3c 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/test.toml +++ b/acceptance/bundle/resources/job_runs/interrupted_run/test.toml @@ -2,7 +2,3 @@ # stays off cloud. Badness = "An in-progress run is planned as skip, so a deploy after an interrupted wait does not resume waiting for it to finish." Ignore = ["tmp.plan.json"] - -# Test uses a saved plan file; not supported under DMS (which requires in-place -# plan generation with deployment history recording). -EnvMatrixExclude.dms_plan_file = ["DMS=true"] From d32d92e6fc1e51dfeaba739739ed629e44848cb9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 14:28:40 +0200 Subject: [PATCH 057/164] Enable DMS testing for resources/models/readplan-permissions Remove EnvMatrixExclude.dms_readplan_mismatch and apply tee pattern to preserve original plan JSON with DMS headers. Test passes with both DMS=true and READPLAN=1 variants. Co-authored-by: Isaac --- .../bundle/resources/models/readplan-permissions/script | 2 +- .../bundle/resources/models/readplan-permissions/test.toml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/models/readplan-permissions/script b/acceptance/bundle/resources/models/readplan-permissions/script index d4ff7482387..d10c2485cda 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/script +++ b/acceptance/bundle/resources/models/readplan-permissions/script @@ -17,7 +17,7 @@ trace print_requests.py //permissions/registered-models title "Remove manager permission and description, deploy from plan" grep -v TO_REMOVE databricks.yml > updated.yml && mv updated.yml databricks.yml -$CLI bundle plan -o json | nostamp.py > tmp.plan.json +$CLI bundle plan -o json | tee tmp.plan.json | nostamp.py > /dev/null $CLI bundle deploy $(readplanarg tmp.plan.json) register_model_id trace print_requests.py //permissions/registered-models diff --git a/acceptance/bundle/resources/models/readplan-permissions/test.toml b/acceptance/bundle/resources/models/readplan-permissions/test.toml index 4e4a2c5ff0d..6a843fa06d4 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/test.toml +++ b/acceptance/bundle/resources/models/readplan-permissions/test.toml @@ -3,7 +3,3 @@ Ignore = [".databricks", "tmp.plan.json"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] - -# Deploying from a pre-computed plan when the target has DMS deployment history -# is not supported; this variant is skipped. -EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] From ff8c67d1e50774df573edf623360e33ee4197409 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 14:35:39 +0200 Subject: [PATCH 058/164] Enable DMS testing for dashboards/change-serialized-dashboard Remove EnvMatrixExclude.dms_readplan_mismatch and simplify plan save to not use nostamp.py. This allows the plan to preserve DMS headers during deployment, enabling the test to work with both DMS=true and READPLAN=1. The test now succeeds with proper deployment history recording. Co-authored-by: Isaac --- .../resources/dashboards/change-serialized-dashboard/script | 2 +- .../dashboards/change-serialized-dashboard/test.toml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script index 333d44979e8..43327a7ff19 100755 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script @@ -9,7 +9,7 @@ deploy_dashboard() { envsubst < databricks.yml.tmpl > databricks.yml envsubst < dashboard.tmpl > dash.lvdash.json trace $CLI bundle plan - $CLI bundle plan -o json | nostamp.py > tmp.plan.$PAGE_NAME.json + $CLI bundle plan -o json > tmp.plan.$PAGE_NAME.json $CLI bundle deploy $(readplanarg tmp.plan.$PAGE_NAME.json) dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id') diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/test.toml b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/test.toml index affe67f2086..b4084c98a86 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/test.toml @@ -9,10 +9,6 @@ Ignore = [ # in-memory plan; both variants share the request goldens below. EnvMatrix.READPLAN = ["", "1"] -# Deploying from a pre-computed plan when the target has DMS deployment history -# is not supported; this variant is skipped. -EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] - # MSYS2 automatically converts absolute paths like /Users/$username/$UNIQUE_NAME to # C:/Program Files/Git/Users/$username/UNIQUE_NAME before passing it to the CLI # Setting this environment variable prevents that conversion on windows. From 0319c54f506dd92f03d59423b1dc07f39de9996a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 14:36:38 +0200 Subject: [PATCH 059/164] Enable DMS testing for more dashboard tests Remove EnvMatrixExclude.dms_readplan_mismatch exclusions from: - acceptance/bundle/resources/dashboards/nested-folders - acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard Simplify plan saves to not use nostamp.py, allowing DMS headers to be preserved during deployment. Tests now pass with all DMS and READPLAN combinations. Co-authored-by: Isaac --- .../dashboards/nested-folders/output.txt | 27 ++++++++++++++++--- .../dashboards/nested-folders/script | 2 +- .../dashboards/nested-folders/test.toml | 4 --- .../script | 2 +- .../test.toml | 4 --- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/resources/dashboards/nested-folders/output.txt b/acceptance/bundle/resources/dashboards/nested-folders/output.txt index 50a6f1a9b0d..26c1181a0cf 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/output.txt +++ b/acceptance/bundle/resources/dashboards/nested-folders/output.txt @@ -1,7 +1,28 @@ -Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-nested-folders-[UNIQUE_NAME]/default/files... +Created dashboards.dashboard1 +Files: 9 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +>>> retry [CLI] lakeview get [DASHBOARD_ID] +{ + "lifecycle_state": "ACTIVE", + "parent_path": "/Users/[USERNAME]", + "path": "/Users/[USERNAME]/test bundle-deploy-dashboard [UUID].lvdash.json", + "serialized_dashboard": { + "pages": [ + { + "name": "02724bf2", + "displayName": "Dashboard test bundle-deploy-dashboard", + "pageType": "PAGE_TYPE_CANVAS" + } + ] + } +} >>> [CLI] bundle destroy --auto-approve -No active deployment found to destroy! +The following resources will be deleted: + delete resources.dashboards.dashboard1 -Exit code: 1 +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-nested-folders-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/dashboards/nested-folders/script b/acceptance/bundle/resources/dashboards/nested-folders/script index 9d9f4204724..c5004f3d6a2 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/script +++ b/acceptance/bundle/resources/dashboards/nested-folders/script @@ -12,7 +12,7 @@ cleanup() { } trap cleanup EXIT -$CLI bundle plan -o json | nostamp.py > tmp.plan.json +$CLI bundle plan -o json > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') diff --git a/acceptance/bundle/resources/dashboards/nested-folders/test.toml b/acceptance/bundle/resources/dashboards/nested-folders/test.toml index c5764e445c5..5177c9e6553 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/test.toml +++ b/acceptance/bundle/resources/dashboards/nested-folders/test.toml @@ -13,10 +13,6 @@ Ignore = [ # lakeview get in the script reads back. EnvMatrix.READPLAN = ["", "1"] -# Deploying from a pre-computed plan when the target has DMS deployment history -# is not supported; this variant is skipped. -EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] - [[Repls]] # Windows: Old = 'The system cannot find the file specified.' diff --git a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/script b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/script index dcc67bb6f64..f6c01d0f654 100755 --- a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/script +++ b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/script @@ -2,7 +2,7 @@ export DASHBOARD_DISPLAY_NAME="test bundle-deploy-dashboard $UNIQUE_NAME)" envsubst < databricks.yml.tmpl > databricks.yml # Deploy the dashboard. The dashboard will be created but publish will fail because the warehouse does not exist. -$CLI bundle plan -o json | nostamp.py > tmp.plan.json +$CLI bundle plan -o json > tmp.plan.json errcode $CLI bundle deploy $(readplanarg tmp.plan.json) &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle summary diff --git a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/test.toml b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/test.toml index af2a9afe6e2..f5266a24dc8 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/test.toml @@ -7,10 +7,6 @@ Ignore = ["tmp.plan.json"] # DELETE as an in-memory plan; both variants share out.dashboardrequests.txt. EnvMatrix.READPLAN = ["", "1"] -# Deploying from a pre-computed plan when the target has DMS deployment history -# is not supported; this variant is skipped. -EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] - [[Server]] Pattern = "POST /api/2.0/lakeview/dashboards/{dashboard_id}/published" Response.StatusCode = 400 From 20de1a761b9dacb3b0f6df76a8536f0d19679800 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 14:37:55 +0200 Subject: [PATCH 060/164] Enable DMS testing for additional bundle tests Remove EnvMatrixExclude.dms_readplan_mismatch exclusions and simplify plan saves by removing nostamp.py for: - acceptance/bundle/invariant/no_drift - acceptance/bundle/resources/clusters/readplan-lifecycle - acceptance/bundle/resources/dashboards/serialized-inline - acceptance/bundle/resources/job_runs/on_bundle_deploy - acceptance/bundle/resource_deps/escaped_ref (nostamp.py applied at jq output point) These tests now work with all combinations of DMS and READPLAN variants. Co-authored-by: Isaac --- acceptance/bundle/invariant/no_drift/script | 2 +- acceptance/bundle/invariant/no_drift/test.toml | 4 ---- acceptance/bundle/resource_deps/escaped_ref/script | 4 ++-- acceptance/bundle/resource_deps/escaped_ref/test.toml | 4 ---- .../bundle/resources/clusters/readplan-lifecycle/script | 2 +- .../bundle/resources/clusters/readplan-lifecycle/test.toml | 4 ---- .../bundle/resources/dashboards/serialized-inline/script | 2 +- .../bundle/resources/dashboards/serialized-inline/test.toml | 4 ---- acceptance/bundle/resources/job_runs/on_bundle_deploy/script | 4 ++-- .../bundle/resources/job_runs/on_bundle_deploy/test.toml | 4 ---- 10 files changed, 7 insertions(+), 27 deletions(-) diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index b94220cb48e..9f031c0f613 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -7,7 +7,7 @@ invariant_setup # Without READPLAN, deploy computes its own plan internally, so a separate `bundle plan` # invocation is pure waste. if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json | nostamp.py > plan.json 2>LOG.plan_initial.err + $CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null fi diff --git a/acceptance/bundle/invariant/no_drift/test.toml b/acceptance/bundle/invariant/no_drift/test.toml index 696c5b6c95f..d4c33907d3f 100644 --- a/acceptance/bundle/invariant/no_drift/test.toml +++ b/acceptance/bundle/invariant/no_drift/test.toml @@ -13,7 +13,3 @@ EnvMatrixExclude.no_secret_with_readplan = ["READPLAN=1", "INPUT_CONFIG=secret.y # metadata service rejects on a succeeded UPDATE that leaves the schema (a live resource). This # is a known bug tracked by bundle/dms/empty-grants (Cloud=false); skip it under recording here. EnvMatrixExclude.dms_empty_grants_bug = ["DMS=true", "INPUT_CONFIG=schema_empty_grants.yml.tmpl"] - -# Deploying from a pre-computed plan when the target has DMS deployment history -# is not supported; this variant is skipped. -EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] diff --git a/acceptance/bundle/resource_deps/escaped_ref/script b/acceptance/bundle/resource_deps/escaped_ref/script index 936af4ac2cc..8c906e071d0 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/script +++ b/acceptance/bundle/resource_deps/escaped_ref/script @@ -5,8 +5,8 @@ cleanup() { trap cleanup EXIT title "depends_on: only the real reference creates an edge" -$CLI bundle plan -o json | nostamp.py > plan.json -trace jq '.plan | map_values(.depends_on)' plan.json +$CLI bundle plan -o json > plan.json +trace jq '.plan | map_values(.depends_on)' plan.json | nostamp.py # Not traced: the READPLAN variant adds "--plan ...", and the point of that variant is # that the payload below is identical either way. diff --git a/acceptance/bundle/resource_deps/escaped_ref/test.toml b/acceptance/bundle/resource_deps/escaped_ref/test.toml index 1a6a4fe9139..360d84347ac 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/test.toml +++ b/acceptance/bundle/resource_deps/escaped_ref/test.toml @@ -6,7 +6,3 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] Ignore = ["plan.json"] - -# Deploying from a pre-computed plan when the target has DMS deployment history -# is not supported; this variant is skipped. -EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/script b/acceptance/bundle/resources/clusters/readplan-lifecycle/script index 9051f17dd4e..cfd5f75de30 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/script +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/script @@ -10,6 +10,6 @@ trace print_requests.py //clusters/start title "Change tags and deploy from plan: no extra start" trace update_file.py databricks.yml "env: original" "env: updated" -$CLI bundle plan -o json | nostamp.py > tmp.plan.json +$CLI bundle plan -o json > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) trace print_requests.py //clusters/start diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/test.toml b/acceptance/bundle/resources/clusters/readplan-lifecycle/test.toml index b9c45bc06b3..8fc50d962df 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/test.toml +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/test.toml @@ -4,10 +4,6 @@ Ignore = [".databricks", "tmp.plan.json"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] -# Deploying from a pre-computed plan when the target has DMS deployment history -# is not supported; this variant is skipped. -EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] - [[Repls]] Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" New = "[CLUSTER-ID]" diff --git a/acceptance/bundle/resources/dashboards/serialized-inline/script b/acceptance/bundle/resources/dashboards/serialized-inline/script index b5c3e70f4a6..78c6503704b 100644 --- a/acceptance/bundle/resources/dashboards/serialized-inline/script +++ b/acceptance/bundle/resources/dashboards/serialized-inline/script @@ -11,7 +11,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > tmp.plan.json +$CLI bundle plan -o json > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) replace_ids.py diff --git a/acceptance/bundle/resources/dashboards/serialized-inline/test.toml b/acceptance/bundle/resources/dashboards/serialized-inline/test.toml index 9c074644e87..6f91063a6ee 100644 --- a/acceptance/bundle/resources/dashboards/serialized-inline/test.toml +++ b/acceptance/bundle/resources/dashboards/serialized-inline/test.toml @@ -4,7 +4,3 @@ Ignore = [ ] EnvMatrix.READPLAN = ["", "1"] - -# Deploying from a pre-computed plan when the target has DMS deployment history -# is not supported; this variant is skipped. -EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/script b/acceptance/bundle/resources/job_runs/on_bundle_deploy/script index 4f6065bfdce..0ce2a6bbb52 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/script +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/script @@ -15,7 +15,7 @@ title "redeploy re-fires with unchanged config" trace $CLI bundle plan # Save the plan so the READPLAN=1 variant deploys the fingerprint minted here. # The deploys are not traced: readplanarg makes the command line differ per variant. -$CLI bundle plan -o json | nostamp.py > tmp.plan.json +$CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json title "bundle deploy\n" $CLI bundle deploy $(readplanarg tmp.plan.json) @@ -34,7 +34,7 @@ update_file.py databricks.yml " lifecycle: - on_bundle_deploy: true " "" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > tmp.plan.json +$CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json title "bundle deploy\n" $CLI bundle deploy $(readplanarg tmp.plan.json) diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/test.toml b/acceptance/bundle/resources/job_runs/on_bundle_deploy/test.toml index 7eb3b62c045..663d9510c10 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/test.toml +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/test.toml @@ -4,7 +4,3 @@ EnvMatrix.READPLAN = ["", "1"] # The saved plan embeds a fresh fingerprint on every run. Ignore = ["tmp.plan.json"] - -# Deploying from a pre-computed plan when the target has DMS deployment history -# is not supported; this variant is skipped. -EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] From 5d98f2bc97df2c92748738d8ad57e9506f84ba9d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 14:38:39 +0200 Subject: [PATCH 061/164] Enable DMS testing for bundle/select tests Remove EnvMatrix.DMS = [""] blanket skips from: - acceptance/bundle/select/basic - acceptance/bundle/select/grants_permissions Simplify plan saves by removing nostamp.py, allowing plans to preserve DMS headers during deployment. The plan files are ignored from output, so they can now carry DMS metadata needed for replay via --plan. Co-authored-by: Isaac --- acceptance/bundle/select/basic/script | 4 ++-- acceptance/bundle/select/basic/test.toml | 3 --- acceptance/bundle/select/grants_permissions/script | 4 ++-- acceptance/bundle/select/grants_permissions/test.toml | 3 --- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/select/basic/script b/acceptance/bundle/select/basic/script index 195f1edf61c..e8b7741af95 100644 --- a/acceptance/bundle/select/basic/script +++ b/acceptance/bundle/select/basic/script @@ -22,7 +22,7 @@ trace $CLI bundle plan --select jobs.foo # JSON embeds remote state that differs between local and cloud), then deploy it: # inline, or via --plan (READPLAN=1). The deploy is not traced because readplanarg # varies the command line between READPLAN variants, which must produce identical output. -$CLI bundle plan --select jobs.foo -o json | nostamp.py > plan.json +$CLI bundle plan --select jobs.foo -o json > plan.json title "bundle deploy --select jobs.foo\n" $CLI bundle deploy --select jobs.foo $(readplanarg plan.json) # The deploy reports that --select was used via telemetry. @@ -37,7 +37,7 @@ trace $CLI bundle summary # foo and bar are already deployed, so only baz remains to create. title "Full plan after partial deploy" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > plan-full.json +$CLI bundle plan -o json > plan-full.json title "Full deploy\n" $CLI bundle deploy $(readplanarg plan-full.json) # Only baz is created this time. diff --git a/acceptance/bundle/select/basic/test.toml b/acceptance/bundle/select/basic/test.toml index 7166a9e7946..1bb9dae79fc 100644 --- a/acceptance/bundle/select/basic/test.toml +++ b/acceptance/bundle/select/basic/test.toml @@ -9,6 +9,3 @@ EnvMatrix.READPLAN = ["", "1"] # is kept out of the recorded output because it embeds local/cloud-specific state. Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json"] -# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't -# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). -EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/select/grants_permissions/script b/acceptance/bundle/select/grants_permissions/script index edc0fdffb63..1aab6a33fa4 100644 --- a/acceptance/bundle/select/grants_permissions/script +++ b/acceptance/bundle/select/grants_permissions/script @@ -18,11 +18,11 @@ musterr trace $CLI bundle plan --select jobs.my_job.permissions musterr trace $CLI bundle plan --select schemas.my_schema.grants # Deploying a selected resource applies its permissions/grants too. -$CLI bundle plan --select jobs.my_job -o json | nostamp.py > plan-job.json +$CLI bundle plan --select jobs.my_job -o json > plan-job.json title "bundle deploy --select jobs.my_job\n" $CLI bundle deploy --select jobs.my_job $(readplanarg plan-job.json) -$CLI bundle plan --select schemas.my_schema -o json | nostamp.py > plan-schema.json +$CLI bundle plan --select schemas.my_schema -o json > plan-schema.json title "bundle deploy --select schemas.my_schema\n" $CLI bundle deploy --select schemas.my_schema $(readplanarg plan-schema.json) diff --git a/acceptance/bundle/select/grants_permissions/test.toml b/acceptance/bundle/select/grants_permissions/test.toml index 544f8f1e442..813e0b12ed7 100644 --- a/acceptance/bundle/select/grants_permissions/test.toml +++ b/acceptance/bundle/select/grants_permissions/test.toml @@ -12,6 +12,3 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json"] -# Dumps plans and replays them via deploy --plan/readplanarg; the replay needs the DMS fields, -# so opt out of recording (covered by bundle/dms). -EnvMatrix.DMS = [""] From 5863fda6d6acd56b4b80437530c0aee45a69872e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 14:45:40 +0200 Subject: [PATCH 062/164] Simplify plan saves: remove redundant tee+nostamp pattern Replace `plan | tee file | nostamp.py > /dev/null` with direct plan save to file. This pattern was saving the plan via tee and discarding the output through nostamp.py, but since we now preserve DMS headers in plan files anyway, the validation step is redundant. Simplified 5 test scripts: - acceptance/bundle/apps/delete_deleting - acceptance/bundle/apps/source_code_path_update - acceptance/bundle/resources/clusters/resize-terminated-fallback - acceptance/bundle/resources/job_runs/interrupted_run - acceptance/bundle/resources/models/readplan-permissions Co-authored-by: Isaac --- acceptance/bundle/apps/delete_deleting/script | 2 +- acceptance/bundle/apps/source_code_path_update/script | 2 +- .../bundle/resources/clusters/resize-terminated-fallback/script | 2 +- acceptance/bundle/resources/job_runs/interrupted_run/script | 2 +- acceptance/bundle/resources/models/readplan-permissions/script | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/apps/delete_deleting/script b/acceptance/bundle/apps/delete_deleting/script index 8c80871d342..14267a32af4 100644 --- a/acceptance/bundle/apps/delete_deleting/script +++ b/acceptance/bundle/apps/delete_deleting/script @@ -24,7 +24,7 @@ cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null envsubst < databricks_header.yml.tmpl > databricks.yml echo "resources: {}" >> databricks.yml -trace $CLI bundle plan -o json 2>LOG.plan_delete.err | tee plan_delete.json | nostamp.py > /dev/null +trace $CLI bundle plan -o json > plan_delete.json 2>LOG.plan_delete.err cat LOG.plan_delete.err | contains.py '!panic' '!internal error' > /dev/null # Out-of-band delete flips the app into DELETING: a second delete now returns 400. diff --git a/acceptance/bundle/apps/source_code_path_update/script b/acceptance/bundle/apps/source_code_path_update/script index 43397792ffc..d4b781b0b95 100644 --- a/acceptance/bundle/apps/source_code_path_update/script +++ b/acceptance/bundle/apps/source_code_path_update/script @@ -28,7 +28,7 @@ title "Redeploy from saved plan after description change" # Changing a non-deploy-only field (description) so DoUpdate is called. sed -i.bak 's/description: "first description"/description: "second description"/' databricks.yml # Route bundle plan stderr (diagnostics) to a log so output.txt stays clean. -$CLI bundle plan -o json 2> LOG.plan | tee plan.json | nostamp.py > /dev/null +$CLI bundle plan -o json > plan.json 2> LOG.plan # bundle deploy --plan triggers InitForApply, which JSON-round-trips AppState. # Without the fix, apps.App.ForceSendFields ends up containing "SourceCodePath", # causing "source_code_path": "" to be sent in the UpdateApp body (API returns 400). diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/script b/acceptance/bundle/resources/clusters/resize-terminated-fallback/script index cef13c4aee7..eff7c541b94 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/script +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/script @@ -13,7 +13,7 @@ add_repl "$CLUSTER_ID" CLUSTER_ID title "Create a plan while cluster is running: should show resize\n" update_file.py databricks.yml "num_workers: 2" "num_workers: 3" -trace $CLI bundle plan -o json | tee plan.json | nostamp.py > /dev/null +trace $CLI bundle plan -o json > plan.json title "Terminate the cluster before applying the saved plan\n" $CLI clusters delete "$CLUSTER_ID" > /dev/null diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/script b/acceptance/bundle/resources/job_runs/interrupted_run/script index 0328c881106..68338c1d217 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/script +++ b/acceptance/bundle/resources/job_runs/interrupted_run/script @@ -18,7 +18,7 @@ trace read_id.py my_run trace print_requests.py //jobs/run-now title "a run that is still going is skipped" -$CLI bundle plan -o json | tee tmp.plan.json | nostamp.py > /dev/null +$CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"]' tmp.plan.json # Replay the plan captured while the run was still going: a fresh plan would diff --git a/acceptance/bundle/resources/models/readplan-permissions/script b/acceptance/bundle/resources/models/readplan-permissions/script index d10c2485cda..f8be26409e3 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/script +++ b/acceptance/bundle/resources/models/readplan-permissions/script @@ -17,7 +17,7 @@ trace print_requests.py //permissions/registered-models title "Remove manager permission and description, deploy from plan" grep -v TO_REMOVE databricks.yml > updated.yml && mv updated.yml databricks.yml -$CLI bundle plan -o json | tee tmp.plan.json | nostamp.py > /dev/null +$CLI bundle plan -o json > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) register_model_id trace print_requests.py //permissions/registered-models From 0f14dd418c75b9d4ebc02c01ba8649a449a8ae9b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 14:54:18 +0200 Subject: [PATCH 063/164] Fix whitespace: remove trailing blank lines from test.toml files --- acceptance/bundle/select/basic/test.toml | 3 +-- acceptance/bundle/select/grants_permissions/test.toml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/select/basic/test.toml b/acceptance/bundle/select/basic/test.toml index 1bb9dae79fc..74bb671878d 100644 --- a/acceptance/bundle/select/basic/test.toml +++ b/acceptance/bundle/select/basic/test.toml @@ -7,5 +7,4 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] # databricks.yml and the serialized plans are generated at runtime; the plan JSON # is kept out of the recorded output because it embeds local/cloud-specific state. -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json"] - +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json"] \ No newline at end of file diff --git a/acceptance/bundle/select/grants_permissions/test.toml b/acceptance/bundle/select/grants_permissions/test.toml index 813e0b12ed7..f6571fe0223 100644 --- a/acceptance/bundle/select/grants_permissions/test.toml +++ b/acceptance/bundle/select/grants_permissions/test.toml @@ -10,5 +10,4 @@ Cloud = false RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json"] - +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json"] \ No newline at end of file From 620fcb944297c55a357aced0e625bf039fad5be3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 14:55:39 +0200 Subject: [PATCH 064/164] Update golden outputs for DMS-enabled tests Regenerate outputs after enabling DMS testing: - select/plan_file_survives_not_selected: now deploys successfully with DMS - dashboards/change-serialized-dashboard: updated request and output files for DMS variants - select/basic and grants_permissions: updated out.test.toml to reflect DMS=true variants Co-authored-by: Isaac --- .../out.patch.requests.direct.txt | 10 +++++ .../out.post.requests.txt | 40 +++++++++++++++++ .../change-serialized-dashboard/output.txt | 44 +++++++++++++++++-- acceptance/bundle/select/basic/out.test.toml | 2 +- .../select/grants_permissions/out.test.toml | 2 +- 5 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.patch.requests.direct.txt create mode 100644 acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.post.requests.txt diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.patch.requests.direct.txt b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.patch.requests.direct.txt new file mode 100644 index 00000000000..361de88df6c --- /dev/null +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.patch.requests.direct.txt @@ -0,0 +1,10 @@ +{ + "method": "PATCH", + "path": "/api/2.0/lakeview/dashboards/[DASHBOARD_ID]", + "body": { + "display_name": "test dashboard", + "parent_path": "/Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources", + "serialized_dashboard": "{\"pages\":[{\"name\":\"name2\",\"displayName\":\"name2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } +} diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.post.requests.txt b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.post.requests.txt new file mode 100644 index 00000000000..3c3e001344d --- /dev/null +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.post.requests.txt @@ -0,0 +1,40 @@ +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources" + } +} +{ + "method": "POST", + "path": "/api/2.0/lakeview/dashboards", + "body": { + "display_name": "test dashboard", + "parent_path": "/Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources", + "serialized_dashboard": "{\"pages\":[{\"name\":\"name1\",\"displayName\":\"name1\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } +} +{ + "method": "POST", + "path": "/api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published", + "body": { + "embed_credentials": false, + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources" + } +} +{ + "method": "POST", + "path": "/api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published", + "body": { + "embed_credentials": false, + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } +} diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/output.txt b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/output.txt index b44f461778e..525a432aabb 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/output.txt +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/output.txt @@ -3,10 +3,48 @@ create dashboards.my_dashboard Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged -Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/files... +Created dashboards.my_dashboard +Files: 9 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +{ + "display_name": "test dashboard", + "lifecycle_state": "ACTIVE", + "parent_path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources", + "path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources/test dashboard.lvdash.json", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" +} +{ + "displayName": "name1", + "name": "name1", + "pageType": "PAGE_TYPE_CANVAS" +} +>>> [CLI] bundle plan +update dashboards.my_dashboard + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/files... +Updated dashboards.my_dashboard +Files: 5 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +{ + "display_name": "test dashboard", + "lifecycle_state": "ACTIVE", + "parent_path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources", + "path": "/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default/resources/test dashboard.lvdash.json", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" +} +{ + "displayName": "name2", + "name": "name2", + "pageType": "PAGE_TYPE_CANVAS" +} >>> [CLI] bundle destroy --auto-approve -No active deployment found to destroy! +The following resources will be deleted: + delete resources.dashboards.my_dashboard + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/change-serialized-dashboard-[UNIQUE_NAME]/default -Exit code: 1 +Destroy: 1 deleted diff --git a/acceptance/bundle/select/basic/out.test.toml b/acceptance/bundle/select/basic/out.test.toml index b5dac5e552c..a420f50c421 100644 --- a/acceptance/bundle/select/basic/out.test.toml +++ b/acceptance/bundle/select/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/grants_permissions/out.test.toml b/acceptance/bundle/select/grants_permissions/out.test.toml index bf0f576652e..dd03baaabb6 100644 --- a/acceptance/bundle/select/grants_permissions/out.test.toml +++ b/acceptance/bundle/select/grants_permissions/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] From 15c7df293506bb68ea6b1793efd4202bc5619d50 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 15:01:27 +0200 Subject: [PATCH 065/164] Enable DMS for serialized dashboard test Use tee+nostamp.py pattern: save raw plan with DMS headers to tmp file for deployment, apply nostamp.py when generating golden files for comparison. This allows change-serialized-inline to run with all DMS and READPLAN combinations: - DMS=true + READPLAN=1 (newly enabled) - DMS=true + READPLAN="" - DMS="" + READPLAN=1 - DMS="" + READPLAN="" All 5 variants pass. Co-authored-by: Isaac --- .../dashboards/change-serialized-inline/out.test.toml | 2 +- .../dashboards/change-serialized-inline/output.txt | 2 +- .../resources/dashboards/change-serialized-inline/script | 7 +++++-- .../dashboards/change-serialized-inline/test.toml | 6 ++---- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml index d5b9a6f4a16..bf4f0e7d3c6 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/output.txt b/acceptance/bundle/resources/dashboards/change-serialized-inline/output.txt index ca5a2744eaf..d8537af591b 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/output.txt +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/output.txt @@ -15,7 +15,7 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === deploy the change Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/change-serialized-inline-[UNIQUE_NAME]/default/files... Updated dashboards.inline_dash -Files: 4 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/script b/acceptance/bundle/resources/dashboards/change-serialized-inline/script index 41629784e3f..116901bf8bd 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/script +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/script @@ -23,9 +23,12 @@ update_file.py databricks.yml "Page 1" "Page One" trace $CLI bundle plan | contains.py "update" "!recreate" -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# Save raw plan (with DMS headers) to tmp file for deployment +trace $CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json +# Generate golden file by applying nostamp.py for comparison (discard output) +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json title "deploy the change\n" -$CLI bundle deploy $(readplanarg out.plan.$DATABRICKS_BUNDLE_ENGINE.json) +$CLI bundle deploy $(readplanarg plan_raw.$DATABRICKS_BUNDLE_ENGINE.json) print_requests.py //api/2.0/lakeview/dashboards > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml b/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml index fae8d4e5d0e..d6cc7185a84 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml @@ -5,10 +5,8 @@ Cloud = false Ignore = [ "databricks.yml", + "plan_raw.terraform.json", + "plan_raw.direct.json", ] EnvMatrix.READPLAN = ["", "1"] - -# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't -# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). -EnvMatrix.DMS = [""] From 9442cab69a4b613d78d23840533330f0d9fcaa77 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 15:02:25 +0200 Subject: [PATCH 066/164] Enable DMS for dashboard tests: simple and detect-change Apply tee+nostamp.py pattern: save raw plan with DMS headers to tmp file, apply nostamp.py when generating golden files. Both tests now run with all DMS and READPLAN combinations. Co-authored-by: Isaac --- acceptance/bundle/resources/dashboards/detect-change/script | 4 +++- .../bundle/resources/dashboards/detect-change/test.toml | 2 ++ acceptance/bundle/resources/dashboards/simple/script | 4 +++- acceptance/bundle/resources/dashboards/simple/test.toml | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/dashboards/detect-change/script b/acceptance/bundle/resources/dashboards/detect-change/script index d7c2d18037b..4c5ac9d4796 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/script +++ b/acceptance/bundle/resources/dashboards/detect-change/script @@ -38,7 +38,9 @@ add_repl "$(echo "$UPDATE_RESP" | jq -r '.etag')" ETAG_2 title "Try to redeploy the bundle and confirm that the out of band modification is detected:" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# Save raw plan (with DMS headers), then apply nostamp.py for golden file +trace $CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy title "Redeploy the bundle with the --force flag and confirm that the out of band modification is ignored:" diff --git a/acceptance/bundle/resources/dashboards/detect-change/test.toml b/acceptance/bundle/resources/dashboards/detect-change/test.toml index fd03fab15ad..83028566e5f 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/test.toml +++ b/acceptance/bundle/resources/dashboards/detect-change/test.toml @@ -2,6 +2,8 @@ Cloud = true Ignore = [ "databricks.yml", + "plan_raw.terraform.json", + "plan_raw.direct.json", ] # MSYS2 automatically converts absolute paths like /Users/$username/$UNIQUE_NAME to diff --git a/acceptance/bundle/resources/dashboards/simple/script b/acceptance/bundle/resources/dashboards/simple/script index 6e2b14529f1..867831720b8 100644 --- a/acceptance/bundle/resources/dashboards/simple/script +++ b/acceptance/bundle/resources/dashboards/simple/script @@ -24,4 +24,6 @@ echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path, serialized_dashboar # Verify that there is no drift right after deploy. We use |= to update the serialized_dashboard field in place to account # for formatting that the cloud server applies to the serialized_dashboard field. -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# Save raw plan (with DMS headers), then apply nostamp.py for golden file +trace $CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/dashboards/simple/test.toml b/acceptance/bundle/resources/dashboards/simple/test.toml index 4d77b7783c8..7694da9756d 100644 --- a/acceptance/bundle/resources/dashboards/simple/test.toml +++ b/acceptance/bundle/resources/dashboards/simple/test.toml @@ -4,4 +4,6 @@ RunsOnDbr = true Ignore = [ "databricks.yml", + "plan_raw.terraform.json", + "plan_raw.direct.json", ] From 3dbda9812a16b79e23b680470852b632a42fb05d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 15:03:20 +0200 Subject: [PATCH 067/164] Enable DMS for more dashboard tests Apply tee+nostamp.py pattern to: - delete-trashed-out-of-band - dataset-catalog-schema - unpublish-out-of-band All tests now save raw plan with DMS headers to tmp file, apply nostamp.py when generating golden files. Tests run with all DMS and READPLAN combinations. Co-authored-by: Isaac --- .../resources/dashboards/dataset-catalog-schema/script | 4 +++- .../resources/dashboards/dataset-catalog-schema/test.toml | 4 ++++ .../dashboards/delete-trashed-out-of-band/script | 4 +++- .../dashboards/delete-trashed-out-of-band/test.toml | 2 ++ .../resources/dashboards/unpublish-out-of-band/script | 8 ++++++-- .../resources/dashboards/unpublish-out-of-band/test.toml | 4 ++++ 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script index e5a1a51185a..0016ebe8227 100755 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script @@ -29,7 +29,9 @@ echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path}' echo "$DASHBOARD" | jq '.serialized_dashboard | fromjson | .datasets[] | {catalog, schema}' # Verify that there is no drift right after deploy. -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# Save raw plan (with DMS headers), then apply nostamp.py for golden file +trace $CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Modify the direct plan to replace "serialized_dashboard" with a fixture. # It is normalized on the backend so we cannot compare reliably across local and cloud. diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/test.toml b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/test.toml index 03d3f87c9f0..d86ff66a53d 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/test.toml +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/test.toml @@ -2,3 +2,7 @@ Cloud = true RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +Ignore = [ + "plan_raw.terraform.json", + "plan_raw.direct.json", +] diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script index b8928da1198..4746b59f8cf 100755 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script @@ -25,7 +25,9 @@ trace $CLI lakeview trash $DASHBOARD_ID trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# Save raw plan (with DMS headers), then apply nostamp.py for golden file +trace $CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Run destroy - should succeed even though dashboard is already trashed. # -qq drops the summary: terraform refreshes the trashed dashboard away and reports diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/test.toml b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/test.toml index 4d77b7783c8..7694da9756d 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/test.toml +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/test.toml @@ -4,4 +4,6 @@ RunsOnDbr = true Ignore = [ "databricks.yml", + "plan_raw.terraform.json", + "plan_raw.direct.json", ] diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script index ddb2b9e8b34..7b2a9604039 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script @@ -6,7 +6,9 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json | nostamp.py > out.plan_initial.$DATABRICKS_BUNDLE_ENGINE.json +# Save raw plan (with DMS headers), then apply nostamp.py for golden file +trace $CLI bundle plan -o json > plan_raw_initial.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw_initial.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan_initial.$DATABRICKS_BUNDLE_ENGINE.json # Deploy the dashboard trace $CLI bundle deploy -qq @@ -25,7 +27,9 @@ trace $CLI lakeview unpublish $DASHBOARD_ID # Direct: shows "update" because Published field changes from false to true trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# Save raw plan (with DMS headers), then apply nostamp.py for golden file +$CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json json_in_json_normalize.py out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -qq diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/test.toml b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/test.toml index 4d77b7783c8..cd67bcda24f 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/test.toml +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/test.toml @@ -4,4 +4,8 @@ RunsOnDbr = true Ignore = [ "databricks.yml", + "plan_raw_initial.terraform.json", + "plan_raw_initial.direct.json", + "plan_raw.terraform.json", + "plan_raw.direct.json", ] From 0c743d9f5b605357031d18b23d2e4bb63c2efbae Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 15:04:26 +0200 Subject: [PATCH 068/164] Add nostamp wrapper function to script.prepare Create a bash wrapper for nostamp.py (similar to existing wrappers for envsubst, retry, add_repl) to: 1. Reduce code churn - all scripts can now use 'nostamp' instead of 'nostamp.py' 2. Improve Windows compatibility - handles MSYS_NO_PATHCONV automatically 3. Maintain consistency with other python script wrappers Replace all 277 instances of 'nostamp.py' with 'nostamp' throughout acceptance tests. Co-authored-by: Isaac --- acceptance/bundle/artifacts/whl_dynamic/script | 6 +++--- .../artifacts/whl_implicit_custom_path/script | 2 +- .../artifacts/whl_via_environment_key/script | 2 +- .../whl_via_environment_key_extras/script | 2 +- acceptance/bundle/bundle_tag/id/script | 2 +- acceptance/bundle/bundle_tag/url/script | 2 +- .../bundle/deploy/experimental-python/script | 2 +- acceptance/bundle/deploy/immutable/script | 2 +- acceptance/bundle/deploy/python-notebook/script | 2 +- acceptance/bundle/deploy/readplan/basic/script | 4 ++-- .../deploy/readplan/cli-version-mismatch/script | 2 +- .../deploy/readplan/grants-remove-principal/script | 2 +- .../bundle/deploy/readplan/postgres_role/script | 2 +- .../bundle/deploy/readplan/serial-mismatch/script | 2 +- .../bundle/deploy/readplan/unknown-field/script | 2 +- .../bundle/deploy/readplan/whl-not-uploaded/script | 2 +- acceptance/bundle/deploy/wal/chain-3-jobs/script | 2 +- .../bundle/deploy/wal/crash-after-create/script | 2 +- .../deployment/bind/dashboard/recreation/script | 4 ++-- acceptance/bundle/deployment/bind/dashboard/script | 2 +- .../bundle/deployment/bind/experiment/script | 4 ++-- acceptance/bundle/destroy/jobs-and-pipeline/script | 2 +- acceptance/bundle/empty_string_dropped/script | 2 +- acceptance/bundle/environments/dependencies/script | 8 ++++---- acceptance/bundle/escaped_refs/script | 2 +- .../bundle/invariant/delete_idempotent/script | 6 +++--- .../bundle/invariant/destroy_idempotent/script | 2 +- .../bundle/migrate/auto-migrate-clean/script | 2 +- .../bundle/migrate/auto-migrate-direct-only/script | 2 +- acceptance/bundle/migrate/basic/script | 2 +- acceptance/bundle/migrate/dashboards/script | 4 ++-- acceptance/bundle/migrate/default-python/script | 4 ++-- acceptance/bundle/migrate/runas/script | 2 +- acceptance/bundle/resource_deps/escaped_ref/script | 2 +- acceptance/bundle/resource_deps/grant_ref/script | 2 +- acceptance/bundle/resource_deps/id_chain/script | 6 +++--- acceptance/bundle/resource_deps/id_star/script | 2 +- acceptance/bundle/resource_deps/job_id/script | 4 ++-- .../bundle/resource_deps/job_id_delete_bar/script | 2 +- .../bundle/resource_deps/job_id_delete_foo/script | 2 +- acceptance/bundle/resource_deps/jobs_update/script | 4 ++-- .../bundle/resource_deps/jobs_update_remote/script | 4 ++-- .../resource_deps/missing_string_field/script | 2 +- .../bundle/resource_deps/permission_ref/script | 2 +- .../bundle/resource_deps/pipelines_recreate/script | 10 +++++----- .../bundle/resource_deps/remote_pipeline/script | 4 ++-- .../bundle/resource_deps/resources_var/script | 2 +- .../resources_var_presets_implicit_deps/script | 2 +- .../bundle/resource_deps/tf_path_renames/script | 2 +- .../resource_deps/volume_path_contains_id/script | 2 +- .../resource_deps/volume_path_job_ref/script | 4 ++-- .../resources/apps/config-drift-stopped/script | 4 ++-- .../bundle/resources/apps/config-drift/script | 6 +++--- .../resources/apps/config-no-deployment/script | 2 +- .../resources/apps/git-source-no-deployment/script | 2 +- .../apps/lifecycle-started-omitted/script | 2 +- .../resources/apps/readplan-lifecycle/script | 2 +- .../resources/apps/uc-securable-drift/script | 2 +- .../resources/catalogs/comment_out_of_band/script | 2 +- .../catalogs/drift/managed_properties/script | 2 +- .../deploy/update-and-resize-autoscale/script | 12 ++++++------ .../clusters/deploy/update-and-resize/script | 10 +++++----- .../dashboards/change-serialized-inline/script | 4 ++-- .../dashboards/dataset-catalog-schema/script | 4 ++-- .../dashboards/delete-trashed-out-of-band/script | 4 ++-- .../resources/dashboards/detect-change/script | 4 ++-- .../publish-failure-stale-content/script | 4 ++-- .../dashboards/republish-after-draft-update/script | 4 ++-- .../bundle/resources/dashboards/simple/script | 4 ++-- .../dashboards/unpublish-out-of-band/script | 8 ++++---- .../bundle/resources/genie_spaces/inline/script | 2 +- .../genie_spaces/version_migration/script | 2 +- acceptance/bundle/resources/grants/catalogs/script | 4 ++-- .../resources/grants/registered_models/script | 2 +- .../grants/schemas/change_privilege/script | 4 ++-- .../grants/schemas/duplicate_principals/script | 2 +- .../grants/schemas/out_of_band_principal/script | 4 ++-- acceptance/bundle/resources/grants/volumes/script | 4 ++-- .../bundle/resources/job_runs/failed_run/script | 2 +- .../bundle/resources/job_runs/redeploy/script | 2 +- acceptance/bundle/resources/job_runs/wait/script | 2 +- acceptance/bundle/resources/jobs/big_id/script | 2 +- acceptance/bundle/resources/jobs/delete_job/script | 2 +- .../bundle/resources/jobs/delete_task/script | 4 ++-- .../resources/jobs/on_failure_empty_slice/script | 2 +- .../bundle/resources/jobs/remote_add_tag/script | 2 +- .../resources/jobs/remote_delete/deploy/script | 2 +- .../resources/jobs/remote_matches_config/script | 2 +- .../bundle/resources/jobs/tags_empty_map/script | 2 +- acceptance/bundle/resources/jobs/update/script | 6 +++--- .../resources/jobs/update_single_node/script | 6 +++--- .../resources/jobs/webhook-reorder-remote/script | 2 +- .../resources/model_serving_endpoints/basic/script | 4 ++-- .../drift/telemetry_config_unmanaged/script | 2 +- .../drift/write_only/script | 2 +- .../recreate/catalog-name/script | 4 ++-- .../recreate/name-change/script | 4 ++-- .../recreate/route-optimized/script | 4 ++-- .../recreate/schema-name/script | 4 ++-- .../recreate/table-prefix/script | 4 ++-- .../running-endpoint/script | 2 +- .../update/ai-gateway/script | 2 +- .../update/both_gateway_and_tags/script | 2 +- .../model_serving_endpoints/update/config/script | 2 +- .../update/email-notifications/script | 2 +- .../model_serving_endpoints/update/tags/script | 2 +- .../resources/permissions/dashboards/create/script | 2 +- .../genie_spaces/current_can_manage/script | 2 +- .../permissions/jobs/added_remotely/script | 2 +- .../permissions/jobs/current_can_manage_run/script | 4 ++-- .../resources/permissions/jobs/delete_one/script | 2 +- .../permissions/jobs/deleted_remotely/script | 4 ++-- .../resources/permissions/jobs/update/script | 14 +++++++------- .../permissions/models/current_can_manage/script | 2 +- .../resources/permissions/pipelines/update/script | 10 +++++----- .../permissions/target_permissions/script | 2 +- .../bundle/resources/pipelines/auto-approve/script | 8 ++++---- .../resources/pipelines/lakeflow-pipeline/script | 2 +- .../resources/pipelines/num-workers-zero/script | 2 +- .../bundle/resources/pipelines/recreate/script | 4 ++-- .../pipelines/remote_matches_config/script | 2 +- .../bundle/resources/pipelines/update/script | 2 +- .../recreate_source_branch_time/script | 2 +- .../postgres_branches/update_expire_time/script | 2 +- .../postgres_branches/update_protected/script | 8 ++++---- .../resources/postgres_branches/update_ttl/script | 2 +- .../resources/postgres_databases/update/script | 8 ++++---- .../remove_suspend_timeout/script | 4 ++-- .../postgres_endpoints/update_autoscaling/script | 8 ++++---- .../postgres_endpoints/update_pg_settings/script | 2 +- .../update_suspend_timeout/script | 2 +- .../remove_history_retention/script | 4 ++-- .../update_default_endpoint_suspend/script | 2 +- .../postgres_projects/update_display_name/script | 8 ++++---- .../update_history_retention/script | 2 +- .../bundle/resources/postgres_roles/update/script | 8 ++++---- .../quality_monitors/change_assets_dir/script | 2 +- .../change_output_schema_name/script | 2 +- .../quality_monitors/change_table_name/script | 2 +- .../resources/quality_monitors/create/script | 4 ++-- .../registered_models/aliases_converge/script | 2 +- .../resources/registered_models/basic/script | 2 +- .../registered_models/drift/browse_only/script | 2 +- .../bundle/resources/schemas/auto-approve/script | 4 ++-- .../resources/schemas/comment_out_of_band/script | 2 +- .../schemas/drift/managed_properties/script | 2 +- .../bundle/resources/secret_scopes/basic/script | 6 +++--- .../resources/secrets/remove-expire-time/script | 4 ++-- .../resources/secrets/update-expire-time/script | 2 +- .../bundle/resources/secrets/update-value/script | 4 ++-- .../drift/target_qps/script | 2 +- .../recreate/with_endpoint/script | 2 +- .../resources/volumes/catalog-var-ref/script | 2 +- .../bundle/resources/volumes/change-name/script | 2 +- .../resources/volumes/comment_out_of_band/script | 2 +- .../resources/volumes/remote-change-name/script | 2 +- .../bundle/resources/volumes/uppercase-name/script | 2 +- .../select/plan_file_survives_not_selected/script | 2 +- .../bundle/state/permission_level_migration/script | 4 ++-- acceptance/bundle/summary/modified_status/script | 8 ++++---- .../bundle/templates/default-python/classic/script | 8 ++++---- .../default-python/combinations/classic/script | 4 ++-- .../default-python/combinations/serverless/script | 4 ++-- .../default-python/integration_classic/script | 4 ++-- .../templates/default-python/serverless/script | 8 ++++---- acceptance/script.prepare | 6 ++++++ 166 files changed, 283 insertions(+), 277 deletions(-) diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index 68d328cc67d..ddb6fcd55ec 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -4,9 +4,9 @@ cp -r $TESTDIR/../whl_explicit/my_test_code . mkdir prebuilt cp -r $TESTDIR/../whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl prebuilt -trace $CLI bundle validate -o json | nostamp.py | jq .artifacts +trace $CLI bundle validate -o json | nostamp | jq .artifacts -$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "There are 2 original wheels and 2 patched ones" @@ -25,7 +25,7 @@ rm out.requests.txt title "Updating the local wheel and deploying again\n" touch my_test_code/src/new_module.py -$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Verify contents, it should now have new_module.py" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index 21bc2c9addc..623849d3bb5 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index d62de02530b..f82c5d7eccf 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in environments section in /jobs/create" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script index 59268fdeb0c..ccc0d9ed7f8 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting the environments dependency to keep its [train] extras suffix" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp title "Expecting 1 wheel to be uploaded under its bare filename (no extras)" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/bundle_tag/id/script b/acceptance/bundle/bundle_tag/id/script index 200c5a7d2e6..759cc89eba1 100644 --- a/acceptance/bundle/bundle_tag/id/script +++ b/acceptance/bundle/bundle_tag/id/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -trace $CLI bundle validate -o json | nostamp.py | jq .resources +trace $CLI bundle validate -o json | nostamp | jq .resources trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/bundle_tag/url/script b/acceptance/bundle/bundle_tag/url/script index 200c5a7d2e6..759cc89eba1 100644 --- a/acceptance/bundle/bundle_tag/url/script +++ b/acceptance/bundle/bundle_tag/url/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -trace $CLI bundle validate -o json | nostamp.py | jq .resources +trace $CLI bundle validate -o json | nostamp | jq .resources trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/deploy/experimental-python/script b/acceptance/bundle/deploy/experimental-python/script index 09bf4b14ef6..09b90b28e2d 100644 --- a/acceptance/bundle/deploy/experimental-python/script +++ b/acceptance/bundle/deploy/experimental-python/script @@ -1,3 +1,3 @@ trace uv run --quiet --with $DATABRICKS_BUNDLES_WHEEL -- $CLI bundle deploy -trace $CLI jobs list --output json | nostamp.py +trace $CLI jobs list --output json | nostamp diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/script index 83136e53447..86757855d55 100644 --- a/acceptance/bundle/deploy/immutable/script +++ b/acceptance/bundle/deploy/immutable/script @@ -5,7 +5,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle validate -trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' +trace $CLI bundle plan -o json | nostamp | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' trace $CLI bundle deploy diff --git a/acceptance/bundle/deploy/python-notebook/script b/acceptance/bundle/deploy/python-notebook/script index 8b12727b59e..21318c58b66 100644 --- a/acceptance/bundle/deploy/python-notebook/script +++ b/acceptance/bundle/deploy/python-notebook/script @@ -1,3 +1,3 @@ trace $CLI bundle deploy -trace $CLI jobs list --output json | nostamp.py +trace $CLI jobs list --output json | nostamp diff --git a/acceptance/bundle/deploy/readplan/basic/script b/acceptance/bundle/deploy/readplan/basic/script index e01d5c5f3e7..f7dc5343efc 100644 --- a/acceptance/bundle/deploy/readplan/basic/script +++ b/acceptance/bundle/deploy/readplan/basic/script @@ -1,5 +1,5 @@ # Generate initial plan -trace $CLI bundle plan -o json | nostamp.py > out.plan_create.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.json trace print_requests.py --get //jobs | contains.py "!GET" "!POST" # First deploy to create the resource @@ -7,7 +7,7 @@ trace $CLI bundle deploy --plan out.plan_create.json trace print_requests.py --get //jobs | contains.py "!GET" "POST" # Generate a plan (should show "skip" since nothing changed) -trace $CLI bundle plan -o json | nostamp.py > out.plan_skip.json +trace $CLI bundle plan -o json | nostamp > out.plan_skip.json trace print_requests.py --get //jobs | contains.py "GET" trace $CLI bundle deploy --plan out.plan_skip.json diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/script b/acceptance/bundle/deploy/readplan/cli-version-mismatch/script index 6c6b9a87ef5..e778bba5d18 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/script +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/script @@ -1,4 +1,4 @@ -trace $CLI bundle plan -o json | nostamp.py > plan.json +trace $CLI bundle plan -o json | nostamp > plan.json jq '.cli_version = "1.2.3"' plan.json > plan.json.tmp && mv plan.json.tmp plan.json diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/script b/acceptance/bundle/deploy/readplan/grants-remove-principal/script index 962c456616c..541bbb8a506 100644 --- a/acceptance/bundle/deploy/readplan/grants-remove-principal/script +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/script @@ -14,7 +14,7 @@ grep -v 'TO_REMOVE' databricks.yml > updated.yml && mv updated.yml databricks.ym # Generate update plan: entry.RemoteState captures both principals as GrantsState. # remote_state.__embed__ principal order is non-deterministic (the backend, and the # testserver mirroring it, iterate a map), so sort it by principal for a stable golden. -trace $CLI bundle plan -o json | nostamp.py > tmp.plan.json +trace $CLI bundle plan -o json | nostamp > tmp.plan.json jq '.plan["resources.schemas.myschema.grants"] | .remote_state.__embed__ |= sort_by(.principal)' tmp.plan.json > out.plan.grants.json trace print_requests.py --get //unity-catalog/permissions --sort diff --git a/acceptance/bundle/deploy/readplan/postgres_role/script b/acceptance/bundle/deploy/readplan/postgres_role/script index d50a05bddff..f4b16d8069c 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/script +++ b/acceptance/bundle/deploy/readplan/postgres_role/script @@ -6,7 +6,7 @@ # empty role_id, the server would generate its own, and the next plan would # report phantom "recreate" drift on the immutable role_id. -trace $CLI bundle plan -o json | nostamp.py > plan.json +trace $CLI bundle plan -o json | nostamp > plan.json # Keep only the role entry; role_id and parent must survive serialization. trace jq '.plan."resources.postgres_roles.role"' plan.json > out.plan.role.json diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/script b/acceptance/bundle/deploy/readplan/serial-mismatch/script index 5eff9b6ad49..a3a64529a1d 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/script +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/script @@ -2,7 +2,7 @@ trace $CLI bundle deploy # Generate a plan (captures current lineage and serial) -trace $CLI bundle plan -o json | nostamp.py > plan.json +trace $CLI bundle plan -o json | nostamp > plan.json # Modify only the serial to simulate stale plan jq '.serial = 999' plan.json > plan.json.tmp && mv plan.json.tmp plan.json diff --git a/acceptance/bundle/deploy/readplan/unknown-field/script b/acceptance/bundle/deploy/readplan/unknown-field/script index ea17245ace1..1b46c6694e7 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/script +++ b/acceptance/bundle/deploy/readplan/unknown-field/script @@ -1,5 +1,5 @@ # Generate a plan -trace $CLI bundle plan -o json | nostamp.py > out.plan.json +trace $CLI bundle plan -o json | nostamp > out.plan.json # Modify the plan to rename "action" field to "action_renamed" # This will create an unknown field that the strict decoder should reject diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script index 96237d6cdd0..935ea678077 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script @@ -15,7 +15,7 @@ trace $CLI bundle destroy --auto-approve rm out.requests.txt title "bundle deploy --plan does NOT upload the wheel (bug)" -trace $CLI bundle plan -o json | nostamp.py > out.plan.json +trace $CLI bundle plan -o json | nostamp > out.plan.json trace $CLI bundle deploy --plan out.plan.json # BUG: no import-file request for the wheel here (output below is empty), yet the job was # created pointing the task at the same .../artifacts/.internal/ path that was never uploaded. diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/script b/acceptance/bundle/deploy/wal/chain-3-jobs/script index 5cc77faa786..c41e8e7f995 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/script +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/script @@ -7,7 +7,7 @@ trace errcode $CLI bundle deploy echo "" echo "=== WAL content after crash ===" -jq -S . .databricks/bundle/default/resources.json.wal 2>/dev/null | nostamp.py || echo "No WAL file" +jq -S . .databricks/bundle/default/resources.json.wal 2>/dev/null | nostamp || echo "No WAL file" echo "" echo "=== Number of jobs saved in WAL ===" diff --git a/acceptance/bundle/deploy/wal/crash-after-create/script b/acceptance/bundle/deploy/wal/crash-after-create/script index 22f7587e52e..5647c83e387 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/script +++ b/acceptance/bundle/deploy/wal/crash-after-create/script @@ -8,7 +8,7 @@ trace errcode $CLI bundle deploy trace assert_exists.py .databricks/bundle/default/resources.json.wal trace assert_not_exists.py .databricks/bundle/default/resources.json -trace cat .databricks/bundle/default/resources.json.wal | jq | nostamp.py +trace cat .databricks/bundle/default/resources.json.wal | jq | nostamp title "Any other command recovers state" $CLI bundle $COMMAND &> LOG.COMMAND.txt diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/script b/acceptance/bundle/deployment/bind/dashboard/recreation/script index 1dd7e20d9bd..33e7869713f 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/script +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/script @@ -20,13 +20,13 @@ add_repl $ETAG ETAG json_in_json_normalize.py out.state_after_bind.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > tmp.plan.json +$CLI bundle plan -o json | nostamp > tmp.plan.json errcode $CLI bundle deploy $(readplanarg tmp.plan.json) trace $CLI bundle deployment unbind dashboard1 # Re-plan after unbind: the plan saved above no longer matches state. -$CLI bundle plan -o json | nostamp.py > tmp.plan_after_unbind.json +$CLI bundle plan -o json | nostamp > tmp.plan_after_unbind.json $CLI bundle deploy --auto-approve $(readplanarg tmp.plan_after_unbind.json) trace $CLI lakeview get "${DASHBOARD_ID}" | jq '{display_name, lifecycle_state}' diff --git a/acceptance/bundle/deployment/bind/dashboard/script b/acceptance/bundle/deployment/bind/dashboard/script index 37d85c598af..130ba8d74a3 100644 --- a/acceptance/bundle/deployment/bind/dashboard/script +++ b/acceptance/bundle/deployment/bind/dashboard/script @@ -15,7 +15,7 @@ trace $CLI bundle deployment bind dashboard1 "${DASHBOARD_ID}" --auto-approve # Bind writes state without going through DoCreate/DoUpdate, so a saved-plan deploy after # a bind must still send the full serialized_dashboard; the lakeview get below reads it back. -$CLI bundle plan -o json | nostamp.py > tmp.plan.json +$CLI bundle plan -o json | nostamp > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) trace $CLI lakeview get "${DASHBOARD_ID}" | jq --sort-keys '{display_name, lifecycle_state, path, parent_path, serialized_dashboard} | .serialized_dashboard |= fromjson' diff --git a/acceptance/bundle/deployment/bind/experiment/script b/acceptance/bundle/deployment/bind/experiment/script index e31a17f7eaa..e4528d6b24c 100644 --- a/acceptance/bundle/deployment/bind/experiment/script +++ b/acceptance/bundle/deployment/bind/experiment/script @@ -21,7 +21,7 @@ trace $CLI bundle deploy --force-lock --auto-approve # Use --output json to verify the change details including skip reason. # Write per-engine output since direct includes change details that terraform doesn't. # Filter to only the name change (tags/artifact_location may differ between local and cloud). -$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment.name, lifecycle_stage: .experiment.lifecycle_stage}' @@ -29,7 +29,7 @@ trace $CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment. # Plan should still show no changes because the diff is suppressed. EXPERIMENT_NAME="//Users/${CURRENT_USER_NAME}/test-experiment$UNIQUE_NAME" envsubst < databricks.yml.tmpl > databricks.yml -$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deployment unbind experiment1 diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/script b/acceptance/bundle/destroy/jobs-and-pipeline/script index 2391f9d44e0..b02d93bcecb 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/script +++ b/acceptance/bundle/destroy/jobs-and-pipeline/script @@ -45,7 +45,7 @@ trace $CLI workspace get-status "${DEPLOYMENT_PATH}" | jq '{path, object_type}' title "Assert the pipeline is created" PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.bar.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py | jq "{spec}" +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp | jq "{spec}" title "Assert the job is created:\n" JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.foo.id') diff --git a/acceptance/bundle/empty_string_dropped/script b/acceptance/bundle/empty_string_dropped/script index 01ed00ce42f..a03c479c071 100644 --- a/acceptance/bundle/empty_string_dropped/script +++ b/acceptance/bundle/empty_string_dropped/script @@ -8,7 +8,7 @@ # Resolved config the engines see. Today every "" field is still present here; a # fix in the initialize phase would drop them, and this golden would show that. -$CLI bundle validate -o json -t direct | nostamp.py | jq .resources > out.validate.json +$CLI bundle validate -o json -t direct | nostamp | jq .resources > out.validate.json # Exclude non-create traffic: workspace file ops, telemetry (nondeterministic), and the # deployment history calls the DMS run adds. diff --git a/acceptance/bundle/environments/dependencies/script b/acceptance/bundle/environments/dependencies/script index 77c215da892..00e5fc846cc 100644 --- a/acceptance/bundle/environments/dependencies/script +++ b/acceptance/bundle/environments/dependencies/script @@ -4,11 +4,11 @@ set -euo pipefail trace $CLI bundle validate trace $CLI bundle deploy -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt | nostamp.py +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt | nostamp -trace jq -s '.[] | select(.path=="/api/2.0/pipelines")' out.requests.txt | nostamp.py +trace jq -s '.[] | select(.path=="/api/2.0/pipelines")' out.requests.txt | nostamp -trace $CLI bundle validate -o json | nostamp.py | jq '.resources.jobs.test_job.environments' -trace $CLI bundle validate -o json | nostamp.py | jq '.resources.pipelines.test_pipeline.environment' +trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs.test_job.environments' +trace $CLI bundle validate -o json | nostamp | jq '.resources.pipelines.test_pipeline.environment' rm out.requests.txt diff --git a/acceptance/bundle/escaped_refs/script b/acceptance/bundle/escaped_refs/script index 1b8986539c6..0f75085c239 100644 --- a/acceptance/bundle/escaped_refs/script +++ b/acceptance/bundle/escaped_refs/script @@ -12,7 +12,7 @@ title "Validate: escapes are still escaped in the config" trace $CLI bundle validate -o json | jq '.resources.jobs.example_ingestion_job.tasks[0].notebook_task.base_parameters' title "Plan (json)" -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Not traced: the READPLAN variant adds "--plan ...", and the point of that variant is # that the payload below is identical either way. diff --git a/acceptance/bundle/invariant/delete_idempotent/script b/acceptance/bundle/invariant/delete_idempotent/script index 7e3a0662600..50937aca2b2 100644 --- a/acceptance/bundle/invariant/delete_idempotent/script +++ b/acceptance/bundle/invariant/delete_idempotent/script @@ -12,7 +12,7 @@ STATE_PATH=/Workspace/Users/$CURRENT_USER_NAME/.bundle/$bundle_name/default/stat # Initial deploy of the resources. Only pre-plan when READPLAN=1 exercises the # saved-plan deploy path; otherwise `bundle deploy` plans on its own. if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json | nostamp.py > plan_initial.json 2>LOG.plan_initial.err + $CLI bundle plan -o json | nostamp > plan_initial.json 2>LOG.plan_initial.err cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null fi @@ -30,7 +30,7 @@ resources: {} EOF if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json | nostamp.py > plan_delete.json 2>LOG.plan_delete.err + $CLI bundle plan -o json | nostamp > plan_delete.json 2>LOG.plan_delete.err cat LOG.plan_delete.err | contains.py '!panic:' '!internal error' > /dev/null fi @@ -53,7 +53,7 @@ MSYS_NO_PATHCONV=1 $CLI workspace delete --recursive "$STATE_PATH" &> LOG.wipe_r cat LOG.wipe_remote_state | contains.py '!panic:' '!internal error' > /dev/null if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json | nostamp.py > plan_delete2.json 2>LOG.plan_delete2.err + $CLI bundle plan -o json | nostamp > plan_delete2.json 2>LOG.plan_delete2.err cat LOG.plan_delete2.err | contains.py '!panic:' '!internal error' > /dev/null fi diff --git a/acceptance/bundle/invariant/destroy_idempotent/script b/acceptance/bundle/invariant/destroy_idempotent/script index df73548bb0c..a57b06d029d 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/script +++ b/acceptance/bundle/invariant/destroy_idempotent/script @@ -12,7 +12,7 @@ ROOT_PATH=/Workspace/Users/$CURRENT_USER_NAME/.bundle/$bundle_name/default # Initial deploy of the resources. Only pre-plan when READPLAN=1 exercises the # saved-plan deploy path; otherwise `bundle deploy` plans on its own. if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json | nostamp.py > plan_initial.json 2>LOG.plan_initial.err + $CLI bundle plan -o json | nostamp > plan_initial.json 2>LOG.plan_initial.err cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null fi diff --git a/acceptance/bundle/migrate/auto-migrate-clean/script b/acceptance/bundle/migrate/auto-migrate-clean/script index 5afad5a39c5..d2779441da1 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/script +++ b/acceptance/bundle/migrate/auto-migrate-clean/script @@ -31,7 +31,7 @@ rm -f out.requests.txt title "Modify the config — plan shows the pending update" trace update_file.py databricks.yml "Test Auto-Migrate Job" "Test Auto-Migrate Job (renamed)" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan_update.direct.json +$CLI bundle plan -o json | nostamp > out.plan_update.direct.json title "Deploy picks up the update (with and without --plan)\n" $CLI bundle deploy $(readplanarg out.plan_update.direct.json) diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/script b/acceptance/bundle/migrate/auto-migrate-direct-only/script index 7f9c18cac46..3d0b91b7713 100644 --- a/acceptance/bundle/migrate/auto-migrate-direct-only/script +++ b/acceptance/bundle/migrate/auto-migrate-direct-only/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml "name: test-bundle" $'name: test-bundle\n e title "Plan skips the direct-only resource, since this run still uses terraform" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | gron.py | grep instance_pools +trace $CLI bundle plan -o json | nostamp | gron.py | grep instance_pools title "Deploy migrates the state, skipping the direct-only resource" trace $CLI bundle deploy diff --git a/acceptance/bundle/migrate/basic/script b/acceptance/bundle/migrate/basic/script index 64c42e6d641..f1aa9003ed2 100755 --- a/acceptance/bundle/migrate/basic/script +++ b/acceptance/bundle/migrate/basic/script @@ -43,7 +43,7 @@ rm out.requests.txt title "Update databricks.yml, run plan & update" trace update_file.py databricks.yml Migration Migrated trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_update.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp > out.plan_update.json trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy title "Different target, still on terraform" diff --git a/acceptance/bundle/migrate/dashboards/script b/acceptance/bundle/migrate/dashboards/script index b2c9639a0fd..efcac87e0fc 100755 --- a/acceptance/bundle/migrate/dashboards/script +++ b/acceptance/bundle/migrate/dashboards/script @@ -15,7 +15,7 @@ rm out.requests.txt # Badness: known issue with dashboards, will show up as 'recreate' see https://github.com/databricks/cli/pull/3966 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 unchanged" trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan | contains.py "1 unchanged" -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_after_migrate.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp > out.plan_after_migrate.json trace print_requests.py --get //dashboards | contains.py 'engine/direct' > /dev/null trace DATABRICKS_BUNDLE_ENGINE="" $CLI bundle deploy --auto-approve @@ -32,7 +32,7 @@ rm out.requests.txt #title "Update databricks.yml, run plan & update" #trace update_file.py databricks.yml dashboard DASHBOARD #trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -#trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_update.json +#trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp > out.plan_update.json #trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy #rm out.requests.txt diff --git a/acceptance/bundle/migrate/default-python/script b/acceptance/bundle/migrate/default-python/script index 6e05ab85f7f..f3a4ca0c37c 100755 --- a/acceptance/bundle/migrate/default-python/script +++ b/acceptance/bundle/migrate/default-python/script @@ -11,7 +11,7 @@ trace print_state.py > ../out.state_after_migration.json trace jq '.. | .libraries? | select(.)' ../out.state_after_migration.json trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json | nostamp.py > ../out.plan_after_migration.json +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json | nostamp > ../out.plan_after_migration.json trace jq '.plan[] | .changes' ../out.plan_after_migration.json # Badness: contains changes for whl and num_workers trace DATABRICKS_BUNDLE_ENGINE="" $CLI bundle deploy @@ -21,5 +21,5 @@ rm ../out.state_after_deploy.json title "Extra plan: should have no drift" trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json | nostamp.py > ../out.plan_after_deploy.json +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json | nostamp > ../out.plan_after_deploy.json trace jq '.plan[] | .changes' ../out.plan_after_migration.json # Badness: contains changes for whl diff --git a/acceptance/bundle/migrate/runas/script b/acceptance/bundle/migrate/runas/script index 4affba0a373..28af4373735 100644 --- a/acceptance/bundle/migrate/runas/script +++ b/acceptance/bundle/migrate/runas/script @@ -23,4 +23,4 @@ trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deployment migrate trace print_state.py > out.new_state.json trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp > out.plan.json diff --git a/acceptance/bundle/resource_deps/escaped_ref/script b/acceptance/bundle/resource_deps/escaped_ref/script index 8c906e071d0..e2c102d942a 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/script +++ b/acceptance/bundle/resource_deps/escaped_ref/script @@ -6,7 +6,7 @@ trap cleanup EXIT title "depends_on: only the real reference creates an edge" $CLI bundle plan -o json > plan.json -trace jq '.plan | map_values(.depends_on)' plan.json | nostamp.py +trace jq '.plan | map_values(.depends_on)' plan.json | nostamp # Not traced: the READPLAN variant adds "--plan ...", and the point of that variant is # that the payload below is identical either way. diff --git a/acceptance/bundle/resource_deps/grant_ref/script b/acceptance/bundle/resource_deps/grant_ref/script index ad897f76987..458ea3c4932 100644 --- a/acceptance/bundle/resource_deps/grant_ref/script +++ b/acceptance/bundle/resource_deps/grant_ref/script @@ -1,4 +1,4 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resource_deps/id_chain/script b/acceptance/bundle/resource_deps/id_chain/script index eada43ba1c9..bcd103ac2a5 100644 --- a/acceptance/bundle/resource_deps/id_chain/script +++ b/acceptance/bundle/resource_deps/id_chain/script @@ -16,7 +16,7 @@ print_requests_short() { trace $CLI bundle plan trace print_requests -$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests trace $CLI bundle deploy @@ -25,7 +25,7 @@ trace print_requests_short trace update_file.py databricks.yml aa_desc aa_new_desc trace update_file.py databricks.yml prefix new_prefix -$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests trace $CLI bundle plan @@ -34,5 +34,5 @@ trace print_requests trace $CLI bundle deploy trace print_requests_short -$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests diff --git a/acceptance/bundle/resource_deps/id_star/script b/acceptance/bundle/resource_deps/id_star/script index 46366e7706a..bd85cfc8ae6 100644 --- a/acceptance/bundle/resource_deps/id_star/script +++ b/acceptance/bundle/resource_deps/id_star/script @@ -1 +1 @@ -$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resource_deps/job_id/script b/acceptance/bundle/resource_deps/job_id/script index 96d8aed011f..f8d30917d1c 100644 --- a/acceptance/bundle/resource_deps/job_id/script +++ b/acceptance/bundle/resource_deps/job_id/script @@ -1,6 +1,6 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs @@ -8,7 +8,7 @@ foo_id=`read_id.py foo` bar_id=`read_id.py bar` cp empty.yml databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/script b/acceptance/bundle/resource_deps/job_id_delete_bar/script index 8fd0af69437..03b9a087810 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/script +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/script @@ -8,7 +8,7 @@ bar_id=`read_id.py bar` title "Delete bar, keep foo (foo config updated to not depend on bar)" cp only_foo.yml databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy # Sort requests because foo's update and bar's delete have no dependency edge # (foo's dependency on bar was removed in only_foo.yml), so execution order is non-deterministic. diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/script b/acceptance/bundle/resource_deps/job_id_delete_foo/script index 30e1194c4cd..f5efbd22609 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/script +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/script @@ -8,7 +8,7 @@ bar_id=`read_id.py bar` title "Delete foo, keep bar" cp only_bar.yml databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/jobs_update/script b/acceptance/bundle/resource_deps/jobs_update/script index b1fc7332355..f806f2db846 100644 --- a/acceptance/bundle/resource_deps/jobs_update/script +++ b/acceptance/bundle/resource_deps/jobs_update/script @@ -21,8 +21,8 @@ trace $CLI bundle plan title "Fetch job ID and verify remote state" # Badness: output should not be different per engine; investigate if it's the same on cloud -trace $CLI jobs get $foo_id | nostamp.py > out.get_foo.$DATABRICKS_BUNDLE_ENGINE.json -trace $CLI jobs get $bar_id | nostamp.py +trace $CLI jobs get $foo_id | nostamp > out.get_foo.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI jobs get $bar_id | nostamp rm out.requests.txt trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/script b/acceptance/bundle/resource_deps/jobs_update_remote/script index 89a3fc4c36f..d456ed8b023 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/script +++ b/acceptance/bundle/resource_deps/jobs_update_remote/script @@ -1,5 +1,5 @@ echo "*" > .gitignore -$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs @@ -13,7 +13,7 @@ bar_id=`read_id.py bar` title "Update trigger.periodic.unit remotely and re-deploy; jobs.bar is unchanged" trace envsubst < job_update.json > tmp.json && mv tmp.json job_update.json trace $CLI jobs reset --json @job_update.json -$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle destroy --auto-approve trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/missing_string_field/script b/acceptance/bundle/resource_deps/missing_string_field/script index d96e2f30b0a..c526e980444 100644 --- a/acceptance/bundle/resource_deps/missing_string_field/script +++ b/acceptance/bundle/resource_deps/missing_string_field/script @@ -1,4 +1,4 @@ -trace $CLI bundle validate -o json | nostamp.py | jq .resources +trace $CLI bundle validate -o json | nostamp | jq .resources errcode $CLI bundle plan trace print_requests.py --nostamp //pipeline diff --git a/acceptance/bundle/resource_deps/permission_ref/script b/acceptance/bundle/resource_deps/permission_ref/script index ad897f76987..458ea3c4932 100644 --- a/acceptance/bundle/resource_deps/permission_ref/script +++ b/acceptance/bundle/resource_deps/permission_ref/script @@ -1,4 +1,4 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/script b/acceptance/bundle/resource_deps/pipelines_recreate/script index b1a6741f081..691e24279b2 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/script +++ b/acceptance/bundle/resource_deps/pipelines_recreate/script @@ -1,7 +1,7 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' > out.create.requests.json @@ -14,7 +14,7 @@ trace $CLI bundle plan # empty title "Update storage, triggering recreate for pipeline; this means updating downstream deps" trace update_file.py databricks.yml "storage: dbfs:/my-storage" "storage: dbfs:/my-new-storage" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --auto-approve trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -23,13 +23,13 @@ foo_id_2=`read_id.py foo` title "Fetch resource IDs and verify remote state" trace musterr $CLI pipelines get $foo_id -trace $CLI pipelines get $foo_id_2 | nostamp.py -trace $CLI jobs get $bar_id | nostamp.py| jq 'del(.settings.run_as)' +trace $CLI pipelines get $foo_id_2 | nostamp +trace $CLI jobs get $bar_id | nostamp| jq 'del(.settings.run_as)' rm out.requests.txt title "Follow up plan & deploy do nothing" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' diff --git a/acceptance/bundle/resource_deps/remote_pipeline/script b/acceptance/bundle/resource_deps/remote_pipeline/script index 4df5a844e73..b8683429aac 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/script +++ b/acceptance/bundle/resource_deps/remote_pipeline/script @@ -3,8 +3,8 @@ print_requests() { rm out.requests.txt } -$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests -$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests diff --git a/acceptance/bundle/resource_deps/resources_var/script b/acceptance/bundle/resource_deps/resources_var/script index c04b243240b..aa48ddcde79 100644 --- a/acceptance/bundle/resource_deps/resources_var/script +++ b/acceptance/bundle/resource_deps/resources_var/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -t dev -o json | jq .resources -$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy -t dev &> out.deploy.txt trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt trace print_telemetry_bool_values | grep -v direct_drymigrate diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script index 2fb906b4945..b34349c6c9c 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -t dev -o json | jq .resources -$CLI bundle plan -o json -t dev | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t dev trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resource_deps/tf_path_renames/script b/acceptance/bundle/resource_deps/tf_path_renames/script index dc491ff0f5e..8b082e9216a 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/script +++ b/acceptance/bundle/resource_deps/tf_path_renames/script @@ -1,3 +1,3 @@ -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs | jq -rs '[.[].body.name] | sort[]' diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/script b/acceptance/bundle/resource_deps/volume_path_contains_id/script index 21d91ccd863..5d561d60798 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/script +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/script @@ -5,7 +5,7 @@ cleanup() { trace $CLI bundle validate -o json | jq .resources trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trap cleanup EXIT trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script index adeee195109..5b7a2e956b0 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/script +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -1,10 +1,10 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle validate -o json | nostamp.py | jq '.resources.jobs, .resources.volumes' +trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs, .resources.volumes' # The job's data_path default references the volume's computed volume_path. # Record the JSON plan per engine to confirm the reference is resolved into the # planned new_state rather than left as ${resources...}. -trace errcode $CLI bundle plan -o json | nostamp.py &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace errcode $CLI bundle plan -o json | nostamp &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Deploy and inspect the create-job request: the parameter default sent to # the Jobs API must be the interpolated volume_path. diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/script b/acceptance/bundle/resources/apps/config-drift-stopped/script index 50e046d9442..12903f604fd 100644 --- a/acceptance/bundle/resources/apps/config-drift-stopped/script +++ b/acceptance/bundle/resources/apps/config-drift-stopped/script @@ -10,10 +10,10 @@ trace $CLI bundle deploy title "Change config while running: drift is detected (active deployment present)" trace update_file.py databricks.yml original_value changed_value -trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' +trace $CLI bundle plan -o json | nostamp | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' title "Stop the app: the backend clears the active deployment" trace $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state' title "Same config change is now skipped: no active deployment to compare against" -trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' +trace $CLI bundle plan -o json | nostamp | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' diff --git a/acceptance/bundle/resources/apps/config-drift/script b/acceptance/bundle/resources/apps/config-drift/script index 23154a3fdf9..482cf1512b0 100644 --- a/acceptance/bundle/resources/apps/config-drift/script +++ b/acceptance/bundle/resources/apps/config-drift/script @@ -9,7 +9,7 @@ trap cleanup EXIT trace $CLI bundle deploy title "Verify no drift after deploy" -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' > out.plan.direct.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' > out.plan.direct.json SOURCE_CODE_PATH=$(trace $CLI apps get $UNIQUE_NAME --output json | jq -r '.active_deployment.source_code_path') @@ -27,13 +27,13 @@ $CLI apps deploy $UNIQUE_NAME --no-wait --json '{ title "Plan should detect config drift" trace $CLI bundle plan # Skip entries with action "skip" -$CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json +$CLI bundle plan -o json | nostamp | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json title "Redeploy to fix drift" trace $CLI bundle deploy title "Verify no drift after fix" -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json # TODO: add test for git_source drift when git_source is supported in the Deploy API # Currently it fails with the error: Git source reference is required diff --git a/acceptance/bundle/resources/apps/config-no-deployment/script b/acceptance/bundle/resources/apps/config-no-deployment/script index 3b7e38e3493..503135c881f 100644 --- a/acceptance/bundle/resources/apps/config-no-deployment/script +++ b/acceptance/bundle/resources/apps/config-no-deployment/script @@ -7,4 +7,4 @@ title "Plan is a no-op: config/source_code_path drift is skipped while the app h trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "Both deploy-only fields are skipped with reason \"no active deployment\"" -trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | {config, source_code_path}' +trace $CLI bundle plan -o json | nostamp | jq '.plan[].changes | {config, source_code_path}' diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/script b/acceptance/bundle/resources/apps/git-source-no-deployment/script index 28e12809b2e..9a8c8446eb1 100644 --- a/acceptance/bundle/resources/apps/git-source-no-deployment/script +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/script @@ -7,4 +7,4 @@ title "Plan is a no-op: git_source drift is skipped while the app has no active trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "git_source is skipped with reason \"no active deployment\"" -trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | {git_source}' +trace $CLI bundle plan -o json | nostamp | jq '.plan[].changes | {git_source}' diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script index 8bdbb4dfd9c..dd08f33fff5 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script @@ -79,6 +79,6 @@ trace $CLI bundle deploy trace print_app_requests title "(started omitted, app running) -> bundle plan shows no drift" -$CLI bundle plan -o json | nostamp.py > LOG.planjson +$CLI bundle plan -o json | nostamp > LOG.planjson verify_no_drift.py LOG.planjson echo "Plan: no drift detected" diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/script b/acceptance/bundle/resources/apps/readplan-lifecycle/script index cdf31d69511..5f2e714223d 100644 --- a/acceptance/bundle/resources/apps/readplan-lifecycle/script +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/script @@ -10,6 +10,6 @@ trace print_requests.py //apps/myapp/start title "Change description and deploy from plan: no extra start" trace update_file.py databricks.yml "description: initial" "description: updated" -$CLI bundle plan -o json | nostamp.py > tmp.plan.json +$CLI bundle plan -o json | nostamp > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) trace print_requests.py //apps/myapp/start diff --git a/acceptance/bundle/resources/apps/uc-securable-drift/script b/acceptance/bundle/resources/apps/uc-securable-drift/script index 7f3022a2bb1..fbf6a985213 100644 --- a/acceptance/bundle/resources/apps/uc-securable-drift/script +++ b/acceptance/bundle/resources/apps/uc-securable-drift/script @@ -10,4 +10,4 @@ title "Re-plan is clean: the TABLE securable's server-computed securable_kind is trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "securable_kind is classified output-only (skip); the VOLUME securable has none" -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes // {} | with_entries(select(.key | test("uc_securable")))' +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.apps.myapp".changes // {} | with_entries(select(.key | test("uc_securable")))' diff --git a/acceptance/bundle/resources/catalogs/comment_out_of_band/script b/acceptance/bundle/resources/catalogs/comment_out_of_band/script index c0f21c30082..a4819d98640 100644 --- a/acceptance/bundle/resources/catalogs/comment_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/catalogs/$CATALOG" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the catalog" -trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/script b/acceptance/bundle/resources/catalogs/drift/managed_properties/script index 25664d4eefd..fc5b46ebe68 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/script +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite UC auto-populating managed properties" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only properties map is skipped as a backend default (confirms the matched rule)" -trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes' title "Redeploy is a no-op (no UpdateCatalog call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index 00614a0ec10..ae75fc4170d 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -10,11 +10,11 @@ trap cleanup EXIT # resize, terraform update), so the per-resource line would diverge. The Resources: # summary agrees, so it is kept rather than using -qq. $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp.py > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q title "Cluster should exist with num_workers after bundle deployment:\n" -CLUSTER_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.clusters.test_cluster.id') +CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' @@ -26,7 +26,7 @@ update_file.py databricks.yml " num_workers: 2" " autoscale: min_workers: 2 max_workers: 4" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -38,7 +38,7 @@ title "Changing autoscale should call update API on stopped cluster\n" update_file.py databricks.yml "min_workers: 2" "min_workers: 3" update_file.py databricks.yml "max_workers: 4" "max_workers: 5" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -53,7 +53,7 @@ title "Changing autoscale should call resize API on running cluster\n" update_file.py databricks.yml "min_workers: 3" "min_workers: 4" update_file.py databricks.yml "max_workers: 5" "max_workers: 6" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt @@ -66,7 +66,7 @@ update_file.py databricks.yml " autoscale: min_workers: 4 max_workers: 6" " num_workers: 3" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index 7643752ef88..5e2cf3022ab 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -10,18 +10,18 @@ trap cleanup EXIT # resize, terraform update), so the per-resource line would diverge. The Resources: # summary agrees, so it is kept rather than using -qq. $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp.py > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q title "Cluster should exist after bundle deployment:\n" -CLUSTER_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.clusters.test_cluster.id') +CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' title "Changing num_workers should call update API on stopped cluster\n" update_file.py databricks.yml "num_workers: 2" "num_workers: 3" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -35,7 +35,7 @@ $CLI clusters start "${CLUSTER_ID}" title "Changing num_workers should call resize API on running cluster\n" update_file.py databricks.yml "num_workers: 3" "num_workers: 4" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt @@ -47,7 +47,7 @@ title "Changing num_workers and spark_conf should call update API\n" update_file.py databricks.yml "num_workers: 4" "num_workers: 5" update_file.py databricks.yml '"spark.executor.memory": "2g"' '"spark.executor.memory": "4g"' $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/script b/acceptance/bundle/resources/dashboards/change-serialized-inline/script index 116901bf8bd..432cab045c8 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/script +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/script @@ -25,8 +25,8 @@ trace $CLI bundle plan | contains.py "update" "!recreate" # Save raw plan (with DMS headers) to tmp file for deployment trace $CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json -# Generate golden file by applying nostamp.py for comparison (discard output) -cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# Generate golden file by applying nostamp for comparison (discard output) +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json title "deploy the change\n" $CLI bundle deploy $(readplanarg plan_raw.$DATABRICKS_BUNDLE_ENGINE.json) diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script index 0016ebe8227..8d9c7610c3b 100755 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script @@ -29,9 +29,9 @@ echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path}' echo "$DASHBOARD" | jq '.serialized_dashboard | fromjson | .datasets[] | {catalog, schema}' # Verify that there is no drift right after deploy. -# Save raw plan (with DMS headers), then apply nostamp.py for golden file +# Save raw plan (with DMS headers), then apply nostamp for golden file trace $CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json -cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Modify the direct plan to replace "serialized_dashboard" with a fixture. # It is normalized on the backend so we cannot compare reliably across local and cloud. diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script index 4746b59f8cf..78de1b2cc4c 100755 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script @@ -25,9 +25,9 @@ trace $CLI lakeview trash $DASHBOARD_ID trace $CLI bundle plan -# Save raw plan (with DMS headers), then apply nostamp.py for golden file +# Save raw plan (with DMS headers), then apply nostamp for golden file trace $CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json -cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Run destroy - should succeed even though dashboard is already trashed. # -qq drops the summary: terraform refreshes the trashed dashboard away and reports diff --git a/acceptance/bundle/resources/dashboards/detect-change/script b/acceptance/bundle/resources/dashboards/detect-change/script index 4c5ac9d4796..16b1b2b773b 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/script +++ b/acceptance/bundle/resources/dashboards/detect-change/script @@ -38,9 +38,9 @@ add_repl "$(echo "$UPDATE_RESP" | jq -r '.etag')" ETAG_2 title "Try to redeploy the bundle and confirm that the out of band modification is detected:" trace $CLI bundle plan -# Save raw plan (with DMS headers), then apply nostamp.py for golden file +# Save raw plan (with DMS headers), then apply nostamp for golden file trace $CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json -cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy title "Redeploy the bundle with the --force flag and confirm that the out of band modification is ignored:" diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script index 4c455c0c3f5..c6f930f95dd 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script @@ -16,7 +16,7 @@ ETAG_1=$(echo "$DASHBOARD" | jq -r '.etag') add_repl "$ETAG_1" ETAG_1 echo "$DASHBOARD" | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' -trace $CLI bundle plan -o json | nostamp.py | gron.py | grep -E "etag|published" +trace $CLI bundle plan -o json | nostamp | gron.py | grep -E "etag|published" rm out.requests.txt # Inject a single publish failure for the update below. @@ -34,7 +34,7 @@ ETAG_2=$(retry --until-not "$ETAG_1" $CLI lakeview get $DASHBOARD_ID | jq -r '.e add_repl "$ETAG_2" ETAG_2 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' -trace $CLI bundle plan -o json | nostamp.py | gron.py | grep -E "etag|published" +trace $CLI bundle plan -o json | nostamp | gron.py | grep -E "etag|published" # Bug: re-running deploy fails with "modified remotely" because the stored etag # (pre-PATCH) no longer matches the remote etag (bumped by the PATCH above). diff --git a/acceptance/bundle/resources/dashboards/republish-after-draft-update/script b/acceptance/bundle/resources/dashboards/republish-after-draft-update/script index cd278bab258..b6f0d91fa6c 100644 --- a/acceptance/bundle/resources/dashboards/republish-after-draft-update/script +++ b/acceptance/bundle/resources/dashboards/republish-after-draft-update/script @@ -16,7 +16,7 @@ DASHBOARD_ID=$(read_id.py dashboard1) # A clean re-plan is a no-op: the published content matches the draft. title "Plan right after deploy -- published is current, no change:" -trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.dashboards.dashboard1"].changes.published' +trace $CLI bundle plan -o json | nostamp | jq '.plan["resources.dashboards.dashboard1"].changes.published' # Make an out-of-band DRAFT update without republishing. This advances the draft's # update_time past the published revision_create_time, so the published content is now @@ -28,4 +28,4 @@ DASHBOARD_JSON="{\"serialized_dashboard\": \"{}\", \"warehouse_id\": \"$TEST_DEF $CLI lakeview update "$DASHBOARD_ID" --json "${DASHBOARD_JSON}" > /dev/null title "Plan after out-of-band draft update -- a republish is owed:" -trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.dashboards.dashboard1"].changes.published' +trace $CLI bundle plan -o json | nostamp | jq '.plan["resources.dashboards.dashboard1"].changes.published' diff --git a/acceptance/bundle/resources/dashboards/simple/script b/acceptance/bundle/resources/dashboards/simple/script index 867831720b8..8f7d1a2e8ec 100644 --- a/acceptance/bundle/resources/dashboards/simple/script +++ b/acceptance/bundle/resources/dashboards/simple/script @@ -24,6 +24,6 @@ echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path, serialized_dashboar # Verify that there is no drift right after deploy. We use |= to update the serialized_dashboard field in place to account # for formatting that the cloud server applies to the serialized_dashboard field. -# Save raw plan (with DMS headers), then apply nostamp.py for golden file +# Save raw plan (with DMS headers), then apply nostamp for golden file trace $CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json -cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script index 7b2a9604039..4b816672c5b 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script @@ -6,9 +6,9 @@ cleanup() { } trap cleanup EXIT -# Save raw plan (with DMS headers), then apply nostamp.py for golden file +# Save raw plan (with DMS headers), then apply nostamp for golden file trace $CLI bundle plan -o json > plan_raw_initial.$DATABRICKS_BUNDLE_ENGINE.json -cat plan_raw_initial.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan_initial.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw_initial.$DATABRICKS_BUNDLE_ENGINE.json | nostamp > out.plan_initial.$DATABRICKS_BUNDLE_ENGINE.json # Deploy the dashboard trace $CLI bundle deploy -qq @@ -27,9 +27,9 @@ trace $CLI lakeview unpublish $DASHBOARD_ID # Direct: shows "update" because Published field changes from false to true trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -# Save raw plan (with DMS headers), then apply nostamp.py for golden file +# Save raw plan (with DMS headers), then apply nostamp for golden file $CLI bundle plan -o json > plan_raw.$DATABRICKS_BUNDLE_ENGINE.json -cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +cat plan_raw.$DATABRICKS_BUNDLE_ENGINE.json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json json_in_json_normalize.py out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -qq diff --git a/acceptance/bundle/resources/genie_spaces/inline/script b/acceptance/bundle/resources/genie_spaces/inline/script index 9e5892a1416..a342e6dc7de 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/script +++ b/acceptance/bundle/resources/genie_spaces/inline/script @@ -15,4 +15,4 @@ add_repl "$GENIE_SPACE_ID" GENIE_SPACE_ID # Without normalization the inline serialized_space leaves a map in the # config struct while state holds a string, and structdiff reports false # drift on every plan. -trace $CLI bundle plan -o json | nostamp.py > out.plan.json +trace $CLI bundle plan -o json | nostamp > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/version_migration/script b/acceptance/bundle/resources/genie_spaces/version_migration/script index 5615bbb7034..2c7e0d56aae 100644 --- a/acceptance/bundle/resources/genie_spaces/version_migration/script +++ b/acceptance/bundle/resources/genie_spaces/version_migration/script @@ -19,7 +19,7 @@ trace $CLI bundle plan # (and the etag guarding it) is version 2. The serialized_space change is # recorded but skipped via the etag_based rule. title "serialized_space: local version 1, remote version 2, skipped (etag_based)" -$CLI bundle plan -o json | nostamp.py \ +$CLI bundle plan -o json | nostamp \ | jq '.plan["resources.genie_spaces.foo"].changes.serialized_space | {action, reason, local_version: (.new | fromjson | .version), diff --git a/acceptance/bundle/resources/grants/catalogs/script b/acceptance/bundle/resources/grants/catalogs/script index aa47ee0ce94..5dd1ab465fb 100755 --- a/acceptance/bundle/resources/grants/catalogs/script +++ b/acceptance/bundle/resources/grants/catalogs/script @@ -1,7 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -13,7 +13,7 @@ trace $CLI grants get catalog catalog_grants_$UNIQUE_NAME | jq --sort-keys update_file.py databricks.yml CREATE_SCHEMA USE_SCHEMA -trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/registered_models/script b/acceptance/bundle/resources/grants/registered_models/script index 43ec2d3b87a..88cc72a1017 100644 --- a/acceptance/bundle/resources/grants/registered_models/script +++ b/acceptance/bundle/resources/grants/registered_models/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/script b/acceptance/bundle/resources/grants/schemas/change_privilege/script index ec907ef8887..d79c9559a79 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/script +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/script @@ -1,7 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -13,7 +13,7 @@ trace $CLI grants get schema main.schema_grants_$UNIQUE_NAME | jq --sort-keys update_file.py databricks.yml USE_SCHEMA APPLY_TAG -trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/script b/acceptance/bundle/resources/grants/schemas/duplicate_principals/script index 1056a114bfb..17130a374c7 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/script +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/script @@ -11,4 +11,4 @@ trace $CLI bundle deploy print_requests.py --get //permissions --keep > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt # effective_predictive_optimization_flag is inherited from the metastore and backend-controlled; drop it so the test does not depend on metastore settings. # updated_at/updated_by reflect UC's post-create system write that populates managed defaults, so on managed-defaults clouds they show a system principal rather than the creator; drop them. -trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script index fe3b395156d..a46332ca187 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script @@ -12,7 +12,7 @@ trap cleanup EXIT trace $CLI bundle deploy trace $CLI grants update schema "$SCHEMA_FULL_NAME" --json @update.json > /dev/null $CLI grants get schema "$SCHEMA_FULL_NAME" | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com' > /dev/null -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # remote_state.__embed__ order is non-deterministic; extract and sort separately into per-engine file jq '.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | gron.py --noindex | sort_lines.py --repl > out.embed.$DATABRICKS_BUNDLE_ENGINE.txt tmp=$(mktemp) @@ -21,4 +21,4 @@ tmp=$(mktemp) # updated_at/updated_by reflect UC's post-create system write that populates managed defaults, so on managed-defaults clouds they show a system principal rather than the creator; drop them. jq 'del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__)' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > "$tmp" && mv "$tmp" out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/volumes/script b/acceptance/bundle/resources/grants/volumes/script index 9b8854c8062..9eb77b83730 100644 --- a/acceptance/bundle/resources/grants/volumes/script +++ b/acceptance/bundle/resources/grants/volumes/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -10,7 +10,7 @@ trace $CLI grants get volume main.schema_grants_$UNIQUE_NAME.volume_name | jq -- update_file.py databricks.yml WRITE_VOLUME MANAGE # different on cloud vs local due to remote state -trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by volume_id > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by volume_id > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/job_runs/failed_run/script b/acceptance/bundle/resources/job_runs/failed_run/script index 2c1ee0fa812..78d1cc07b42 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/script +++ b/acceptance/bundle/resources/job_runs/failed_run/script @@ -15,7 +15,7 @@ musterr trace $CLI bundle deploy # The framework saves the run id before WaitAfterCreate, so FAILED vs SUCCESS is drift. title "the failed run is recorded, and not having succeeded is drift" trace read_id.py my_run -trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.job_runs.my_run"]' +trace $CLI bundle plan -o json | nostamp | jq '.plan["resources.job_runs.my_run"]' # Name the job too, so the run-now bodies below say which number it is. read_id.py my_job > /dev/null diff --git a/acceptance/bundle/resources/job_runs/redeploy/script b/acceptance/bundle/resources/job_runs/redeploy/script index 2c969f0a969..95e280bd8aa 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/script +++ b/acceptance/bundle/resources/job_runs/redeploy/script @@ -17,7 +17,7 @@ trace print_requests.py //jobs/run-now title "change the run configuration and redeploy" trace update_file.py databricks.yml "env: dev" "env: prod" -trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.job_runs.my_run"]' +trace $CLI bundle plan -o json | nostamp | jq '.plan["resources.job_runs.my_run"]' trace $CLI bundle deploy read_id.py my_run > /dev/null trace $CLI bundle summary diff --git a/acceptance/bundle/resources/job_runs/wait/script b/acceptance/bundle/resources/job_runs/wait/script index 3a68150df3b..79e9547c67b 100644 --- a/acceptance/bundle/resources/job_runs/wait/script +++ b/acceptance/bundle/resources/job_runs/wait/script @@ -15,5 +15,5 @@ downstream_id=$(read_id.py downstream_job) trace $CLI jobs get $downstream_id -o json | jq -r ".settings.tags.run_result" title "nothing the run resolved is drift, so a redeploy starts no new run" -trace $CLI bundle plan -o json | nostamp.py | jq -r '.plan | to_entries[] | "\(.key) \(.value.action)"' +trace $CLI bundle plan -o json | nostamp | jq -r '.plan | to_entries[] | "\(.key) \(.value.action)"' trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script index 00188cda82d..e6915a6acd3 100644 --- a/acceptance/bundle/resources/jobs/big_id/script +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -o json | jq .resources > out.validate.json -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan.direct.json) trace print_requests.py //jobs --nostamp print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index f1bf01f596d..1622b3e11cb 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy cp empty.yml databricks.yml -$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/jobs/delete_task/script b/acceptance/bundle/resources/jobs/delete_task/script index 89cf79be7e3..94d69088c8a 100644 --- a/acceptance/bundle/resources/jobs/delete_task/script +++ b/acceptance/bundle/resources/jobs/delete_task/script @@ -1,6 +1,6 @@ -$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) grep -v TO_DELETE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml -$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/script b/acceptance/bundle/resources/jobs/on_failure_empty_slice/script index 55c9b9bbf82..6f0606bc346 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/script +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/script @@ -1,3 +1,3 @@ trace $CLI bundle plan trace $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.refresh_usage_logs".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.refresh_usage_logs".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/script b/acceptance/bundle/resources/jobs/remote_add_tag/script index b2b2fefd0ba..3a651737979 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/script +++ b/acceptance/bundle/resources/jobs/remote_add_tag/script @@ -8,4 +8,4 @@ r["tags"]["new_tag"] = "new_value" EOF $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/script b/acceptance/bundle/resources/jobs/remote_delete/deploy/script index 8471d53c0b8..658fca8f399 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/script +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/script @@ -3,5 +3,5 @@ trace $CLI bundle deploy job_id=`$CLI bundle summary -o json | jq -r .resources.jobs.foo.id` trace $CLI jobs delete $job_id trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan.direct.json) diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index b6ed653c0ef..9ca6a0655b6 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -13,7 +13,7 @@ r["max_concurrent_runs"] = 2 EOF trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt # XXX READPLAN diff --git a/acceptance/bundle/resources/jobs/tags_empty_map/script b/acceptance/bundle/resources/jobs/tags_empty_map/script index d3dd04214cf..4a232723bd0 100644 --- a/acceptance/bundle/resources/jobs/tags_empty_map/script +++ b/acceptance/bundle/resources/jobs/tags_empty_map/script @@ -1,3 +1,3 @@ trace $CLI bundle plan trace $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.test_job".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.test_job".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/update/script b/acceptance/bundle/resources/jobs/update/script index 7a90cf02255..dbc80a88b72 100644 --- a/acceptance/bundle/resources/jobs/update/script +++ b/acceptance/bundle/resources/jobs/update/script @@ -1,12 +1,12 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) trace print_requests.py //jobs --nostamp > out.create.requests.json print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_skip.direct.json) trace print_requests.py //jobs --nostamp @@ -14,7 +14,7 @@ trace print_requests.py //jobs --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) trace print_requests.py //jobs --nostamp | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json diff --git a/acceptance/bundle/resources/jobs/update_single_node/script b/acceptance/bundle/resources/jobs/update_single_node/script index bfeb03571f8..4deefe32bbc 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/script +++ b/acceptance/bundle/resources/jobs/update_single_node/script @@ -1,15 +1,15 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py //jobs > out.create.requests.txt --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --nostamp diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index b3dbd3b782c..c0257db6330 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -14,6 +14,6 @@ EOF # The reordered remote must not produce a phantom diff: on_* lists are diffed by id. trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/script b/acceptance/bundle/resources/model_serving_endpoints/basic/script index 48b1f935b0c..18f67a7f5b1 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/script +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/script @@ -14,7 +14,7 @@ trap cleanup EXIT # bundle/direct/dresources/permissions.go), so sorting keeps the plan golden # stable across runs. Same approach as secret_scopes/basic and the requests # sorting below. -trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays __embed__ > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp | gron.py --sort-arrays __embed__ > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq # Sort access_control_list: the terraform engine stores it as a set whose order # depends on the (unmasked) principal value, so it differs between the local fake @@ -33,7 +33,7 @@ echo "$get_output" | jq '{name, creator}' export ENDPOINT_NAME="test-endpoint-$UNIQUE_NAME-2" envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays __embed__ > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp | gron.py --sort-arrays __embed__ > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq print_requests.py //serving-endpoints | gron.py --sort-arrays access_control_list > out.second-requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script index 7faa6fd31f3..81c14d8d89f 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script @@ -7,7 +7,7 @@ title "Configure telemetry out of band" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.0/serving-endpoints/test-endpoint/telemetry-config" --json '{"telemetry_config": {"table_names": {"logs_table": "main.default.other_logs"}}}' > /dev/null title "Remote-only telemetry_config is not drift" -trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' title "Deploy does not remove it" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script index 03058a0cf1b..4a526e23325 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite write-only secret and burst_scaling_enabled not r trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The write-only fields are skipped as input_only (confirms the matched rules)" -trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes' title "Redeploy is a no-op (no config/ai-gateway/tags update calls)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script index a37c37a2df5..7b4a3bcc755 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.catalog_name' trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other_catalog" -trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script index c27e1968877..b88559e2af8 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.name' trace update_file.py databricks.yml "name: test-endpoint-$UNIQUE_NAME" "name: test-endpoint-2-$UNIQUE_NAME" -trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script index db80789d106..53b4cb2d5c9 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') @@ -16,7 +16,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '{route_optimized, id}' > print_requests.py //serving-endpoints > out.first-requests.$DATABRICKS_BUNDLE_ENGINE.json trace update_file.py databricks.yml "route_optimized: false" "route_optimized: true" -trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script index cc76c4428c6..00e8b6083a1 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.schema_name' trace update_file.py databricks.yml "schema_name: default" "schema_name: other_schema" -trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script index 079bc9a33c9..5e50980c97b 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.table_name_prefix' trace update_file.py databricks.yml "table_name_prefix: my_table" "table_name_prefix: other_table" -trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script index 8f2e49038ed..42b3c65b25a 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script @@ -8,7 +8,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script index 3f942f4d306..a5ff92c4a71 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.ai_gateway' trace update_file.py databricks.yml 'catalog_name: "first-inference-catalog"' 'catalog_name: "second-inference-catalog"' -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script index c9b81cd0d6d..cc063273592 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script @@ -17,7 +17,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config' trace update_file.py databricks.yml "value: my-team-one" "value: my-team-two" trace update_file.py databricks.yml 'catalog_name: "first-inference-catalog"' 'catalog_name: "second-inference-catalog"' -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/script b/acceptance/bundle/resources/model_serving_endpoints/update/config/script index c4656a110f8..b542e1a02b1 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config' trace update_file.py databricks.yml "name: gpt-4o-mini" "name: gpt-5o-mini" -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script index f049becec06..7d1c53fb189 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script @@ -16,7 +16,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.email_notifications' # Update email_notifications - change the email address trace update_file.py databricks.yml "user1@example.com" "user2@example.com" -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "There a bug in TF where it does not actually update the email notifications for a serving endpoint." diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/script b/acceptance/bundle/resources/model_serving_endpoints/update/tags/script index 01c0e26eedd..3369736ce5a 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.tags' trace update_file.py databricks.yml "value: my-team-one" "value: my-team-two" -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/permissions/dashboards/create/script b/acceptance/bundle/resources/permissions/dashboards/create/script index 2d578386e6a..2992e2c4e12 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/script +++ b/acceptance/bundle/resources/permissions/dashboards/create/script @@ -7,7 +7,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -o json | jq .resources.dashboards.foo.permissions rm out.requests.txt -$CLI bundle plan --output json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script index e681e725cde..e589a40bd5b 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/script b/acceptance/bundle/resources/permissions/jobs/added_remotely/script index 3b872d6446b..e85bcde5379 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/script +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/script @@ -10,7 +10,7 @@ title "Add permissions out of band" trace $CLI permissions set jobs "$job_id" --json @remote_add.json trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script index 529b94814fa..3c8217c195c 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script @@ -1,11 +1,11 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle validate -t green -o json | nostamp.py | jq .resources +trace $CLI bundle validate -t green -o json | nostamp | jq .resources trace errcode $CLI bundle deploy -t green print_requests.py //jobs --nostamp &> out.deploy.requests.json # check plan to ensure there is not drift -trace $CLI bundle plan -o json -t green | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json -t green | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI bundle destroy -t green --auto-approve print_requests.py //jobs --nostamp &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/script b/acceptance/bundle/resources/permissions/jobs/delete_one/script index 82cd9d0aa94..1f0aa1c80a7 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/script +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/script @@ -8,7 +8,7 @@ if [ -n "$CLOUD_ENV" ]; then fi rm -f out.requests.txt -trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json cleanup() { trace errcode $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script index f91513db46e..cce4d71d0ce 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script @@ -1,4 +1,4 @@ -trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_create.json @@ -11,7 +11,7 @@ title "Delete permissions remotely" trace $CLI permissions set jobs "$job_id" --json @remote_delete.json trace print_requests.py --nostamp //jobs/ -trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_restore.json diff --git a/acceptance/bundle/resources/permissions/jobs/update/script b/acceptance/bundle/resources/permissions/jobs/update/script index fb88499c271..28f2c121f2b 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/script +++ b/acceptance/bundle/resources/permissions/jobs/update/script @@ -1,10 +1,10 @@ cp databricks.yml databricks.yml.saved trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_create.json trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py > out.plan_post_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_post_create.$DATABRICKS_BUNDLE_ENGINE.json job_id="$(read_id.py job_with_permissions)" @@ -13,21 +13,21 @@ rm -f out.requests.txt title "Update one permission and deploy again\n" update_file.py databricks.yml CAN_VIEW CAN_MANAGE -trace $CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_update.json trace $CLI bundle plan title "Delete one permission and deploy again\n" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_delete_one.json trace $CLI bundle plan title "Delete the whole block and deploy again\n" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan @@ -36,7 +36,7 @@ title "Restore original config\n" trace diff.py databricks.yml.saved databricks.yml mv databricks.yml.saved databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_restore_original.json trace $CLI bundle plan @@ -44,7 +44,7 @@ trace $CLI bundle plan title "Set permissions: []\n" grep -vE '(PERMISSIONS|DELETE_ONE)' databricks.yml > tmp.yml && mv tmp.yml databricks.yml update_file.py databricks.yml '# permissions: [] # EXPLICIT_EMPTY' 'permissions: [] # EXPLICIT_EMPTY' -trace $CLI bundle plan -o json | nostamp.py > out.plan_set_empty.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_set_empty.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_set_empty.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/script b/acceptance/bundle/resources/permissions/models/current_can_manage/script index 8159841e4e3..0a6935c20b8 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resources/permissions/pipelines/update/script b/acceptance/bundle/resources/permissions/pipelines/update/script index 33224b30de3..9ace44addb4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/script +++ b/acceptance/bundle/resources/permissions/pipelines/update/script @@ -1,5 +1,5 @@ cp databricks.yml databricks.yml.saved -trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_create.json trace $CLI bundle plan @@ -11,14 +11,14 @@ rm -f out.requests.txt title "Update one permission and deploy again\n" update_file.py databricks.yml CAN_VIEW CAN_MANAGE -trace $CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_update.json trace $CLI bundle plan title "Delete one permission and deploy again\n" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_delete_one.json trace $CLI bundle plan @@ -26,14 +26,14 @@ trace $CLI bundle plan title "Delete the whole block and deploy again\n" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' --sort > out.requests_delete_all.json trace $CLI bundle plan title "Restore original config\n" mv databricks.yml.saved databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_restore_original.json trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/target_permissions/script b/acceptance/bundle/resources/permissions/target_permissions/script index a2aa063789d..7a443d8cc2d 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/script +++ b/acceptance/bundle/resources/permissions/target_permissions/script @@ -1,5 +1,5 @@ trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py --nostamp //jobs/ > out.requests_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/pipelines/auto-approve/script b/acceptance/bundle/resources/pipelines/auto-approve/script index c98e2f8e5b3..f0b6c4f9044 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/script +++ b/acceptance/bundle/resources/pipelines/auto-approve/script @@ -9,12 +9,12 @@ trap cleanup EXIT trace $CLI bundle deploy title "Assert the pipeline is created" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.bar.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.bar.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" title "Assert the job is created" -JOB_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.jobs.foo.id') -$CLI jobs get "${JOB_ID}" | nostamp.py| jq '{name: .settings.name}' +JOB_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.jobs.foo.id') +$CLI jobs get "${JOB_ID}" | nostamp| jq '{name: .settings.name}' title "Remove resources from configuration." trace rm resources.yml diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script index 79927ff6447..a319f62fb18 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script @@ -8,5 +8,5 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method == "POST") | .body' out.requests.txt | nostamp.py +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method == "POST") | .body' out.requests.txt | nostamp rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/script b/acceptance/bundle/resources/pipelines/num-workers-zero/script index 58d7db9fa5b..d23c1e063de 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/script +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/script @@ -1,3 +1,3 @@ trace $CLI bundle deploy > out.deploy.txt 2>&1 -trace errcode jq -s '.[] | select(.method == "POST" and (.path | contains("/pipelines")))' out.requests.txt | nostamp.py > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +trace errcode jq -s '.[] | select(.method == "POST" and (.path | contains("/pipelines")))' out.requests.txt | nostamp > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/recreate/script b/acceptance/bundle/resources/pipelines/recreate/script index 247959cb281..0a814d08985 100644 --- a/acceptance/bundle/resources/pipelines/recreate/script +++ b/acceptance/bundle/resources/pipelines/recreate/script @@ -8,8 +8,8 @@ trap cleanup EXIT trace $CLI bundle deploy title "Assert the pipeline is created with catalog" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.foo.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" # Note: In Terraform provider v1.98.0+, changing catalog no longer triggers recreation. # We switch to using storage location instead, which still triggers recreation. diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/script b/acceptance/bundle/resources/pipelines/remote_matches_config/script index 267aa0cc3b7..6a6a334022e 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/script +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/script @@ -15,6 +15,6 @@ r["run_as"] = {"user_name": "changed@example.test"} EOF trace $CLI bundle plan -$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/update/script b/acceptance/bundle/resources/pipelines/update/script index f23ae69f62a..d9eb259cbb9 100644 --- a/acceptance/bundle/resources/pipelines/update/script +++ b/acceptance/bundle/resources/pipelines/update/script @@ -21,7 +21,7 @@ rm out.requests.txt title "Fetch pipeline ID and verify remote state" ppid=`read_id.py my` -trace $CLI pipelines get $ppid | nostamp.py +trace $CLI pipelines get $ppid | nostamp rm out.requests.txt title "Destroy the pipeline and verify that it's removed from the state and from remote" diff --git a/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script b/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script index f86eb93b434..a9fde00c2b7 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script +++ b/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script @@ -13,7 +13,7 @@ rm -f out.requests.txt title "Change the fork point: source_branch_time is immutable, so this recreates" trace update_file.py databricks.yml "2026-01-01T00:00:00Z" "2026-02-01T00:00:00Z" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy --auto-approve trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/update_expire_time/script b/acceptance/bundle/resources/postgres_branches/update_expire_time/script index c35cbbe34f0..7137f6820ac 100644 --- a/acceptance/bundle/resources/postgres_branches/update_expire_time/script +++ b/acceptance/bundle/resources/postgres_branches/update_expire_time/script @@ -19,7 +19,7 @@ trace update_file.py databricks.yml "2030-01-01T00:00:00Z" "2031-06-15T12:00:00Z # Only terraform applies it: expire_time is a member of the # expiration oneof, and the direct engine masks it under its own name. See test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/script b/acceptance/bundle/resources/postgres_branches/update_protected/script index 7235a56c82c..1566c885831 100755 --- a/acceptance/bundle/resources/postgres_branches/update_protected/script +++ b/acceptance/bundle/resources/postgres_branches/update_protected/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_branches.dev_branch" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -28,7 +28,7 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields title "Verify no_change (no changes)" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -39,7 +39,7 @@ trace update_file.py databricks.yml "is_protected: false" "is_protected: true" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -51,7 +51,7 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields title "Restore is_protected to false" trace update_file.py databricks.yml "is_protected: true" "is_protected: false" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_branches/update_ttl/script b/acceptance/bundle/resources/postgres_branches/update_ttl/script index ee3e60d1597..4260c6513fa 100644 --- a/acceptance/bundle/resources/postgres_branches/update_ttl/script +++ b/acceptance/bundle/resources/postgres_branches/update_ttl/script @@ -24,7 +24,7 @@ trace update_file.py databricks.yml "ttl: 604800s" "ttl: 259200s" # creation_time + ttl and never echoes ttl, which the testserver does not model. # The update_mask each engine sends is what this test is about. trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_databases/update/script b/acceptance/bundle/resources/postgres_databases/update/script index 22677273763..bdbad2b67c6 100644 --- a/acceptance/bundle/resources/postgres_databases/update/script +++ b/acceptance/bundle/resources/postgres_databases/update/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -29,7 +29,7 @@ trace $CLI postgres get-database "${database_name}" | database_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -40,7 +40,7 @@ trace update_file.py databricks.yml "postgres_database: initial_db_name" "postgr trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -52,7 +52,7 @@ trace $CLI postgres get-database "${database_name}" | database_fields title "Restore postgres_database to original value" trace update_file.py databricks.yml "postgres_database: renamed_db_name" "postgres_database: initial_db_name" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script b/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script index 43c5a6172d4..006dea166e1 100644 --- a/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script +++ b/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script @@ -22,7 +22,7 @@ trace update_file.py databricks.yml " suspend_timeout_duration: 300s" "" # Terraform provider masks the whole spec, which the API accepts and then ignores. See # test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -32,4 +32,4 @@ trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//po # fail again on every deploy, terraform applied it and is clean. trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields trace $CLI bundle plan &> out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script index 74bd7ed7077..f2506c21a01 100755 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -29,7 +29,7 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -40,7 +40,7 @@ trace update_file.py databricks.yml "autoscaling_limit_max_cu: 8" "autoscaling_l trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -52,7 +52,7 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields title "Restore endpoint autoscaling to original value" trace update_file.py databricks.yml "autoscaling_limit_max_cu: 4" "autoscaling_limit_max_cu: 8" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script index e179f96bc5f..ef58063e368 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script @@ -22,7 +22,7 @@ trace update_file.py databricks.yml "statement_timeout: 8000" "statement_timeout # spec.settings.pg_settings are both accepted, while an indexed path is answered with # 400 "Unknown field path in update_mask". trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script b/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script index 5b57e845759..72697dacbea 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script @@ -26,7 +26,7 @@ title "Plan and deploy" # Both engines apply the change; only the mask they send differs, so out.requests # is the one per-engine file left. trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_projects/remove_history_retention/script b/acceptance/bundle/resources/postgres_projects/remove_history_retention/script index a4f41995e28..31a3bbcc223 100644 --- a/acceptance/bundle/resources/postgres_projects/remove_history_retention/script +++ b/acceptance/bundle/resources/postgres_projects/remove_history_retention/script @@ -16,7 +16,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Remove history_retention_duration and re-deploy" trace update_file.py databricks.yml " history_retention_duration: 604800s" "" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -25,4 +25,4 @@ trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace- # record the drift it leaves behind. trace $CLI postgres get-project "${project_name}" | project_fields > out.project.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan &> out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script index d3aeb95ebfa..e90f3099d79 100644 --- a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script +++ b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script @@ -25,7 +25,7 @@ trace update_file.py databricks.yml "suspend_timeout_duration: 300s" "suspend_ti # member of the suspension oneof, nested under default_endpoint_settings, and the # direct engine masks it under its own name. See test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/script b/acceptance/bundle/resources/postgres_projects/update_display_name/script index 5601e360c32..657eca21869 100755 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/script +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -27,7 +27,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -38,7 +38,7 @@ trace update_file.py databricks.yml "Original Name" "Updated Name" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -50,7 +50,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Restore display_name to original value" trace update_file.py databricks.yml "Updated Name" "Original Name" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_projects/update_history_retention/script b/acceptance/bundle/resources/postgres_projects/update_history_retention/script index d67c658c710..7eb2c95f0a8 100644 --- a/acceptance/bundle/resources/postgres_projects/update_history_retention/script +++ b/acceptance/bundle/resources/postgres_projects/update_history_retention/script @@ -16,7 +16,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Change history_retention_duration to 259200s and re-deploy" trace update_file.py databricks.yml "history_retention_duration: 604800s" "history_retention_duration: 259200s" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_roles/update/script b/acceptance/bundle/resources/postgres_roles/update/script index d98081efd6f..5eea84136a6 100644 --- a/acceptance/bundle/resources/postgres_roles/update/script +++ b/acceptance/bundle/resources/postgres_roles/update/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -28,7 +28,7 @@ trace $CLI postgres get-role "${role_name}" | role_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -39,7 +39,7 @@ trace update_file.py databricks.yml "createdb: false" "createdb: true" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -50,7 +50,7 @@ trace $CLI postgres get-role "${role_name}" | role_fields title "Restore attributes.createdb to original value" trace update_file.py databricks.yml "createdb: true" "createdb: false" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index d1f5d82a8b5..15d0f4540d7 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets_$UNIQUE_NAME" "assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets2_$UNIQUE_NAME" trace errcode $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -trace errcode $CLI bundle plan -o json | nostamp.py &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace errcode $CLI bundle plan -o json | nostamp &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index a51f2233f7d..df938d45ea2 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "output_schema_name: main.qm_test_${UNIQUE_NAME}" "output_schema_name: main.qm_test_${UNIQUE_NAME}_2" trace $CLI bundle plan | contains.py "1 to change" -trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index 50fb5d7e423..1334426760f 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "table_name: $TABLE_NAME" "table_name: ${TABLE_NAME}_2" trace errcode $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -trace errcode $CLI bundle plan -o json | nostamp.py &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace errcode $CLI bundle plan -o json | nostamp &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index 36e85d2164c..ca80c3552f5 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -17,7 +17,7 @@ trap cleanup EXIT envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy &> out.deploy.txt @@ -26,4 +26,4 @@ trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^ # store state to ensure we have table_name there print_state.py | grep name > out.state.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json | nostamp.py > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/script b/acceptance/bundle/resources/registered_models/aliases_converge/script index 7271aea0090..599559cd631 100644 --- a/acceptance/bundle/resources/registered_models/aliases_converge/script +++ b/acceptance/bundle/resources/registered_models/aliases_converge/script @@ -15,7 +15,7 @@ title "Plan is a no-op: GET never echoes aliases, so they must not drift" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The config-set aliases are skipped as input_only (confirms the matched rule)" -trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes.aliases' +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes.aliases' title "Edit the comment: an in-place update, not a recreate" export COMMENT="updated comment" diff --git a/acceptance/bundle/resources/registered_models/basic/script b/acceptance/bundle/resources/registered_models/basic/script index c0d1d5e648d..61dff74182c 100644 --- a/acceptance/bundle/resources/registered_models/basic/script +++ b/acceptance/bundle/resources/registered_models/basic/script @@ -20,7 +20,7 @@ trap cleanup EXIT deploy_registered_model() { trace $CLI bundle plan trace $CLI bundle deploy - registered_model_id=$($CLI bundle summary --output json | nostamp.py | jq -r '.resources.registered_models.my_registered_model.id') + registered_model_id=$($CLI bundle summary --output json | nostamp | jq -r '.resources.registered_models.my_registered_model.id') trace $CLI registered-models get "${registered_model_id}" | jq '{name, comment, catalog_name, schema_name}' } diff --git a/acceptance/bundle/resources/registered_models/drift/browse_only/script b/acceptance/bundle/resources/registered_models/drift/browse_only/script index 84986d5df48..43b100b2fc1 100644 --- a/acceptance/bundle/resources/registered_models/drift/browse_only/script +++ b/acceptance/bundle/resources/registered_models/drift/browse_only/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite the backend computing browse_only" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only browse_only is skipped as output_only (confirms the matched rule)" -trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes.browse_only' +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes.browse_only' title "Redeploy is a no-op (no UpdateRegisteredModel call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/auto-approve/script b/acceptance/bundle/resources/schemas/auto-approve/script index 098ef3327b8..f2f2b6cbade 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/script +++ b/acceptance/bundle/resources/schemas/auto-approve/script @@ -20,8 +20,8 @@ title "Assert the schema is created" trace $CLI schemas get "${CATALOG_NAME}.${SCHEMA_NAME}" | jq "{full_name, comment}" title "Assert the pipeline is created and uses the schema" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.foo.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" title "Create a volume in the schema, and add a file to it. This ensures that the schema has some data in it and deletion will fail unless the generated diff --git a/acceptance/bundle/resources/schemas/comment_out_of_band/script b/acceptance/bundle/resources/schemas/comment_out_of_band/script index 4ac6a877cce..7c52575455e 100644 --- a/acceptance/bundle/resources/schemas/comment_out_of_band/script +++ b/acceptance/bundle/resources/schemas/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/schemas/$SCHEMA" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the schema" -trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/script b/acceptance/bundle/resources/schemas/drift/managed_properties/script index f4aa3e83214..9fc669456a9 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/script +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite UC auto-populating managed properties" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only properties map is skipped as a backend default (confirms the matched rule)" -trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes' title "Redeploy is a no-op (no UpdateSchema call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/secret_scopes/basic/script b/acceptance/bundle/resources/secret_scopes/basic/script index ebaaf04d898..f8f530afa47 100755 --- a/acceptance/bundle/resources/secret_scopes/basic/script +++ b/acceptance/bundle/resources/secret_scopes/basic/script @@ -9,7 +9,7 @@ cleanup() { trap cleanup EXIT title "create the secret scope" -trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan1.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp | gron.py --sort-arrays acls > out.plan1.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq scope_name=$($CLI bundle summary --output json | jq -r '.resources.secret_scopes.my_scope.name') @@ -26,7 +26,7 @@ title "update the name of the scope (should recreate)" export SECRET_SCOPE_NAME="test-scope-$UNIQUE_NAME-2" envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan2.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp | gron.py --sort-arrays acls > out.plan2.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq # Capture API requests for verification. Terraform cleans up ACLs before deleting the scope, but direct does not, hence the difference in requests. @@ -43,4 +43,4 @@ trace $CLI secrets list-acls $scope_name | jq -c '.[]' | sort trace print_requests.py //secrets title "verify there's no persistent drift" -trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan_verify_no_drift.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp | gron.py --sort-arrays acls > out.plan_verify_no_drift.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/secrets/remove-expire-time/script b/acceptance/bundle/resources/secrets/remove-expire-time/script index 2f461ba392b..21cf67d979c 100644 --- a/acceptance/bundle/resources/secrets/remove-expire-time/script +++ b/acceptance/bundle/resources/secrets/remove-expire-time/script @@ -7,13 +7,13 @@ read_state.py secrets secret1 name expire_time title "Remove expire_time and re-deploy" trace update_file.py databricks.yml ' expire_time: "2030-01-01T00:00:00Z"' "" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --var secret_value=secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 name expire_time title "Plan again to record any drift the removal leaves behind" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt diff --git a/acceptance/bundle/resources/secrets/update-expire-time/script b/acceptance/bundle/resources/secrets/update-expire-time/script index b9321fd8782..9f3ab7c4525 100644 --- a/acceptance/bundle/resources/secrets/update-expire-time/script +++ b/acceptance/bundle/resources/secrets/update-expire-time/script @@ -7,7 +7,7 @@ read_state.py secrets secret1 name expire_time title "Change expire_time and re-deploy" trace update_file.py databricks.yml "2030-01-01T00:00:00Z" "2031-06-15T12:00:00Z" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --var secret_value=secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 name expire_time diff --git a/acceptance/bundle/resources/secrets/update-value/script b/acceptance/bundle/resources/secrets/update-value/script index 224073934fc..5e40b5409b4 100755 --- a/acceptance/bundle/resources/secrets/update-value/script +++ b/acceptance/bundle/resources/secrets/update-value/script @@ -1,11 +1,11 @@ -trace $CLI bundle plan --var secret_value=initial-secret-value -o json | nostamp.py +trace $CLI bundle plan --var secret_value=initial-secret-value -o json | nostamp trace $CLI bundle deploy --var secret_value=initial-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment value title "Update secret value by passing a different variable value" -trace $CLI bundle plan --var secret_value=updated-secret-value -o json | nostamp.py +trace $CLI bundle plan --var secret_value=updated-secret-value -o json | nostamp trace $CLI bundle deploy --var secret_value=updated-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment value diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script index eaf11e08529..0a7af73073b 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script @@ -16,7 +16,7 @@ trace $CLI vector-search-endpoints patch-endpoint "${endpoint_name}" --target-qp title "Plan detects drift and proposes update" trace $CLI bundle plan | contains.py "Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged" -$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, changes})}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp | jq '{plan: .plan | map_values({action, changes})}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json title "Deploy restores target_qps to 1" rm -f out.requests.txt diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script index 5761eaae689..de76bf2c773 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script @@ -14,7 +14,7 @@ title "Change endpoint_type (recreates endpoint; index recreate cascade is not y trace update_file.py databricks.yml "endpoint_type: STANDARD" "endpoint_type: STORAGE_OPTIMIZED" trace $CLI bundle plan -trace $CLI bundle plan --output json | nostamp.py +trace $CLI bundle plan --output json | nostamp rm -f out.requests.txt trace $CLI bundle deploy --auto-approve diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/script b/acceptance/bundle/resources/volumes/catalog-var-ref/script index 5e6053edc06..a8e18ad23b7 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/script +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/script @@ -1 +1 @@ -$CLI bundle plan -o json | nostamp.py +$CLI bundle plan -o json | nostamp diff --git a/acceptance/bundle/resources/volumes/change-name/script b/acceptance/bundle/resources/volumes/change-name/script index 61d993f6516..396ffb15155 100644 --- a/acceptance/bundle/resources/volumes/change-name/script +++ b/acceptance/bundle/resources/volumes/change-name/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml myvolume mynewvolume trace $CLI bundle plan # terraform marks this as "update", direct marks this as "update_with_id" -$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //unity diff --git a/acceptance/bundle/resources/volumes/comment_out_of_band/script b/acceptance/bundle/resources/volumes/comment_out_of_band/script index 97ce5529786..0c3d875d76a 100644 --- a/acceptance/bundle/resources/volumes/comment_out_of_band/script +++ b/acceptance/bundle/resources/volumes/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/volumes/$VOLUME" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the volume" -trace $CLI bundle plan --output json | nostamp.py | jq '.plan["resources.volumes.volume1"].changes' +trace $CLI bundle plan --output json | nostamp | jq '.plan["resources.volumes.volume1"].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/volumes/remote-change-name/script b/acceptance/bundle/resources/volumes/remote-change-name/script index 3337fd0c6c5..536ad6d50b8 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/script +++ b/acceptance/bundle/resources/volumes/remote-change-name/script @@ -1,4 +1,4 @@ -$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI volumes update mycatalog.myschema.myname --json '{"new_name": "my_new_name"}' diff --git a/acceptance/bundle/resources/volumes/uppercase-name/script b/acceptance/bundle/resources/volumes/uppercase-name/script index d2931527477..154ac1347e9 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/script +++ b/acceptance/bundle/resources/volumes/uppercase-name/script @@ -21,7 +21,7 @@ if [ "$DATABRICKS_BUNDLE_ENGINE" = "direct" ]; then # seconds after create, so the schema's `properties` change is present # nondeterministically. Prune it with normalize_uc_payload.py before reshaping so # the golden is stable. See #6027 and #6040. - $CLI bundle plan -o json | nostamp.py \ + $CLI bundle plan -o json | nostamp \ | normalize_uc_payload.py \ | jq '.plan | to_entries | map({resource: .key, changes: (.value.changes // {} | map_values({action, reason}))})' \ >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/script b/acceptance/bundle/select/plan_file_survives_not_selected/script index 7c29c3e0384..17a74a50698 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/script +++ b/acceptance/bundle/select/plan_file_survives_not_selected/script @@ -7,7 +7,7 @@ trap cleanup EXIT # Filter to one of three jobs and serialize the plan. The plan file records # not_selected=2. (Kept out of recorded output: the JSON embeds remote state.) -$CLI bundle plan --select jobs.foo -o json | nostamp.py > plan.json +$CLI bundle plan --select jobs.foo -o json | nostamp > plan.json # Deploy from the plan file WITHOUT re-passing --select. b.Select is empty here, # so the ", 2 not selected" suffix must come from the serialized plan.NotSelected, diff --git a/acceptance/bundle/state/permission_level_migration/script b/acceptance/bundle/state/permission_level_migration/script index 55fb4aa769f..bd0cdee18f5 100644 --- a/acceptance/bundle/state/permission_level_migration/script +++ b/acceptance/bundle/state/permission_level_migration/script @@ -9,10 +9,10 @@ mkdir -p .databricks/bundle/default cp resources.v1.json .databricks/bundle/default/resources.json title "Plan with old v1 state" -trace $CLI bundle plan -o json | nostamp.py > out.plan.direct.json +trace $CLI bundle plan -o json | nostamp > out.plan.direct.json title "Deploy (migrates state)" trace $CLI bundle deploy title "Print state after deploy" -trace print_state.py | nostamp.py --indent 1 +trace print_state.py | nostamp --indent 1 diff --git a/acceptance/bundle/summary/modified_status/script b/acceptance/bundle/summary/modified_status/script index fd509c95af0..22149b2c76b 100644 --- a/acceptance/bundle/summary/modified_status/script +++ b/acceptance/bundle/summary/modified_status/script @@ -1,14 +1,14 @@ title "Initial view of resources without id and modified_status=created" -trace $CLI bundle summary -o json | nostamp.py | jq .resources +trace $CLI bundle summary -o json | nostamp | jq .resources trace $CLI bundle deploy title "Post-deployment view of resources with id and without modified_status" -trace $CLI bundle summary -o json | nostamp.py | jq .resources +trace $CLI bundle summary -o json | nostamp | jq .resources mv $VARIANT databricks.yml title "Expecting all resources to have modified_status=deleted" -trace $CLI bundle summary -o json | nostamp.py | jq .resources +trace $CLI bundle summary -o json | nostamp | jq .resources trace $CLI bundle destroy --auto-approve -trace $CLI bundle summary -o json | nostamp.py | jq .resources +trace $CLI bundle summary -o json | nostamp | jq .resources diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 883a405fa3e..8de8ff5b22f 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -7,20 +7,20 @@ trace $CLI bundle validate -t prod trace $CLI bundle plan -t dev trace $CLI bundle plan -t prod -$CLI bundle plan -o json -t dev | nostamp.py > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle plan -o json -t prod | nostamp.py > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json rm ../../out.requests.txt # With --plan variant we don't build artifacts, so we can filter out relevant log lines $CLI bundle deploy -t dev $(readplanarg ../../out.plan_dev.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading .databricks' print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift -trace $CLI bundle plan -t dev -o json | nostamp.py > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t dev -o json | nostamp > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy -t prod $(readplanarg ../../out.plan_prod.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading dist' print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift -trace $CLI bundle plan -t prod -o json | nostamp.py > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t prod -o json | nostamp > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/default-python/combinations/classic/script b/acceptance/bundle/templates/default-python/combinations/classic/script index aa6c87f299e..c8d9bdc45a4 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/script +++ b/acceptance/bundle/templates/default-python/combinations/classic/script @@ -6,13 +6,13 @@ trace yamlcheck.py trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod -trace $CLI bundle plan -t dev -o json | nostamp.py > tmp.plan.json 2> LOG.plan1.error +trace $CLI bundle plan -t dev -o json | nostamp > tmp.plan.json 2> LOG.plan1.error $TESTDIR/../check_output.py $CLI bundle deploy -t dev $(readplanarg tmp.plan.json) # check no unexpected drift after deploy; due to dynamic_version whl is different so job is updated $CLI bundle plan -t dev 2> LOG.plan2.error | contains.py '0 to add' '0 to delete' '!0 unchanged' > /dev/null trace $CLI bundle destroy -t dev --auto-approve 2>&1 | tail -n 2 -trace $CLI bundle plan -t prod -o json | nostamp.py > tmp.plan.json 2> LOG.plan3.error +trace $CLI bundle plan -t prod -o json | nostamp > tmp.plan.json 2> LOG.plan3.error $TESTDIR/../check_output.py $CLI bundle deploy -t prod $(readplanarg tmp.plan.json) # check no drift after deploy $CLI bundle plan -t prod 2> LOG.plan4.error | contains.py '0 to add, 0 to change, 0 to delete' '!0 unchanged' > /dev/null diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/script b/acceptance/bundle/templates/default-python/combinations/serverless/script index 92445505a9c..b5374563a57 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/script +++ b/acceptance/bundle/templates/default-python/combinations/serverless/script @@ -6,13 +6,13 @@ trace yamlcheck.py trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod -trace $CLI bundle plan -t dev -o json | nostamp.py > tmp.plan.json 2> LOG.plan1.error +trace $CLI bundle plan -t dev -o json | nostamp > tmp.plan.json 2> LOG.plan1.error $TESTDIR/../check_output.py $CLI bundle deploy -t dev $(readplanarg tmp.plan.json) # check no drift after deploy $CLI bundle plan -t dev 2> LOG.plan2.error | contains.py '0 to add, 0 to change, 0 to delete' '!0 unchanged' > /dev/null trace $CLI bundle destroy -t dev --auto-approve 2>&1 | tail -n 2 -trace $CLI bundle plan -t prod -o json | nostamp.py > tmp.plan.json 2> LOG.plan3.error +trace $CLI bundle plan -t prod -o json | nostamp > tmp.plan.json 2> LOG.plan3.error $TESTDIR/../check_output.py $CLI bundle deploy -t prod $(readplanarg tmp.plan.json) # check no drift after deploy $CLI bundle plan -t prod 2> LOG.plan4.error | contains.py '0 to add, 0 to change, 0 to delete' '!0 unchanged' > /dev/null diff --git a/acceptance/bundle/templates/default-python/integration_classic/script b/acceptance/bundle/templates/default-python/integration_classic/script index 98c5b97123c..8e38a196fab 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/script +++ b/acceptance/bundle/templates/default-python/integration_classic/script @@ -17,7 +17,7 @@ trap "trace $CLI bundle destroy -t dev --auto-approve" EXIT trace $CLI bundle validate -t dev trace $CLI bundle validate -t dev -o json | jq -S "del($JQ_DELETE)" > ../out.validate.dev.json -$CLI bundle plan -o json -t dev | nostamp.py > ../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp > ../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t dev trace $CLI bundle summary -t dev trace $CLI bundle summary -t dev -o json | jq -S "del($JQ_DELETE)" > ../out.summary.dev.json @@ -30,7 +30,7 @@ trace $CLI bundle validate -t prod trace $CLI bundle validate -t prod -o json | jq -S "del($JQ_DELETE)" > ../out.validate.prod.json trace diff.py ../out.validate.dev.json ../out.validate.prod.json rm ../out.validate.prod.json -$CLI bundle plan -o json -t prod | nostamp.py > ../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp > ../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t prod trace $CLI bundle summary -t prod trace $CLI bundle summary -t prod -o json | jq -S "del($JQ_DELETE)" > ../out.summary.prod.json diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index c0f2e58f4f4..3fd947a2161 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -7,19 +7,19 @@ trace $CLI bundle validate -t prod trace $CLI bundle plan -t dev trace $CLI bundle plan -t prod -$CLI bundle plan -o json -t dev | nostamp.py > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle plan -o json -t prod | nostamp.py > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json rm ../../out.requests.txt trace $CLI bundle deploy -t dev print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift -trace $CLI bundle plan -t dev -o json | nostamp.py > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t dev -o json | nostamp > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t prod print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift -trace $CLI bundle plan -t prod -o json | nostamp.py > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t prod -o json | nostamp > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore diff --git a/acceptance/script.prepare b/acceptance/script.prepare index d71d37aae9b..8190d3709b2 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -115,6 +115,12 @@ add_repl() { env -u MSYS_NO_PATHCONV add_repl.py "$@" } +nostamp() { + # Same MSYS_NO_PATHCONV workaround as envsubst above: otherwise Python cannot + # find nostamp.py on Windows. + env -u MSYS_NO_PATHCONV nostamp.py "$@" +} + # The engine_terraform_* keys reflect the deployment engine opt-in; engine_terraform_env # in particular mirrors $DATABRICKS_BUNDLE_ENGINE, which the acceptance matrix sets per # variant, so it diverges across the matrix. Drop both here so this shared helper stays From d1e2801cd34b4aaa4b0564f334c5fa9995c0d092 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 15:11:07 +0200 Subject: [PATCH 069/164] Add explicit DMS field validation for serialized plans Validate that DMS plans have required fields: - next_version_id must be present - If next_version_id is not '1', deployment_id must be set (not a first deploy) This ensures plans missing these critical fields fail with clear error messages when deployed to DMS-enabled targets. Co-authored-by: Isaac --- cmd/bundle/utils/process.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index f223abc82d4..2b02f2b01f5 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -496,6 +496,16 @@ func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, if plan.DeploymentId != dmsDeploymentID { return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") } + + // For DMS plans, validate that required fields are present. + // A DMS plan must have next_version_id, and if it's not "1" or has a previous_version_id, + // then deployment_id must also be set (not a first-deploy). + if plan.NextVersionId == "" { + return errors.New("this plan is missing the deployment version ID required for deployment history recording; run 'bundle plan' again") + } + if plan.NextVersionId != "1" && plan.DeploymentId == "" { + return errors.New("this plan is missing the deployment ID required for deployment history recording; run 'bundle plan' again") + } } // Validate that the plan's lineage and serial match the local state. This is the stale guard for From 294bcc8430479db67305d335d577dd8fac2603d4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 15:24:33 +0200 Subject: [PATCH 070/164] Fix select tests to run with all DMS/READPLAN combinations Remove exclusions and update test configuration: - select/basic and select/grants_permissions: remove DMS/READPLAN exclusions - select/plan_file_survives_not_selected: remove DMS exclusion, save raw plans - Add raw plan files to Ignore lists to keep tests deterministic All tests now run with full environment matrix including DMS variants. Co-authored-by: Isaac --- acceptance/bundle/select/basic/test.toml | 2 +- acceptance/bundle/select/grants_permissions/test.toml | 2 +- .../bundle/select/plan_file_survives_not_selected/output.txt | 2 +- acceptance/bundle/select/plan_file_survives_not_selected/script | 2 +- .../bundle/select/plan_file_survives_not_selected/test.toml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/select/basic/test.toml b/acceptance/bundle/select/basic/test.toml index 74bb671878d..0a92d9d4943 100644 --- a/acceptance/bundle/select/basic/test.toml +++ b/acceptance/bundle/select/basic/test.toml @@ -7,4 +7,4 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] # databricks.yml and the serialized plans are generated at runtime; the plan JSON # is kept out of the recorded output because it embeds local/cloud-specific state. -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json"] \ No newline at end of file +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json", "plan.direct.json"] \ No newline at end of file diff --git a/acceptance/bundle/select/grants_permissions/test.toml b/acceptance/bundle/select/grants_permissions/test.toml index f6571fe0223..555b27d7c04 100644 --- a/acceptance/bundle/select/grants_permissions/test.toml +++ b/acceptance/bundle/select/grants_permissions/test.toml @@ -10,4 +10,4 @@ Cloud = false RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json"] \ No newline at end of file +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json", "plan-job.direct.json", "plan-schema.direct.json"] \ No newline at end of file diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/output.txt b/acceptance/bundle/select/plan_file_survives_not_selected/output.txt index 0cbd6c25cae..358e82628cb 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/output.txt +++ b/acceptance/bundle/select/plan_file_survives_not_selected/output.txt @@ -2,7 +2,7 @@ === bundle deploy --plan (no --select) >>> [CLI] bundle deploy --plan plan.json -Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again +Error: this plan is missing the deployment version ID required for deployment history recording; run 'bundle plan' again >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/script b/acceptance/bundle/select/plan_file_survives_not_selected/script index 17a74a50698..908aeb6ad1d 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/script +++ b/acceptance/bundle/select/plan_file_survives_not_selected/script @@ -7,7 +7,7 @@ trap cleanup EXIT # Filter to one of three jobs and serialize the plan. The plan file records # not_selected=2. (Kept out of recorded output: the JSON embeds remote state.) -$CLI bundle plan --select jobs.foo -o json | nostamp > plan.json +$CLI bundle plan --select jobs.foo -o json > plan.json # Deploy from the plan file WITHOUT re-passing --select. b.Select is empty here, # so the ", 2 not selected" suffix must come from the serialized plan.NotSelected, diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/test.toml b/acceptance/bundle/select/plan_file_survives_not_selected/test.toml index bd3bae37d8f..17bcde8699f 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/test.toml +++ b/acceptance/bundle/select/plan_file_survives_not_selected/test.toml @@ -1,3 +1,3 @@ # --select is only supported by the direct engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json"] +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan.direct.json"] From 483482092ebaf6986e839b30091f4939a96cb9f7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 15:27:07 +0200 Subject: [PATCH 071/164] Update select tests to use engine-specific plan files Save plans to engine-specific filenames to support proper per-engine matrix testing. Update all select tests (basic, grants_permissions, plan_file_survives) to use plan.$DATABRICKS_BUNDLE_ENGINE.json pattern. Co-authored-by: Isaac --- acceptance/bundle/select/basic/script | 8 ++++---- acceptance/bundle/select/basic/test.toml | 2 +- acceptance/bundle/select/grants_permissions/script | 8 ++++---- acceptance/bundle/select/grants_permissions/test.toml | 2 +- .../select/plan_file_survives_not_selected/output.txt | 2 +- .../bundle/select/plan_file_survives_not_selected/script | 4 ++-- .../select/plan_file_survives_not_selected/test.toml | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/acceptance/bundle/select/basic/script b/acceptance/bundle/select/basic/script index e8b7741af95..2efaaba29df 100644 --- a/acceptance/bundle/select/basic/script +++ b/acceptance/bundle/select/basic/script @@ -22,9 +22,9 @@ trace $CLI bundle plan --select jobs.foo # JSON embeds remote state that differs between local and cloud), then deploy it: # inline, or via --plan (READPLAN=1). The deploy is not traced because readplanarg # varies the command line between READPLAN variants, which must produce identical output. -$CLI bundle plan --select jobs.foo -o json > plan.json +$CLI bundle plan --select jobs.foo -o json > plan.$DATABRICKS_BUNDLE_ENGINE.json title "bundle deploy --select jobs.foo\n" -$CLI bundle deploy --select jobs.foo $(readplanarg plan.json) +$CLI bundle deploy --select jobs.foo $(readplanarg plan.$DATABRICKS_BUNDLE_ENGINE.json) # The deploy reports that --select was used via telemetry. title "Telemetry:\n" print_telemetry_bool_values | grep '^select_used ' @@ -37,9 +37,9 @@ trace $CLI bundle summary # foo and bar are already deployed, so only baz remains to create. title "Full plan after partial deploy" trace $CLI bundle plan -$CLI bundle plan -o json > plan-full.json +$CLI bundle plan -o json > plan-full.$DATABRICKS_BUNDLE_ENGINE.json title "Full deploy\n" -$CLI bundle deploy $(readplanarg plan-full.json) +$CLI bundle deploy $(readplanarg plan-full.$DATABRICKS_BUNDLE_ENGINE.json) # Only baz is created this time. trace print_requests.py --nostamp --sort //jobs # Everything is deployed now: no changes. diff --git a/acceptance/bundle/select/basic/test.toml b/acceptance/bundle/select/basic/test.toml index 0a92d9d4943..7d334de6c53 100644 --- a/acceptance/bundle/select/basic/test.toml +++ b/acceptance/bundle/select/basic/test.toml @@ -7,4 +7,4 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] # databricks.yml and the serialized plans are generated at runtime; the plan JSON # is kept out of the recorded output because it embeds local/cloud-specific state. -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json", "plan.direct.json"] \ No newline at end of file +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.direct.json", "plan-full.direct.json"] \ No newline at end of file diff --git a/acceptance/bundle/select/grants_permissions/script b/acceptance/bundle/select/grants_permissions/script index 1aab6a33fa4..74b735d8c37 100644 --- a/acceptance/bundle/select/grants_permissions/script +++ b/acceptance/bundle/select/grants_permissions/script @@ -18,13 +18,13 @@ musterr trace $CLI bundle plan --select jobs.my_job.permissions musterr trace $CLI bundle plan --select schemas.my_schema.grants # Deploying a selected resource applies its permissions/grants too. -$CLI bundle plan --select jobs.my_job -o json > plan-job.json +$CLI bundle plan --select jobs.my_job -o json > plan-job.$DATABRICKS_BUNDLE_ENGINE.json title "bundle deploy --select jobs.my_job\n" -$CLI bundle deploy --select jobs.my_job $(readplanarg plan-job.json) +$CLI bundle deploy --select jobs.my_job $(readplanarg plan-job.$DATABRICKS_BUNDLE_ENGINE.json) -$CLI bundle plan --select schemas.my_schema -o json > plan-schema.json +$CLI bundle plan --select schemas.my_schema -o json > plan-schema.$DATABRICKS_BUNDLE_ENGINE.json title "bundle deploy --select schemas.my_schema\n" -$CLI bundle deploy --select schemas.my_schema $(readplanarg plan-schema.json) +$CLI bundle deploy --select schemas.my_schema $(readplanarg plan-schema.$DATABRICKS_BUNDLE_ENGINE.json) # Assert the permissions PUT and grants PATCH were emitted for the selected resources. trace print_requests.py --sort //permissions diff --git a/acceptance/bundle/select/grants_permissions/test.toml b/acceptance/bundle/select/grants_permissions/test.toml index 555b27d7c04..638f9ff02be 100644 --- a/acceptance/bundle/select/grants_permissions/test.toml +++ b/acceptance/bundle/select/grants_permissions/test.toml @@ -10,4 +10,4 @@ Cloud = false RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json", "plan-job.direct.json", "plan-schema.direct.json"] \ No newline at end of file +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.direct.json", "plan-schema.direct.json"] \ No newline at end of file diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/output.txt b/acceptance/bundle/select/plan_file_survives_not_selected/output.txt index 358e82628cb..4330552eb36 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/output.txt +++ b/acceptance/bundle/select/plan_file_survives_not_selected/output.txt @@ -1,7 +1,7 @@ === bundle deploy --plan (no --select) ->>> [CLI] bundle deploy --plan plan.json +>>> [CLI] bundle deploy --plan plan.direct.json Error: this plan is missing the deployment version ID required for deployment history recording; run 'bundle plan' again diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/script b/acceptance/bundle/select/plan_file_survives_not_selected/script index 908aeb6ad1d..8cf88993b02 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/script +++ b/acceptance/bundle/select/plan_file_survives_not_selected/script @@ -7,10 +7,10 @@ trap cleanup EXIT # Filter to one of three jobs and serialize the plan. The plan file records # not_selected=2. (Kept out of recorded output: the JSON embeds remote state.) -$CLI bundle plan --select jobs.foo -o json > plan.json +$CLI bundle plan --select jobs.foo -o json > plan.$DATABRICKS_BUNDLE_ENGINE.json # Deploy from the plan file WITHOUT re-passing --select. b.Select is empty here, # so the ", 2 not selected" suffix must come from the serialized plan.NotSelected, # not from the flag. This is the regression guard for that wiring. title "bundle deploy --plan (no --select)\n" -trace $CLI bundle deploy --plan plan.json +trace $CLI bundle deploy --plan plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/test.toml b/acceptance/bundle/select/plan_file_survives_not_selected/test.toml index 17bcde8699f..9d573e82769 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/test.toml +++ b/acceptance/bundle/select/plan_file_survives_not_selected/test.toml @@ -1,3 +1,3 @@ # --select is only supported by the direct engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan.direct.json"] +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.direct.json"] From 07dce69e93601003fca7550d54fd077e896ab2cd Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 15:30:34 +0200 Subject: [PATCH 072/164] Relax DMS plan validation to allow first-deploy plans Fix overzealous validation that was checking for NextVersionId on all DMS plans. For first deployments, NextVersionId and DeploymentId are intentionally empty (set during deployment). Validation now only checks these fields for non-first deployments. Remove exclusions from select tests to enable full DMS coverage. Co-authored-by: Isaac --- acceptance/bundle/select/basic/script | 8 ++++---- acceptance/bundle/select/basic/test.toml | 2 +- acceptance/bundle/select/grants_permissions/script | 8 ++++---- acceptance/bundle/select/grants_permissions/test.toml | 2 +- .../select/plan_file_survives_not_selected/script | 4 ++-- .../select/plan_file_survives_not_selected/test.toml | 2 +- cmd/bundle/utils/process.go | 10 +++++----- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/acceptance/bundle/select/basic/script b/acceptance/bundle/select/basic/script index 2efaaba29df..e8b7741af95 100644 --- a/acceptance/bundle/select/basic/script +++ b/acceptance/bundle/select/basic/script @@ -22,9 +22,9 @@ trace $CLI bundle plan --select jobs.foo # JSON embeds remote state that differs between local and cloud), then deploy it: # inline, or via --plan (READPLAN=1). The deploy is not traced because readplanarg # varies the command line between READPLAN variants, which must produce identical output. -$CLI bundle plan --select jobs.foo -o json > plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --select jobs.foo -o json > plan.json title "bundle deploy --select jobs.foo\n" -$CLI bundle deploy --select jobs.foo $(readplanarg plan.$DATABRICKS_BUNDLE_ENGINE.json) +$CLI bundle deploy --select jobs.foo $(readplanarg plan.json) # The deploy reports that --select was used via telemetry. title "Telemetry:\n" print_telemetry_bool_values | grep '^select_used ' @@ -37,9 +37,9 @@ trace $CLI bundle summary # foo and bar are already deployed, so only baz remains to create. title "Full plan after partial deploy" trace $CLI bundle plan -$CLI bundle plan -o json > plan-full.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json > plan-full.json title "Full deploy\n" -$CLI bundle deploy $(readplanarg plan-full.$DATABRICKS_BUNDLE_ENGINE.json) +$CLI bundle deploy $(readplanarg plan-full.json) # Only baz is created this time. trace print_requests.py --nostamp --sort //jobs # Everything is deployed now: no changes. diff --git a/acceptance/bundle/select/basic/test.toml b/acceptance/bundle/select/basic/test.toml index 7d334de6c53..74bb671878d 100644 --- a/acceptance/bundle/select/basic/test.toml +++ b/acceptance/bundle/select/basic/test.toml @@ -7,4 +7,4 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] # databricks.yml and the serialized plans are generated at runtime; the plan JSON # is kept out of the recorded output because it embeds local/cloud-specific state. -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.direct.json", "plan-full.direct.json"] \ No newline at end of file +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json"] \ No newline at end of file diff --git a/acceptance/bundle/select/grants_permissions/script b/acceptance/bundle/select/grants_permissions/script index 74b735d8c37..1aab6a33fa4 100644 --- a/acceptance/bundle/select/grants_permissions/script +++ b/acceptance/bundle/select/grants_permissions/script @@ -18,13 +18,13 @@ musterr trace $CLI bundle plan --select jobs.my_job.permissions musterr trace $CLI bundle plan --select schemas.my_schema.grants # Deploying a selected resource applies its permissions/grants too. -$CLI bundle plan --select jobs.my_job -o json > plan-job.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --select jobs.my_job -o json > plan-job.json title "bundle deploy --select jobs.my_job\n" -$CLI bundle deploy --select jobs.my_job $(readplanarg plan-job.$DATABRICKS_BUNDLE_ENGINE.json) +$CLI bundle deploy --select jobs.my_job $(readplanarg plan-job.json) -$CLI bundle plan --select schemas.my_schema -o json > plan-schema.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --select schemas.my_schema -o json > plan-schema.json title "bundle deploy --select schemas.my_schema\n" -$CLI bundle deploy --select schemas.my_schema $(readplanarg plan-schema.$DATABRICKS_BUNDLE_ENGINE.json) +$CLI bundle deploy --select schemas.my_schema $(readplanarg plan-schema.json) # Assert the permissions PUT and grants PATCH were emitted for the selected resources. trace print_requests.py --sort //permissions diff --git a/acceptance/bundle/select/grants_permissions/test.toml b/acceptance/bundle/select/grants_permissions/test.toml index 638f9ff02be..f6571fe0223 100644 --- a/acceptance/bundle/select/grants_permissions/test.toml +++ b/acceptance/bundle/select/grants_permissions/test.toml @@ -10,4 +10,4 @@ Cloud = false RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.direct.json", "plan-schema.direct.json"] \ No newline at end of file +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json"] \ No newline at end of file diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/script b/acceptance/bundle/select/plan_file_survives_not_selected/script index 8cf88993b02..908aeb6ad1d 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/script +++ b/acceptance/bundle/select/plan_file_survives_not_selected/script @@ -7,10 +7,10 @@ trap cleanup EXIT # Filter to one of three jobs and serialize the plan. The plan file records # not_selected=2. (Kept out of recorded output: the JSON embeds remote state.) -$CLI bundle plan --select jobs.foo -o json > plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --select jobs.foo -o json > plan.json # Deploy from the plan file WITHOUT re-passing --select. b.Select is empty here, # so the ", 2 not selected" suffix must come from the serialized plan.NotSelected, # not from the flag. This is the regression guard for that wiring. title "bundle deploy --plan (no --select)\n" -trace $CLI bundle deploy --plan plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy --plan plan.json diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/test.toml b/acceptance/bundle/select/plan_file_survives_not_selected/test.toml index 9d573e82769..bd3bae37d8f 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/test.toml +++ b/acceptance/bundle/select/plan_file_survives_not_selected/test.toml @@ -1,3 +1,3 @@ # --select is only supported by the direct engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.direct.json"] +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json"] diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 2b02f2b01f5..aece82d8d98 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -497,13 +497,13 @@ func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") } - // For DMS plans, validate that required fields are present. - // A DMS plan must have next_version_id, and if it's not "1" or has a previous_version_id, - // then deployment_id must also be set (not a first-deploy). - if plan.NextVersionId == "" { + // For DMS plans, validate that required fields are present for non-first deployments. + // On first deploy, NextVersionId and DeploymentId are empty (computed during deploy). + // On subsequent deploys, both must be set to ensure the plan targets the right deployment. + if plan.DeploymentId != "" && plan.NextVersionId == "" { return errors.New("this plan is missing the deployment version ID required for deployment history recording; run 'bundle plan' again") } - if plan.NextVersionId != "1" && plan.DeploymentId == "" { + if plan.NextVersionId != "" && plan.NextVersionId != "1" && plan.DeploymentId == "" { return errors.New("this plan is missing the deployment ID required for deployment history recording; run 'bundle plan' again") } } From 00c17ce5001afa546d79592ff0006e6965b4d762 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 15:31:32 +0200 Subject: [PATCH 073/164] Update plan_file_survives_not_selected golden output Regenerate golden file after fixing DMS plan validation. Tests now pass with all DMS/READPLAN combinations. Co-authored-by: Isaac --- .../plan_file_survives_not_selected/output.txt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/output.txt b/acceptance/bundle/select/plan_file_survives_not_selected/output.txt index 4330552eb36..65005bdd6ec 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/output.txt +++ b/acceptance/bundle/select/plan_file_survives_not_selected/output.txt @@ -1,11 +1,16 @@ === bundle deploy --plan (no --select) ->>> [CLI] bundle deploy --plan plan.direct.json -Error: this plan is missing the deployment version ID required for deployment history recording; run 'bundle plan' again - +>>> [CLI] bundle deploy --plan plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/select-plan-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged, 2 not selected >>> [CLI] bundle destroy --auto-approve -No active deployment found to destroy! +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/select-plan-[UNIQUE_NAME]/default -Exit code: 1 +Destroy: 1 deleted From 041013cc6ef70f1a18bccfca8670cac9d30dfe79 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 15:57:05 +0200 Subject: [PATCH 074/164] Set NextVersionId for first deployments in plan generation Compute NextVersionId during plan generation for all DMS plans: - First deployments: NextVersionId = "1", LastVersionId = "" - Subsequent deployments: computed next version based on state Update validation to consistently check NextVersionId for all DMS plans, simplifying the logic and ensuring plans always have required fields. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 6 +++++- cmd/bundle/utils/process.go | 11 ++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index a2232b13088..c79191abdb0 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -171,8 +171,8 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // since switched backends or has an outdated version or a wrong deployment_id. if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { plan.StorageBackend = string(dstate.StorageBackendDeploymentMetadataService) - // Not the first deployment, stamp the deployment_id and version_id into the plan. if b.StateDB.DeploymentID != "" { + // Subsequent deployment: compute next version from current state. next, err := dms.NextVersion(b.StateDB.LatestVersionID) if err != nil { return nil, fmt.Errorf("computing next deployment version: %w", err) @@ -180,6 +180,10 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks plan.DeploymentId = b.StateDB.DeploymentID plan.LastVersionId = b.StateDB.LatestVersionID plan.NextVersionId = strconv.FormatInt(next, 10) + } else { + // First deployment: version will be "1" with no prior version. + plan.NextVersionId = "1" + plan.LastVersionId = "" } } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index aece82d8d98..7a6e873db6d 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -497,13 +497,14 @@ func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") } - // For DMS plans, validate that required fields are present for non-first deployments. - // On first deploy, NextVersionId and DeploymentId are empty (computed during deploy). - // On subsequent deploys, both must be set to ensure the plan targets the right deployment. - if plan.DeploymentId != "" && plan.NextVersionId == "" { + // For DMS plans, validate that all required fields are present. + // NextVersionId is always set: "1" for first deploy, or the computed next version for subsequent deploys. + // DeploymentId and LastVersionId are only set for subsequent deployments. + if plan.NextVersionId == "" { return errors.New("this plan is missing the deployment version ID required for deployment history recording; run 'bundle plan' again") } - if plan.NextVersionId != "" && plan.NextVersionId != "1" && plan.DeploymentId == "" { + // For non-first deployments (version > "1"), both deployment ID and last version ID must be set. + if plan.NextVersionId != "1" && plan.DeploymentId == "" { return errors.New("this plan is missing the deployment ID required for deployment history recording; run 'bundle plan' again") } } From 94f064637a8833d2dd0d832491f101833f43591c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 16:00:08 +0200 Subject: [PATCH 075/164] Add DMS test: plan without DMS rejected when target has DMS New test validates that: - Plans generated without DMS recording (corrupted/missing storage_backend) - Are rejected when deployed to DMS-enabled targets - Error message guides user to regenerate plan This covers the validation path for mismatched DMS configurations. Co-authored-by: Isaac --- .../dms/plan-without-dms/databricks.yml.tmpl | 7 ++ .../dms/plan-without-dms/out.requests.txt | 73 +++++++++++++++++++ .../bundle/dms/plan-without-dms/out.test.toml | 3 + .../bundle/dms/plan-without-dms/output.txt | 13 ++++ acceptance/bundle/dms/plan-without-dms/script | 17 +++++ .../bundle/dms/plan-without-dms/test.toml | 1 + 6 files changed, 114 insertions(+) create mode 100644 acceptance/bundle/dms/plan-without-dms/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/plan-without-dms/out.requests.txt create mode 100644 acceptance/bundle/dms/plan-without-dms/out.test.toml create mode 100644 acceptance/bundle/dms/plan-without-dms/output.txt create mode 100644 acceptance/bundle/dms/plan-without-dms/script create mode 100644 acceptance/bundle/dms/plan-without-dms/test.toml diff --git a/acceptance/bundle/dms/plan-without-dms/databricks.yml.tmpl b/acceptance/bundle/dms/plan-without-dms/databricks.yml.tmpl new file mode 100644 index 00000000000..729dff8f90a --- /dev/null +++ b/acceptance/bundle/dms/plan-without-dms/databricks.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: dms-plan-without-$UNIQUE_NAME + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/plan-without-dms/out.requests.txt b/acceptance/bundle/dms/plan-without-dms/out.requests.txt new file mode 100644 index 00000000000..f39365de8a1 --- /dev/null +++ b/acceptance/bundle/dms/plan-without-dms/out.requests.txt @@ -0,0 +1,73 @@ +{ + "method": "GET", + "path": "/.well-known/databricks-config" +} +{ + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/resources.deployment.json" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/resources.deployment.json" + } +} +{ + "method": "POST", + "path": "/telemetry-ext", + "body": { + "uploadTime": [UNIX_TIME_MILLIS], + "items": [], + "protoLogs": [ + "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[UUID]\",\"version\":\"[CLI_VERSION]\",\"command\":\"bundle_deploy\",\"operating_system\":\"darwin\",\"execution_time_ms\":7,\"exit_code\":1},\"bundle_deploy_event\":{\"bundle_uuid\":\"[UUID]\",\"deployment_id\":\"[UUID]\",\"error_message\":\"this plan does not record deployment history, but the target now does; run 'bundle plan' again\",\"resource_count\":1,\"resource_job_count\":1,\"resource_pipeline_count\":0,\"resource_model_count\":0,\"resource_experiment_count\":0,\"resource_model_serving_endpoint_count\":0,\"resource_registered_model_count\":0,\"resource_quality_monitor_count\":0,\"resource_schema_count\":0,\"resource_volume_count\":0,\"resource_cluster_count\":0,\"resource_dashboard_count\":0,\"resource_app_count\":0,\"resources_metadata\":{\"resources\":[{\"resource_type\":\"jobs\",\"count\":1}]},\"experimental\":{\"configuration_file_count\":1,\"variable_count\":0,\"complex_variable_count\":0,\"lookup_variable_count\":0,\"target_count\":1,\"bool_values\":[{\"key\":\"local.cache.attempt\",\"value\":true},{\"key\":\"local.cache.hit\",\"value\":true},{\"key\":\"experimental.use_legacy_run_as\",\"value\":false},{\"key\":\"run_as_set\",\"value\":false},{\"key\":\"presets_name_prefix_is_set\",\"value\":false}],\"bundle_mode\":\"TYPE_UNSPECIFIED\",\"workspace_artifact_path_type\":\"WORKSPACE_FILE_SYSTEM\",\"bundle_mutator_execution_time_ms\":[{\"key\":\"phases.Initialize\",\"value\":4},{\"key\":\"resourcemutator.(processStaticResources)\",\"value\":1}],\"upload_file_count\":0}}}}}" + ] + } +} diff --git a/acceptance/bundle/dms/plan-without-dms/out.test.toml b/acceptance/bundle/dms/plan-without-dms/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/plan-without-dms/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/plan-without-dms/output.txt b/acceptance/bundle/dms/plan-without-dms/output.txt new file mode 100644 index 00000000000..1f049f181e8 --- /dev/null +++ b/acceptance/bundle/dms/plan-without-dms/output.txt @@ -0,0 +1,13 @@ + +=== Generate and save plan WITHOUT DMS recording +>>> jq {storage_backend, deployment_id, next_version_id} +{ + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "deployment_id": null, + "next_version_id": "1" +} + +=== Enable DMS recording and try to deploy the non-DMS plan +>>> musterr [CLI] bundle deploy --plan tmp.plan.corrupted.json +Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again + diff --git a/acceptance/bundle/dms/plan-without-dms/script b/acceptance/bundle/dms/plan-without-dms/script new file mode 100644 index 00000000000..9c4c8290ee3 --- /dev/null +++ b/acceptance/bundle/dms/plan-without-dms/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Generate and save plan WITHOUT DMS recording" +$CLI bundle plan -o json > tmp.plan.json +trace jq '{storage_backend, deployment_id, next_version_id}' < tmp.plan.json + +title "Enable DMS recording and try to deploy the non-DMS plan" +# Corrupt the plan by removing the storage_backend field to simulate a non-DMS plan +jq 'del(.storage_backend)' tmp.plan.json > tmp.plan.corrupted.json +# Now enable DMS in the config +update_file.py databricks.yml 'bundle:' 'experimental: + record_deployment_history: true + +bundle:' +trace musterr $CLI bundle deploy --plan tmp.plan.corrupted.json + +rm -f tmp.plan.json tmp.plan.corrupted.json diff --git a/acceptance/bundle/dms/plan-without-dms/test.toml b/acceptance/bundle/dms/plan-without-dms/test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/bundle/dms/plan-without-dms/test.toml @@ -0,0 +1 @@ +Cloud = false From a0730e25154141b675c950a66e2b8d753bb5f0da Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 16:01:17 +0200 Subject: [PATCH 076/164] Update golden file for plan-without-dms test Regenerate after timing variation in telemetry. Co-authored-by: Isaac --- acceptance/bundle/dms/plan-without-dms/out.requests.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/dms/plan-without-dms/out.requests.txt b/acceptance/bundle/dms/plan-without-dms/out.requests.txt index f39365de8a1..bf6798d6cf9 100644 --- a/acceptance/bundle/dms/plan-without-dms/out.requests.txt +++ b/acceptance/bundle/dms/plan-without-dms/out.requests.txt @@ -41,7 +41,7 @@ "method": "GET", "path": "/api/2.0/workspace/get-status", "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/resources.json", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/terraform.tfstate", "return_export_info": "true" } } @@ -49,7 +49,7 @@ "method": "GET", "path": "/api/2.0/workspace/get-status", "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/terraform.tfstate", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/resources.json", "return_export_info": "true" } } @@ -67,7 +67,7 @@ "uploadTime": [UNIX_TIME_MILLIS], "items": [], "protoLogs": [ - "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[UUID]\",\"version\":\"[CLI_VERSION]\",\"command\":\"bundle_deploy\",\"operating_system\":\"darwin\",\"execution_time_ms\":7,\"exit_code\":1},\"bundle_deploy_event\":{\"bundle_uuid\":\"[UUID]\",\"deployment_id\":\"[UUID]\",\"error_message\":\"this plan does not record deployment history, but the target now does; run 'bundle plan' again\",\"resource_count\":1,\"resource_job_count\":1,\"resource_pipeline_count\":0,\"resource_model_count\":0,\"resource_experiment_count\":0,\"resource_model_serving_endpoint_count\":0,\"resource_registered_model_count\":0,\"resource_quality_monitor_count\":0,\"resource_schema_count\":0,\"resource_volume_count\":0,\"resource_cluster_count\":0,\"resource_dashboard_count\":0,\"resource_app_count\":0,\"resources_metadata\":{\"resources\":[{\"resource_type\":\"jobs\",\"count\":1}]},\"experimental\":{\"configuration_file_count\":1,\"variable_count\":0,\"complex_variable_count\":0,\"lookup_variable_count\":0,\"target_count\":1,\"bool_values\":[{\"key\":\"local.cache.attempt\",\"value\":true},{\"key\":\"local.cache.hit\",\"value\":true},{\"key\":\"experimental.use_legacy_run_as\",\"value\":false},{\"key\":\"run_as_set\",\"value\":false},{\"key\":\"presets_name_prefix_is_set\",\"value\":false}],\"bundle_mode\":\"TYPE_UNSPECIFIED\",\"workspace_artifact_path_type\":\"WORKSPACE_FILE_SYSTEM\",\"bundle_mutator_execution_time_ms\":[{\"key\":\"phases.Initialize\",\"value\":4},{\"key\":\"resourcemutator.(processStaticResources)\",\"value\":1}],\"upload_file_count\":0}}}}}" + "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[UUID]\",\"version\":\"[CLI_VERSION]\",\"command\":\"bundle_deploy\",\"operating_system\":\"darwin\",\"execution_time_ms\":8,\"exit_code\":1},\"bundle_deploy_event\":{\"bundle_uuid\":\"[UUID]\",\"deployment_id\":\"[UUID]\",\"error_message\":\"this plan does not record deployment history, but the target now does; run 'bundle plan' again\",\"resource_count\":1,\"resource_job_count\":1,\"resource_pipeline_count\":0,\"resource_model_count\":0,\"resource_experiment_count\":0,\"resource_model_serving_endpoint_count\":0,\"resource_registered_model_count\":0,\"resource_quality_monitor_count\":0,\"resource_schema_count\":0,\"resource_volume_count\":0,\"resource_cluster_count\":0,\"resource_dashboard_count\":0,\"resource_app_count\":0,\"resources_metadata\":{\"resources\":[{\"resource_type\":\"jobs\",\"count\":1}]},\"experimental\":{\"configuration_file_count\":1,\"variable_count\":0,\"complex_variable_count\":0,\"lookup_variable_count\":0,\"target_count\":1,\"bool_values\":[{\"key\":\"local.cache.attempt\",\"value\":true},{\"key\":\"local.cache.hit\",\"value\":true},{\"key\":\"experimental.use_legacy_run_as\",\"value\":false},{\"key\":\"run_as_set\",\"value\":false},{\"key\":\"presets_name_prefix_is_set\",\"value\":false}],\"bundle_mode\":\"TYPE_UNSPECIFIED\",\"workspace_artifact_path_type\":\"WORKSPACE_FILE_SYSTEM\",\"bundle_mutator_execution_time_ms\":[{\"key\":\"phases.Initialize\",\"value\":4},{\"key\":\"resourcemutator.(processStaticResources)\",\"value\":2}],\"upload_file_count\":0}}}}}" ] } } From b5960ca8940c803e19c28e629844bfa606f9a593 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 16:03:07 +0200 Subject: [PATCH 077/164] Simplify plan-without-dms test to use print_requests.py - Remove out.requests.txt recording, use print_requests.py instead - Override DMS matrix to false so initial plan has no storage_backend - Clean output by removing requests file after test Co-authored-by: Isaac --- .../dms/plan-without-dms/out.requests.txt | 73 ------------------- .../bundle/dms/plan-without-dms/out.test.toml | 2 +- .../bundle/dms/plan-without-dms/output.txt | 6 +- acceptance/bundle/dms/plan-without-dms/script | 3 +- .../bundle/dms/plan-without-dms/test.toml | 3 + 5 files changed, 10 insertions(+), 77 deletions(-) delete mode 100644 acceptance/bundle/dms/plan-without-dms/out.requests.txt diff --git a/acceptance/bundle/dms/plan-without-dms/out.requests.txt b/acceptance/bundle/dms/plan-without-dms/out.requests.txt deleted file mode 100644 index bf6798d6cf9..00000000000 --- a/acceptance/bundle/dms/plan-without-dms/out.requests.txt +++ /dev/null @@ -1,73 +0,0 @@ -{ - "method": "GET", - "path": "/.well-known/databricks-config" -} -{ - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/resources.deployment.json" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-without-[UNIQUE_NAME]/default/state/resources.deployment.json" - } -} -{ - "method": "POST", - "path": "/telemetry-ext", - "body": { - "uploadTime": [UNIX_TIME_MILLIS], - "items": [], - "protoLogs": [ - "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[UUID]\",\"version\":\"[CLI_VERSION]\",\"command\":\"bundle_deploy\",\"operating_system\":\"darwin\",\"execution_time_ms\":8,\"exit_code\":1},\"bundle_deploy_event\":{\"bundle_uuid\":\"[UUID]\",\"deployment_id\":\"[UUID]\",\"error_message\":\"this plan does not record deployment history, but the target now does; run 'bundle plan' again\",\"resource_count\":1,\"resource_job_count\":1,\"resource_pipeline_count\":0,\"resource_model_count\":0,\"resource_experiment_count\":0,\"resource_model_serving_endpoint_count\":0,\"resource_registered_model_count\":0,\"resource_quality_monitor_count\":0,\"resource_schema_count\":0,\"resource_volume_count\":0,\"resource_cluster_count\":0,\"resource_dashboard_count\":0,\"resource_app_count\":0,\"resources_metadata\":{\"resources\":[{\"resource_type\":\"jobs\",\"count\":1}]},\"experimental\":{\"configuration_file_count\":1,\"variable_count\":0,\"complex_variable_count\":0,\"lookup_variable_count\":0,\"target_count\":1,\"bool_values\":[{\"key\":\"local.cache.attempt\",\"value\":true},{\"key\":\"local.cache.hit\",\"value\":true},{\"key\":\"experimental.use_legacy_run_as\",\"value\":false},{\"key\":\"run_as_set\",\"value\":false},{\"key\":\"presets_name_prefix_is_set\",\"value\":false}],\"bundle_mode\":\"TYPE_UNSPECIFIED\",\"workspace_artifact_path_type\":\"WORKSPACE_FILE_SYSTEM\",\"bundle_mutator_execution_time_ms\":[{\"key\":\"phases.Initialize\",\"value\":4},{\"key\":\"resourcemutator.(processStaticResources)\",\"value\":2}],\"upload_file_count\":0}}}}}" - ] - } -} diff --git a/acceptance/bundle/dms/plan-without-dms/out.test.toml b/acceptance/bundle/dms/plan-without-dms/out.test.toml index 9921e91a794..27ec2a7fcd6 100644 --- a/acceptance/bundle/dms/plan-without-dms/out.test.toml +++ b/acceptance/bundle/dms/plan-without-dms/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/plan-without-dms/output.txt b/acceptance/bundle/dms/plan-without-dms/output.txt index 1f049f181e8..b682a1756ab 100644 --- a/acceptance/bundle/dms/plan-without-dms/output.txt +++ b/acceptance/bundle/dms/plan-without-dms/output.txt @@ -2,12 +2,14 @@ === Generate and save plan WITHOUT DMS recording >>> jq {storage_backend, deployment_id, next_version_id} { - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "storage_backend": null, "deployment_id": null, - "next_version_id": "1" + "next_version_id": null } === Enable DMS recording and try to deploy the non-DMS plan >>> musterr [CLI] bundle deploy --plan tmp.plan.corrupted.json Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again + +>>> print_requests.py //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/plan-without-dms/script b/acceptance/bundle/dms/plan-without-dms/script index 9c4c8290ee3..5b55c88576d 100644 --- a/acceptance/bundle/dms/plan-without-dms/script +++ b/acceptance/bundle/dms/plan-without-dms/script @@ -13,5 +13,6 @@ update_file.py databricks.yml 'bundle:' 'experimental: bundle:' trace musterr $CLI bundle deploy --plan tmp.plan.corrupted.json +trace print_requests.py //api/2.0/bundle --get -rm -f tmp.plan.json tmp.plan.corrupted.json +rm -f tmp.plan.json tmp.plan.corrupted.json out.requests.txt diff --git a/acceptance/bundle/dms/plan-without-dms/test.toml b/acceptance/bundle/dms/plan-without-dms/test.toml index 18b1a88417e..d80809d201b 100644 --- a/acceptance/bundle/dms/plan-without-dms/test.toml +++ b/acceptance/bundle/dms/plan-without-dms/test.toml @@ -1 +1,4 @@ Cloud = false +RecordRequests = true +# Override inherited DMS matrix to disable it for the initial plan generation +EnvMatrix.DMS = [""] From 4024655763e0749c9b79a7c68087954c37bfa44e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 16:05:43 +0200 Subject: [PATCH 078/164] Remove unnecessary plan corruption from test Plan is already generated without DMS headers (EnvMatrix.DMS = [""]), so no need to manually corrupt it with jq. Deploy the plan directly with DMS enabled and expect the validation error. Co-authored-by: Isaac --- acceptance/bundle/dms/plan-without-dms/output.txt | 2 +- acceptance/bundle/dms/plan-without-dms/script | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/dms/plan-without-dms/output.txt b/acceptance/bundle/dms/plan-without-dms/output.txt index b682a1756ab..84718b40da8 100644 --- a/acceptance/bundle/dms/plan-without-dms/output.txt +++ b/acceptance/bundle/dms/plan-without-dms/output.txt @@ -8,7 +8,7 @@ } === Enable DMS recording and try to deploy the non-DMS plan ->>> musterr [CLI] bundle deploy --plan tmp.plan.corrupted.json +>>> musterr [CLI] bundle deploy --plan tmp.plan.json Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again diff --git a/acceptance/bundle/dms/plan-without-dms/script b/acceptance/bundle/dms/plan-without-dms/script index 5b55c88576d..fb1648d57ab 100644 --- a/acceptance/bundle/dms/plan-without-dms/script +++ b/acceptance/bundle/dms/plan-without-dms/script @@ -5,14 +5,11 @@ $CLI bundle plan -o json > tmp.plan.json trace jq '{storage_backend, deployment_id, next_version_id}' < tmp.plan.json title "Enable DMS recording and try to deploy the non-DMS plan" -# Corrupt the plan by removing the storage_backend field to simulate a non-DMS plan -jq 'del(.storage_backend)' tmp.plan.json > tmp.plan.corrupted.json -# Now enable DMS in the config update_file.py databricks.yml 'bundle:' 'experimental: record_deployment_history: true bundle:' -trace musterr $CLI bundle deploy --plan tmp.plan.corrupted.json +trace musterr $CLI bundle deploy --plan tmp.plan.json trace print_requests.py //api/2.0/bundle --get -rm -f tmp.plan.json tmp.plan.corrupted.json out.requests.txt +rm -f tmp.plan.json out.requests.txt From 61235855ca576012376d37e1420a87237897900b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 16:09:17 +0200 Subject: [PATCH 079/164] acceptance/bundle/deploy/readplan: Enable DMS testing for grants-remove-principal - Save raw plan to tmp.plan.json without nostamp (preserves DMS headers for deployment) - Apply nostamp only when generating the golden output file for comparison - Enables all DMS/READPLAN combinations to run and pass - Remove parent test.toml that only had DMS exclusion; inheritance from grandparent works fine This pattern allows serialized plan tests to work with DMS by keeping plan files unmodified for deployment (preserving deployment_id and version_id headers) while applying nostamp only to the golden output for deterministic comparisons. Co-authored-by: Isaac --- .../deploy/readplan/grants-remove-principal/out.test.toml | 2 +- .../bundle/deploy/readplan/grants-remove-principal/script | 4 ++-- acceptance/bundle/deploy/readplan/test.toml | 3 --- 3 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 acceptance/bundle/deploy/readplan/test.toml diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml index bf0f576652e..dd03baaabb6 100644 --- a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/script b/acceptance/bundle/deploy/readplan/grants-remove-principal/script index 541bbb8a506..e7dd63bce56 100644 --- a/acceptance/bundle/deploy/readplan/grants-remove-principal/script +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/script @@ -14,8 +14,8 @@ grep -v 'TO_REMOVE' databricks.yml > updated.yml && mv updated.yml databricks.ym # Generate update plan: entry.RemoteState captures both principals as GrantsState. # remote_state.__embed__ principal order is non-deterministic (the backend, and the # testserver mirroring it, iterate a map), so sort it by principal for a stable golden. -trace $CLI bundle plan -o json | nostamp > tmp.plan.json -jq '.plan["resources.schemas.myschema.grants"] | .remote_state.__embed__ |= sort_by(.principal)' tmp.plan.json > out.plan.grants.json +trace $CLI bundle plan -o json > tmp.plan.json +nostamp < tmp.plan.json | jq '.plan["resources.schemas.myschema.grants"] | .remote_state.__embed__ |= sort_by(.principal)' > out.plan.grants.json trace print_requests.py --get //unity-catalog/permissions --sort # Deploy from serialized plan (READPLAN=1) or without (READPLAN=""). diff --git a/acceptance/bundle/deploy/readplan/test.toml b/acceptance/bundle/deploy/readplan/test.toml deleted file mode 100644 index 576ff1ab001..00000000000 --- a/acceptance/bundle/deploy/readplan/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Dumps post-deploy state/plan, which carry the DMS deployment stamp under recording (applied at -# deploy, not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. -EnvMatrix.DMS = [""] From e767ed85eab79bec75440811775feb838f64d832 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 16:11:31 +0200 Subject: [PATCH 080/164] acceptance/bin: consolidate DMS helpers into print_state.py Move all DMS (deployment metadata service) functions from dms_resources.py into print_state.py to eliminate the circular dependency: print_state.py imported get_recorded_state from dms_resources.py, and dms_resources.py imported get_state_file from print_state.py. Since print_state.py acts as both a script and a library for other scripts, consolidating all DMS-related functionality there is the natural fit. Update all imports in dependent scripts (replace_ids.py, read_id.py, read_state.py) to import directly from print_state. This removes the circular import and keeps the helper functions co-located with their primary client. Co-authored-by: Isaac --- acceptance/bin/dms_resources.py | 117 -------------------------------- acceptance/bin/print_state.py | 103 ++++++++++++++++++++++++++-- acceptance/bin/read_id.py | 3 +- acceptance/bin/read_state.py | 2 +- acceptance/bin/replace_ids.py | 3 +- 5 files changed, 102 insertions(+), 126 deletions(-) delete mode 100755 acceptance/bin/dms_resources.py diff --git a/acceptance/bin/dms_resources.py b/acceptance/bin/dms_resources.py deleted file mode 100755 index 77b1da5b23f..00000000000 --- a/acceptance/bin/dms_resources.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 -""" -Read resource ids and state from the deployment metadata service. - -While a bundle records deployment history the service owns the resource set, so the state file is -not where ids and state come from. The service is asked instead, which takes two lookups: the CLI -keeps no deployment id locally, and the id is the object id of the workspace node the service -registers under /resources.deployment.json (see libs/dms/resolve.go). -""" - -import functools -import glob -import json -import os -import posixpath -import subprocess -import sys - -sys.path.insert(0, os.path.dirname(__file__)) -from print_state import get_state_file - -CLI = os.environ.get("CLI", "databricks") - -# Must match dms.DeploymentNodeName. -DEPLOYMENT_NODE_NAME = "resources.deployment.json" - - -def run_json(cmd, allow_failure=False): - """Run cmd and parse its stdout, or return None if it fails and allow_failure is set. stderr is - captured rather than inherited: these lookups are plumbing, and a CLI warning like "no files to - sync" would otherwise land in the test output.""" - result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") - if result.returncode != 0: - if allow_failure: - return None - raise SystemExit(f"{cmd} failed with code {result.returncode}\n{result.stdout}{result.stderr}".strip()) - return json.loads(result.stdout) - - -def records_deployment_history(): - """Whether this run records deployment history, so the service is what to ask.""" - return os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true" - - -def get_remote_state_path(target): - """The bundle's remote state directory. - - Preferred source is the sync snapshot, because it needs no CLI call: re-running the config - load would need whatever --var and flags the test deployed with, which a helper cannot know. - A bundle with no files to sync writes no snapshot, so fall back to asking the CLI - those - bundles are the ones with nothing to parameterize.""" - target_dir = os.path.dirname(get_state_file(target, False)) - snapshots = glob.glob(f"{target_dir}/sync-snapshots/*.json") - if snapshots: - # One snapshot per remote path, so a test that moved its root leaves several: the newest - # is the one the last deploy used. - newest = max(snapshots, key=os.path.getmtime) - remote_path = json.loads(open(newest).read())["remote_path"] - # state and files are siblings under the bundle root. - return posixpath.join(posixpath.dirname(remote_path), "state") - - args = [CLI, "bundle", "validate", "--output", "json"] - if target: - args += ["-t", target] - return run_json(args)["workspace"]["state_path"] - - -@functools.cache -def get_resources(target): - """Map every recorded resource key ("jobs.foo") to its {"id", "state"}. - - Empty when the bundle has no deployment recorded yet. Cached because a lookup costs three - round trips and a script asks for one resource at a time. - """ - state_path = get_remote_state_path(target) - if not state_path: - return {} - - # No node means nothing has been recorded, the conclusion dms.resolveDeploymentID also draws - # from a 404 - the deployment is gone once the bundle is destroyed. - node = run_json([CLI, "workspace", "get-status", f"{state_path}/{DEPLOYMENT_NODE_NAME}"], allow_failure=True) - if not node or not node.get("object_id"): - return {} - deployment_id = node["object_id"] - - result = {} - # The service pages at 50 resources; the local fake returns everything at once. - page_token = None - while True: - url = f"/api/2.0/bundle/deployments/{deployment_id}/resources" - if page_token: - url += f"?page_token={page_token}" - listed = run_json([CLI, "api", "get", url]) - for resource in listed.get("resources") or []: - # The service stores state as the opaque envelope the CLI wrote (dstate.RecordedState), - # so unwrap it to the resource state itself. - envelope = json.loads(resource["state"]) if resource.get("state") else {} - result[resource["resource_key"]] = { - "id": resource.get("resource_id"), - "state": envelope.get("state") or {}, - "depends_on": envelope.get("depends_on") or [], - } - page_token = listed.get("next_page_token") - if not page_token: - return result - - -def get_recorded_state(target): - """The recorded resources in the on-disk state file's `state` shape (resources. -> - {__id__, state, depends_on}), so a recording run prints the same shape as a non-recording one.""" - state = {} - for key, value in sorted(get_resources(target).items()): - entry = {"__id__": value["id"], "state": value["state"]} - if value["depends_on"]: - entry["depends_on"] = value["depends_on"] - state[f"resources.{key}"] = entry - return state diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 6055a3c5ecf..13bfe409134 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -1,15 +1,19 @@ #!/usr/bin/env python3 """ -Print resources state from default target. +Print resources state from default target, and handle DMS (deployment metadata service) interactions. Note, this intentionally has no logic on guessing what is the right state file (e.g. via DATABRICKS_BUNDLE_ENGINE), the goal is to record all states that are available. """ import argparse +import functools import glob import json import os +import posixpath +import subprocess +import sys def records_deployment_history(): @@ -58,15 +62,106 @@ def get_state_file(target, backup): return filtered[0] if filtered else result[0] +# DMS (deployment metadata service) interaction helpers +# https://go.databricks.com/dms-service + +CLI = os.environ.get("CLI", "databricks") +DEPLOYMENT_NODE_NAME = "resources.deployment.json" # Must match dms.DeploymentNodeName + + +def run_json(cmd, allow_failure=False): + """Run cmd and parse its stdout, or return None if it fails and allow_failure is set. + stderr is captured rather than inherited: these lookups are plumbing, and a CLI warning + like "no files to sync" would otherwise land in the test output.""" + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") + if result.returncode != 0: + if allow_failure: + return None + raise SystemExit(f"{cmd} failed with code {result.returncode}\n{result.stdout}{result.stderr}".strip()) + return json.loads(result.stdout) + + +def get_remote_state_path(target): + """The bundle's remote state directory. + + Preferred source is the sync snapshot, because it needs no CLI call: re-running the config + load would need whatever --var and flags the test deployed with, which a helper cannot know. + A bundle with no files to sync writes no snapshot, so fall back to asking the CLI - those + bundles are the ones with nothing to parameterize.""" + target_dir = os.path.dirname(get_state_file(target, False)) + snapshots = glob.glob(f"{target_dir}/sync-snapshots/*.json") + if snapshots: + # One snapshot per remote path, so a test that moved its root leaves several: the newest + # is the one the last deploy used. + newest = max(snapshots, key=os.path.getmtime) + remote_path = json.loads(open(newest).read())["remote_path"] + # state and files are siblings under the bundle root. + return posixpath.join(posixpath.dirname(remote_path), "state") + + args = [CLI, "bundle", "validate", "--output", "json"] + if target: + args += ["-t", target] + return run_json(args)["workspace"]["state_path"] + + +@functools.cache +def get_resources(target): + """Map every recorded resource key ("jobs.foo") to its {"id", "state"}. + + Empty when the bundle has no deployment recorded yet. Cached because a lookup costs three + round trips and a script asks for one resource at a time. + """ + state_path = get_remote_state_path(target) + if not state_path: + return {} + + # No node means nothing has been recorded, the conclusion dms.resolveDeploymentID also draws + # from a 404 - the deployment is gone once the bundle is destroyed. + node = run_json([CLI, "workspace", "get-status", f"{state_path}/{DEPLOYMENT_NODE_NAME}"], allow_failure=True) + if not node or not node.get("object_id"): + return {} + deployment_id = node["object_id"] + + result = {} + # The service pages at 50 resources; the local fake returns everything at once. + page_token = None + while True: + url = f"/api/2.0/bundle/deployments/{deployment_id}/resources" + if page_token: + url += f"?page_token={page_token}" + listed = run_json([CLI, "api", "get", url]) + for resource in listed.get("resources") or []: + # The service stores state as the opaque envelope the CLI wrote (dstate.RecordedState), + # so unwrap it to the resource state itself. + envelope = json.loads(resource["state"]) if resource.get("state") else {} + result[resource["resource_key"]] = { + "id": resource.get("resource_id"), + "state": envelope.get("state") or {}, + "depends_on": envelope.get("depends_on") or [], + } + page_token = listed.get("next_page_token") + if not page_token: + return result + + +def get_recorded_state(target): + """The recorded resources in the on-disk state file's `state` shape (resources. -> + {__id__, state, depends_on}), so a recording run prints the same shape as a non-recording one.""" + state = {} + for key, value in sorted(get_resources(target).items()): + entry = {"__id__": value["id"], "state": value["state"]} + if value["depends_on"]: + entry["depends_on"] = value["depends_on"] + state[f"resources.{key}"] = entry + return state + + def print_recorded_state(filename, target): """Print the state file with its resources filled in from the deployment metadata service. While recording, the file itself carries only the header - the service holds the resources - so printing it raw would show an empty state and differ from the same test's non-recording run. """ - # Imported here rather than at module level: dms_resources reads get_state_file from this module. - from dms_resources import get_recorded_state - data = json.loads(open(filename).read()) data["state"] = get_recorded_state(target) print(json.dumps(data, indent=1)) diff --git a/acceptance/bin/read_id.py b/acceptance/bin/read_id.py index 06e7b69281e..6e11a3d866c 100755 --- a/acceptance/bin/read_id.py +++ b/acceptance/bin/read_id.py @@ -15,8 +15,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from add_repl import add_repl -from dms_resources import get_resources, records_deployment_history -from print_state import get_state_file +from print_state import get_resources, records_deployment_history, get_state_file def get_id_terraform(filename, name): diff --git a/acceptance/bin/read_state.py b/acceptance/bin/read_state.py index 4d9bf84186e..64c4d54832d 100755 --- a/acceptance/bin/read_state.py +++ b/acceptance/bin/read_state.py @@ -10,7 +10,7 @@ import sys sys.path.insert(0, os.path.dirname(__file__)) -from dms_resources import get_resources, records_deployment_history +from print_state import get_resources, records_deployment_history def print_resource_terraform(group, name, *attrs): diff --git a/acceptance/bin/replace_ids.py b/acceptance/bin/replace_ids.py index 2f5d8165032..9d6d6bcdfce 100755 --- a/acceptance/bin/replace_ids.py +++ b/acceptance/bin/replace_ids.py @@ -10,8 +10,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from add_repl import add_repl -from dms_resources import get_resources, records_deployment_history -from print_state import get_state_file +from print_state import get_resources, records_deployment_history, get_state_file def iter_ids_terraform(filename): From bde2b0ebb0d3d27fa8afa62da87d4558f0fe0947 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 16:14:29 +0200 Subject: [PATCH 081/164] acceptance/bundle/deploy/readplan: Fix tests to work with DMS by using raw plan + nostamp pattern Apply the established pattern for serialized plan tests: - Save raw plan to tmp file (preserves DMS headers for deployment validation) - Apply nostamp only when generating golden output files - Add tmp files to Ignore list in test.toml Fixed tests: - basic: now passes all DMS/READPLAN combinations - postgres_role: now passes all DMS/READPLAN combinations - whl-not-uploaded: now passes all DMS/READPLAN combinations - cli-version-mismatch: now passes all DMS/READPLAN combinations - serial-mismatch: now passes all DMS/READPLAN combinations Excluded from DMS testing (use committed plans without DMS headers): - lineage-mismatch: excluded DMS=true (tests lineage validation, not DMS) All readplan tests now inherit DMS matrix from bundle level (DMS=["" , "true"]), enabling comprehensive DMS coverage across all test variants. Co-authored-by: Isaac --- acceptance/bundle/deploy/readplan/basic/out.test.toml | 2 +- acceptance/bundle/deploy/readplan/basic/script | 10 ++++++---- acceptance/bundle/deploy/readplan/basic/test.toml | 1 + .../deploy/readplan/cli-version-mismatch/out.test.toml | 2 +- .../bundle/deploy/readplan/cli-version-mismatch/script | 4 ++-- .../deploy/readplan/cli-version-mismatch/test.toml | 1 + .../bundle/deploy/readplan/invalid-plan/out.test.toml | 2 +- .../deploy/readplan/lineage-mismatch/out.test.toml | 2 +- .../bundle/deploy/readplan/lineage-mismatch/test.toml | 2 ++ .../deploy/readplan/plan-not-found/out.test.toml | 2 +- .../readplan/plan-version-mismatch/out.test.toml | 2 +- .../bundle/deploy/readplan/postgres_role/out.test.toml | 2 +- acceptance/bundle/deploy/readplan/postgres_role/script | 6 +++--- .../bundle/deploy/readplan/postgres_role/test.toml | 2 +- .../deploy/readplan/serial-mismatch/out.test.toml | 2 +- .../bundle/deploy/readplan/serial-mismatch/script | 6 +++--- .../bundle/deploy/readplan/serial-mismatch/test.toml | 1 + .../deploy/readplan/terraform-error/out.test.toml | 2 +- .../bundle/deploy/readplan/unknown-field/out.test.toml | 2 +- .../deploy/readplan/whl-not-uploaded/out.test.toml | 2 +- .../bundle/deploy/readplan/whl-not-uploaded/script | 5 +++-- .../bundle/deploy/readplan/whl-not-uploaded/test.toml | 2 +- 22 files changed, 35 insertions(+), 27 deletions(-) diff --git a/acceptance/bundle/deploy/readplan/basic/out.test.toml b/acceptance/bundle/deploy/readplan/basic/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.test.toml +++ b/acceptance/bundle/deploy/readplan/basic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/basic/script b/acceptance/bundle/deploy/readplan/basic/script index f7dc5343efc..3ea3a23f500 100644 --- a/acceptance/bundle/deploy/readplan/basic/script +++ b/acceptance/bundle/deploy/readplan/basic/script @@ -1,16 +1,18 @@ # Generate initial plan -trace $CLI bundle plan -o json | nostamp > out.plan_create.json +trace $CLI bundle plan -o json > tmp.plan_create.json +nostamp < tmp.plan_create.json > out.plan_create.json trace print_requests.py --get //jobs | contains.py "!GET" "!POST" # First deploy to create the resource -trace $CLI bundle deploy --plan out.plan_create.json +trace $CLI bundle deploy --plan tmp.plan_create.json trace print_requests.py --get //jobs | contains.py "!GET" "POST" # Generate a plan (should show "skip" since nothing changed) -trace $CLI bundle plan -o json | nostamp > out.plan_skip.json +trace $CLI bundle plan -o json > tmp.plan_skip.json +nostamp < tmp.plan_skip.json > out.plan_skip.json trace print_requests.py --get //jobs | contains.py "GET" -trace $CLI bundle deploy --plan out.plan_skip.json +trace $CLI bundle deploy --plan tmp.plan_skip.json # Show what requests were made (should not create/update job again; should not read remote state) trace print_requests.py --get //jobs | contains.py "!GET" "!POST" diff --git a/acceptance/bundle/deploy/readplan/basic/test.toml b/acceptance/bundle/deploy/readplan/basic/test.toml index ab71315de0d..c026765cce2 100644 --- a/acceptance/bundle/deploy/readplan/basic/test.toml +++ b/acceptance/bundle/deploy/readplan/basic/test.toml @@ -1,3 +1,4 @@ # Test basic --plan functionality EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = true +Ignore = ["tmp.plan_*.json"] diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/script b/acceptance/bundle/deploy/readplan/cli-version-mismatch/script index e778bba5d18..e9db79b8ec0 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/script +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/script @@ -1,6 +1,6 @@ -trace $CLI bundle plan -o json | nostamp > plan.json +trace $CLI bundle plan -o json > tmp.plan.json -jq '.cli_version = "1.2.3"' plan.json > plan.json.tmp && mv plan.json.tmp plan.json +jq '.cli_version = "1.2.3"' tmp.plan.json > plan.json trace $CLI bundle deploy --plan plan.json rm plan.json diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/test.toml b/acceptance/bundle/deploy/readplan/cli-version-mismatch/test.toml index c8f942fbf84..f69f2af077d 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/test.toml +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/test.toml @@ -1,3 +1,4 @@ # Test that --plan with different cli_version produces a warning but succeeds EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = true +Ignore = ["tmp.plan.json"] diff --git a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml +++ b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml index de0ba18b25f..7591246cc57 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml @@ -1,2 +1,4 @@ # Test that --plan with mismatched lineage produces an error EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +# Committed plan.json lacks DMS headers, so skip DMS=true +EnvMatrixExclude.plan_no_dms = ["DMS=true"] diff --git a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml index 47a2f0c8b42..429e714df15 100644 --- a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml +++ b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/postgres_role/script b/acceptance/bundle/deploy/readplan/postgres_role/script index f4b16d8069c..8d46d700aef 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/script +++ b/acceptance/bundle/deploy/readplan/postgres_role/script @@ -6,12 +6,12 @@ # empty role_id, the server would generate its own, and the next plan would # report phantom "recreate" drift on the immutable role_id. -trace $CLI bundle plan -o json | nostamp > plan.json +trace $CLI bundle plan -o json > tmp.plan.json # Keep only the role entry; role_id and parent must survive serialization. -trace jq '.plan."resources.postgres_roles.role"' plan.json > out.plan.role.json +nostamp < tmp.plan.json | jq '.plan."resources.postgres_roles.role"' > out.plan.role.json -trace $CLI bundle deploy --plan plan.json +trace $CLI bundle deploy --plan tmp.plan.json # The create request must carry the configured role_id, not an empty one. trace print_requests.py //roles diff --git a/acceptance/bundle/deploy/readplan/postgres_role/test.toml b/acceptance/bundle/deploy/readplan/postgres_role/test.toml index 1f5b6b228a1..babde706a4f 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/test.toml +++ b/acceptance/bundle/deploy/readplan/postgres_role/test.toml @@ -2,4 +2,4 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = true # Full plan fed to `deploy --plan`; the committed golden keeps only the role entry. -Ignore = ["plan.json"] +Ignore = ["tmp.plan.json"] diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/script b/acceptance/bundle/deploy/readplan/serial-mismatch/script index a3a64529a1d..4f37c55f218 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/script +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/script @@ -2,13 +2,13 @@ trace $CLI bundle deploy # Generate a plan (captures current lineage and serial) -trace $CLI bundle plan -o json | nostamp > plan.json +trace $CLI bundle plan -o json > tmp.plan.json # Modify only the serial to simulate stale plan -jq '.serial = 999' plan.json > plan.json.tmp && mv plan.json.tmp plan.json +jq '.serial = 999' tmp.plan.json > plan.json # Try to deploy with stale plan (should fail with serial mismatch) musterr $CLI bundle deploy --plan plan.json -rm plan.json +rm plan.json tmp.plan.json trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/test.toml b/acceptance/bundle/deploy/readplan/serial-mismatch/test.toml index dc4dcf06fc7..a35ede05603 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/test.toml +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/test.toml @@ -1,3 +1,4 @@ # Test that --plan with mismatched serial produces an error EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = false +Ignore = ["tmp.plan.json"] diff --git a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml index a52570d2eee..5d0f3e4aac2 100644 --- a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml +++ b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml +++ b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script index 935ea678077..042936bede3 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script @@ -15,8 +15,9 @@ trace $CLI bundle destroy --auto-approve rm out.requests.txt title "bundle deploy --plan does NOT upload the wheel (bug)" -trace $CLI bundle plan -o json | nostamp > out.plan.json -trace $CLI bundle deploy --plan out.plan.json +trace $CLI bundle plan -o json > tmp.plan.json +nostamp < tmp.plan.json > out.plan.json +trace $CLI bundle deploy --plan tmp.plan.json # BUG: no import-file request for the wheel here (output below is empty), yet the job was # created pointing the task at the same .../artifacts/.internal/ path that was never uploaded. trace print_requests.py //artifacts/.internal --del-field raw_body diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml b/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml index 9325fa161c0..f971f7bb2ae 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml @@ -3,4 +3,4 @@ Badness = "bundle deploy --plan skips the build phase, so a local wheel referenc # --plan is direct-engine only. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = true -Ignore = [".databricks"] +Ignore = [".databricks", "tmp.plan.json"] From b0ae7f220c972657ee3e41118fcfa75ae2ea6c12 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 16:15:12 +0200 Subject: [PATCH 082/164] acceptance/bundle/deploy/readplan: Regenerate outputs for fixed tests --- acceptance/bundle/deploy/readplan/basic/output.txt | 8 ++++---- .../deploy/readplan/cli-version-mismatch/output.txt | 2 +- .../bundle/deploy/readplan/lineage-mismatch/out.test.toml | 2 +- .../bundle/deploy/readplan/postgres_role/output.txt | 4 +--- .../bundle/deploy/readplan/whl-not-uploaded/output.txt | 4 ++-- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/deploy/readplan/basic/output.txt b/acceptance/bundle/deploy/readplan/basic/output.txt index 5211ba04b57..d2e2627c79b 100644 --- a/acceptance/bundle/deploy/readplan/basic/output.txt +++ b/acceptance/bundle/deploy/readplan/basic/output.txt @@ -3,10 +3,10 @@ >>> print_requests.py --get //jobs ->>> [CLI] bundle deploy --plan out.plan_create.json +>>> [CLI] bundle deploy --plan tmp.plan_create.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.job -Files: 6 uploaded, 0 deleted +Files: 7 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --get //jobs @@ -39,9 +39,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } ->>> [CLI] bundle deploy --plan out.plan_skip.json +>>> [CLI] bundle deploy --plan tmp.plan_skip.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Files: 3 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> print_requests.py --get //jobs diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/output.txt b/acceptance/bundle/deploy/readplan/cli-version-mismatch/output.txt index 7d6f07a44cd..a20a3149cb1 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/output.txt +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/output.txt @@ -5,7 +5,7 @@ Warn: Plan was created with CLI version 1.2.3 but current version is [CLI_VERSION] Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.job -Files: 6 uploaded, 0 deleted +Files: 7 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "method": "POST", diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/readplan/postgres_role/output.txt b/acceptance/bundle/deploy/readplan/postgres_role/output.txt index 5a422d8e993..4d42b75aaea 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/output.txt +++ b/acceptance/bundle/deploy/readplan/postgres_role/output.txt @@ -1,9 +1,7 @@ >>> [CLI] bundle plan -o json ->>> jq .plan."resources.postgres_roles.role" plan.json - ->>> [CLI] bundle deploy --plan plan.json +>>> [CLI] bundle deploy --plan tmp.plan.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created postgres_branches.branch Created postgres_projects.project diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt b/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt index 73e36e6a842..b8606589cbf 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt @@ -27,10 +27,10 @@ Destroy: 1 deleted === bundle deploy --plan does NOT upload the wheel (bug) >>> [CLI] bundle plan -o json ->>> [CLI] bundle deploy --plan out.plan.json +>>> [CLI] bundle deploy --plan tmp.plan.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/files... Created jobs.my_job -Files: 7 uploaded, 0 deleted +Files: 8 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //artifacts/.internal --del-field raw_body From 633e35714616e8a44e298c73a7cfb9488e721750 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 16:18:04 +0200 Subject: [PATCH 083/164] acceptance/readplan: normalize recorded requests with nostamp under DMS The job POST bodies carry the deployment stamp when recording, so pipe print_requests.py through nostamp instead of adding Repls entries. The plan itself already goes to a tmp file, so deploy --plan keeps its DMS headers. Co-authored-by: Isaac --- acceptance/bundle/deploy/readplan/basic/script | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/deploy/readplan/basic/script b/acceptance/bundle/deploy/readplan/basic/script index 3ea3a23f500..b73ca75bf25 100644 --- a/acceptance/bundle/deploy/readplan/basic/script +++ b/acceptance/bundle/deploy/readplan/basic/script @@ -1,20 +1,20 @@ # Generate initial plan trace $CLI bundle plan -o json > tmp.plan_create.json nostamp < tmp.plan_create.json > out.plan_create.json -trace print_requests.py --get //jobs | contains.py "!GET" "!POST" +trace print_requests.py --get //jobs | nostamp | contains.py "!GET" "!POST" # First deploy to create the resource trace $CLI bundle deploy --plan tmp.plan_create.json -trace print_requests.py --get //jobs | contains.py "!GET" "POST" +trace print_requests.py --get //jobs | nostamp | contains.py "!GET" "POST" # Generate a plan (should show "skip" since nothing changed) trace $CLI bundle plan -o json > tmp.plan_skip.json nostamp < tmp.plan_skip.json > out.plan_skip.json -trace print_requests.py --get //jobs | contains.py "GET" +trace print_requests.py --get //jobs | nostamp | contains.py "GET" trace $CLI bundle deploy --plan tmp.plan_skip.json # Show what requests were made (should not create/update job again; should not read remote state) -trace print_requests.py --get //jobs | contains.py "!GET" "!POST" +trace print_requests.py --get //jobs | nostamp | contains.py "!GET" "!POST" trace $CLI bundle destroy --auto-approve rm out.requests.txt From e55afd5dc4c79e2bd8a1de75fab187e830ca2ff0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 16:20:55 +0200 Subject: [PATCH 084/164] acceptance/readplan: nostamp recorded requests in cli-version-mismatch Last readplan test still diverging under DMS: the job create body carries the deployment stamp. Same fix as basic - pipe print_requests.py through nostamp. The whole deploy/readplan subtree now passes with DMS enabled. Co-authored-by: Isaac --- acceptance/bundle/deploy/readplan/cli-version-mismatch/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/script b/acceptance/bundle/deploy/readplan/cli-version-mismatch/script index e9db79b8ec0..75122c8abee 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/script +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/script @@ -4,6 +4,6 @@ jq '.cli_version = "1.2.3"' tmp.plan.json > plan.json trace $CLI bundle deploy --plan plan.json rm plan.json -print_requests.py //jobs +print_requests.py //jobs | nostamp trace $CLI bundle destroy --auto-approve rm out.requests.txt From aab63370a1d54fe951937179732901cd275dcbc1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 16:30:32 +0200 Subject: [PATCH 085/164] acceptance/bin: inline the deployment-history env check Drop records_deployment_history() and read DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY directly at each call site, matching the DATABRICKS_BUNDLE_ENGINE check right below it in read_state.py. Addresses review feedback. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 6 +----- acceptance/bin/read_id.py | 5 +++-- acceptance/bin/read_state.py | 4 ++-- acceptance/bin/replace_ids.py | 5 +++-- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 13bfe409134..5a2cd80b697 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -16,10 +16,6 @@ import sys -def records_deployment_history(): - return os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true" - - def print_file(filename): data = open(filename).read() print(data, end="") @@ -182,7 +178,7 @@ def main(): for filename in get_state_files(args.target, args.backup): if not os.path.exists(filename): continue - if records_deployment_history() and not args.no_dms: + if os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true" and not args.no_dms: print_recorded_state(filename, args.target) else: print_file(filename) diff --git a/acceptance/bin/read_id.py b/acceptance/bin/read_id.py index 6e11a3d866c..2d7434d3a7d 100755 --- a/acceptance/bin/read_id.py +++ b/acceptance/bin/read_id.py @@ -10,12 +10,13 @@ import argparse import json +import os import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) from add_repl import add_repl -from print_state import get_resources, records_deployment_history, get_state_file +from print_state import get_resources, get_state_file def get_id_terraform(filename, name): @@ -62,7 +63,7 @@ def main(): parser.add_argument("name") args = parser.parse_args() - if records_deployment_history(): + if os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true": id = get_id_recorded(args.target, args.name) else: filename = get_state_file(args.target, args.backup) diff --git a/acceptance/bin/read_state.py b/acceptance/bin/read_state.py index 64c4d54832d..11d6e4b7b4d 100755 --- a/acceptance/bin/read_state.py +++ b/acceptance/bin/read_state.py @@ -10,7 +10,7 @@ import sys sys.path.insert(0, os.path.dirname(__file__)) -from print_state import get_resources, records_deployment_history +from print_state import get_resources def print_resource_terraform(group, name, *attrs): @@ -65,7 +65,7 @@ def print_resource_recorded(group, name, *attrs): print(group, name, " ".join(values)) -if records_deployment_history(): +if os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true": print_resource_recorded(*sys.argv[1:]) elif os.environ.get("DATABRICKS_BUNDLE_ENGINE", "").startswith("direct"): print_resource_direct(*sys.argv[1:]) diff --git a/acceptance/bin/replace_ids.py b/acceptance/bin/replace_ids.py index 9d6d6bcdfce..09a04133610 100755 --- a/acceptance/bin/replace_ids.py +++ b/acceptance/bin/replace_ids.py @@ -5,12 +5,13 @@ import argparse import json +import os import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) from add_repl import add_repl -from print_state import get_resources, records_deployment_history, get_state_file +from print_state import get_resources, get_state_file def iter_ids_terraform(filename): @@ -50,7 +51,7 @@ def main(): parser.add_argument("--backup", action="store_true") args = parser.parse_args() - if records_deployment_history(): + if os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true": it = iter_ids_recorded(args.target) else: filename = get_state_file(args.target, args.backup) From 92c61c5ed94f7308b0ab3bf70c628eae9a34b8b3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 17:24:16 +0200 Subject: [PATCH 086/164] Revert first-deploy version stamping and the extra plan validation A first-deploy plan has no deployment yet, so CalculatePlan must not stamp NextVersionId = "1": doing so broke bundle/dms/depends-on and stale-plan, which assert that a pre-first-deploy plan carries no version fields. Drop the accompanying validatePlan checks too. The existing storage_backend mode check ("this plan does not record deployment history, but the target now does") plus the plan.DeploymentId comparison already reject a stale or wrong-mode plan; bundle/dms/plan-without-dms covers that path. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 4 ---- cmd/bundle/utils/process.go | 11 ----------- 2 files changed, 15 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index c79191abdb0..c11152cab26 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -180,10 +180,6 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks plan.DeploymentId = b.StateDB.DeploymentID plan.LastVersionId = b.StateDB.LatestVersionID plan.NextVersionId = strconv.FormatInt(next, 10) - } else { - // First deployment: version will be "1" with no prior version. - plan.NextVersionId = "1" - plan.LastVersionId = "" } } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 7a6e873db6d..f223abc82d4 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -496,17 +496,6 @@ func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, if plan.DeploymentId != dmsDeploymentID { return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") } - - // For DMS plans, validate that all required fields are present. - // NextVersionId is always set: "1" for first deploy, or the computed next version for subsequent deploys. - // DeploymentId and LastVersionId are only set for subsequent deployments. - if plan.NextVersionId == "" { - return errors.New("this plan is missing the deployment version ID required for deployment history recording; run 'bundle plan' again") - } - // For non-first deployments (version > "1"), both deployment ID and last version ID must be set. - if plan.NextVersionId != "1" && plan.DeploymentId == "" { - return errors.New("this plan is missing the deployment ID required for deployment history recording; run 'bundle plan' again") - } } // Validate that the plan's lineage and serial match the local state. This is the stale guard for From 37e5d6ba922bb02f14b30325f2f4eed6f8907372 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 17:32:56 +0200 Subject: [PATCH 087/164] acceptance: match PLAN_VERSION repl to plan version 3; drop unused sys import main normalizes "plan_version": 2 to [PLAN_VERSION] so a version bump does not churn every recorded plan. This branch bumps the plan version to 3, so the repl no longer matched and goldens diverged. Point it at 3 and regenerate the DMS goldens that held the literal. Also remove the now-unused `import sys` from print_state.py (ruff F401, CI lint). Co-authored-by: Isaac --- acceptance/bin/print_state.py | 1 - acceptance/bundle/dms/declined-deploy/output.txt | 2 +- acceptance/bundle/dms/depends-on/output.txt | 4 ++-- acceptance/bundle/dms/deployment-metadata-change/output.txt | 4 ++-- acceptance/bundle/dms/empty-grants/output.txt | 2 +- acceptance/bundle/dms/existing-state/output.txt | 2 +- acceptance/bundle/dms/failed-delete/output.txt | 2 +- acceptance/bundle/dms/failed-recreate/output.txt | 2 +- acceptance/bundle/dms/failed-update-permissions/output.txt | 2 +- acceptance/bundle/dms/multiple-resources/output.txt | 4 ++-- acceptance/bundle/dms/no-drift/output.txt | 4 ++-- acceptance/bundle/dms/no-resources/output.txt | 4 ++-- acceptance/bundle/dms/operation-upload-fails-once/output.txt | 2 +- acceptance/bundle/dms/provenance/output.txt | 2 +- acceptance/bundle/dms/record-failure/output.txt | 2 +- acceptance/bundle/dms/stale-plan/output.txt | 2 +- acceptance/bundle/dms/successful-recreate/output.txt | 4 ++-- acceptance/bundle/dms/version-never-created/output.txt | 2 +- acceptance/test.toml | 4 ++-- 19 files changed, 25 insertions(+), 26 deletions(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 5a2cd80b697..8eec03940c7 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -13,7 +13,6 @@ import os import posixpath import subprocess -import sys def print_file(filename): diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 2aa56f9ab78..abe3950b66c 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -62,7 +62,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1 >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index ae9e3cda49e..fe37a153f94 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -2,7 +2,7 @@ === Plan before first deploy: no deployment yet, so no deployment_id or version fields >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } @@ -54,7 +54,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 8d1199d14c1..556c9a1f980 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -62,7 +62,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1 >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -143,7 +143,7 @@ A common practice is to use a username or principal name in this path, i.e. use root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index 12373101601..12f5c5a56aa 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -133,7 +133,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index b4c1b705c28..08073ddeac9 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -8,7 +8,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1 diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index c196fc9e64e..cef0978f56e 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -62,7 +62,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index d5ae120eec7..99b280227ab 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -62,7 +62,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index e7bf63ed164..e947a7aeb1f 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -10,7 +10,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === The plan records DEPLOYMENT_METADATA_SERVICE backend and the job >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index bf9a24cba12..607d3663a43 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -20,7 +20,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged === The plan records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -89,7 +89,7 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged === The plan records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index f37562b84f0..ca653c21fe5 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -30,7 +30,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -165,7 +165,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged === The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA_SERVICE as backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 22b9703b44a..cb774043dd6 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -47,7 +47,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === The plan records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "next_version_id": "2", @@ -123,7 +123,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === The plan records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "next_version_id": "3", diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index b1a6a64ea06..c658c1b380c 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -9,7 +9,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan records DEPLOYMENT_METADATA_SERVICE backend and the schema >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 57fee97f7fc..5cde7587967 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -54,7 +54,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index c03c9cfda2a..0dc1e64891c 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -13,7 +13,7 @@ Files: 6 uploaded, 0 deleted >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "next_version_id": "2", diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index f15171aa94e..ce52df879cc 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -2,7 +2,7 @@ === Save a plan, then deploy it: records version 1 >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index db1033110d2..e8f16c3d5b6 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -62,7 +62,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -166,7 +166,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged === The plan now shows the recreate action >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 6a963c0ff82..f917f962000 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -13,7 +13,7 @@ Files: 5 uploaded, 0 deleted >>> [CLI] bundle plan -o json { - "plan_version": 3, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "next_version_id": "1", diff --git a/acceptance/test.toml b/acceptance/test.toml index a732c2db784..68eb1f55c5a 100644 --- a/acceptance/test.toml +++ b/acceptance/test.toml @@ -95,9 +95,9 @@ New = "[METASTORE_NAME]" # Normalize plan_version so bumping it doesn't churn every recorded plan. [[Repls]] -Old = '"plan_version": 2,' +Old = '"plan_version": 3,' New = '"plan_version": [PLAN_VERSION],' [[Repls]] -Old = 'json\.plan_version = 2;' +Old = 'json\.plan_version = 3;' New = 'json.plan_version = [PLAN_VERSION];' From 83c4fb824dda8105d895b8e404e93c76098e0c69 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 17:52:59 +0200 Subject: [PATCH 088/164] acceptance/bin: document why get_id_recorded lists instead of GET Moves the explanation from PR review into the code: the GET resource API needs the full resource key ("jobs.foo") but callers pass only the leaf name, mirroring how user-visible commands take a bare name. Notes that switching to GET is a reasonable follow-up and likely needs the service to key resources by name. Co-authored-by: Isaac --- acceptance/bin/read_id.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acceptance/bin/read_id.py b/acceptance/bin/read_id.py index 2d7434d3a7d..6876455965b 100755 --- a/acceptance/bin/read_id.py +++ b/acceptance/bin/read_id.py @@ -36,6 +36,14 @@ def get_id_terraform(filename, name): def get_id_recorded(target, name): + """Find a recorded resource's id by its bare name ("foo"), scanning the resource list. + + The GET resource API would be the direct lookup, but it needs the full resource key + ("jobs.foo"), and callers pass only the leaf name - matching how user-visible commands + take a bare name ("bundle run my_job"), which DABs allows because it enforces name + uniqueness across types. So list and match on the leaf instead. Switching to GET is a + reasonable follow-up; it likely needs the service to key resources by name as well. + """ resources = get_resources(target) for key, value in resources.items(): if key.split(".")[1] == name: From 0e452f4169502b75ee22d2f93ccb3b0fe9e3a18b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 17:59:55 +0200 Subject: [PATCH 089/164] acceptance: stop nostamping plans that feed deploy --plan These files (plan_*.json, tmp.plan*.json) are inputs to `bundle deploy --plan`, not goldens, so they must stay raw: nostamp strips storage_backend and the version fields the deploy path validates. It was also swallowing the CLI's stderr, since `2>LOG` bound to nostamp rather than to the plan command. Co-authored-by: Isaac --- .../bundle/deployment/bind/dashboard/recreation/script | 4 ++-- acceptance/bundle/deployment/bind/dashboard/script | 2 +- acceptance/bundle/invariant/delete_idempotent/script | 6 +++--- acceptance/bundle/invariant/destroy_idempotent/script | 2 +- acceptance/bundle/resources/apps/readplan-lifecycle/script | 2 +- .../templates/default-python/combinations/classic/script | 4 ++-- .../templates/default-python/combinations/serverless/script | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/script b/acceptance/bundle/deployment/bind/dashboard/recreation/script index 33e7869713f..ce63de92924 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/script +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/script @@ -20,13 +20,13 @@ add_repl $ETAG ETAG json_in_json_normalize.py out.state_after_bind.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > tmp.plan.json +$CLI bundle plan -o json > tmp.plan.json errcode $CLI bundle deploy $(readplanarg tmp.plan.json) trace $CLI bundle deployment unbind dashboard1 # Re-plan after unbind: the plan saved above no longer matches state. -$CLI bundle plan -o json | nostamp > tmp.plan_after_unbind.json +$CLI bundle plan -o json > tmp.plan_after_unbind.json $CLI bundle deploy --auto-approve $(readplanarg tmp.plan_after_unbind.json) trace $CLI lakeview get "${DASHBOARD_ID}" | jq '{display_name, lifecycle_state}' diff --git a/acceptance/bundle/deployment/bind/dashboard/script b/acceptance/bundle/deployment/bind/dashboard/script index 130ba8d74a3..3d4163a947f 100644 --- a/acceptance/bundle/deployment/bind/dashboard/script +++ b/acceptance/bundle/deployment/bind/dashboard/script @@ -15,7 +15,7 @@ trace $CLI bundle deployment bind dashboard1 "${DASHBOARD_ID}" --auto-approve # Bind writes state without going through DoCreate/DoUpdate, so a saved-plan deploy after # a bind must still send the full serialized_dashboard; the lakeview get below reads it back. -$CLI bundle plan -o json | nostamp > tmp.plan.json +$CLI bundle plan -o json > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) trace $CLI lakeview get "${DASHBOARD_ID}" | jq --sort-keys '{display_name, lifecycle_state, path, parent_path, serialized_dashboard} | .serialized_dashboard |= fromjson' diff --git a/acceptance/bundle/invariant/delete_idempotent/script b/acceptance/bundle/invariant/delete_idempotent/script index 50937aca2b2..aba945dbb2e 100644 --- a/acceptance/bundle/invariant/delete_idempotent/script +++ b/acceptance/bundle/invariant/delete_idempotent/script @@ -12,7 +12,7 @@ STATE_PATH=/Workspace/Users/$CURRENT_USER_NAME/.bundle/$bundle_name/default/stat # Initial deploy of the resources. Only pre-plan when READPLAN=1 exercises the # saved-plan deploy path; otherwise `bundle deploy` plans on its own. if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json | nostamp > plan_initial.json 2>LOG.plan_initial.err + $CLI bundle plan -o json > plan_initial.json 2>LOG.plan_initial.err cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null fi @@ -30,7 +30,7 @@ resources: {} EOF if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json | nostamp > plan_delete.json 2>LOG.plan_delete.err + $CLI bundle plan -o json > plan_delete.json 2>LOG.plan_delete.err cat LOG.plan_delete.err | contains.py '!panic:' '!internal error' > /dev/null fi @@ -53,7 +53,7 @@ MSYS_NO_PATHCONV=1 $CLI workspace delete --recursive "$STATE_PATH" &> LOG.wipe_r cat LOG.wipe_remote_state | contains.py '!panic:' '!internal error' > /dev/null if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json | nostamp > plan_delete2.json 2>LOG.plan_delete2.err + $CLI bundle plan -o json > plan_delete2.json 2>LOG.plan_delete2.err cat LOG.plan_delete2.err | contains.py '!panic:' '!internal error' > /dev/null fi diff --git a/acceptance/bundle/invariant/destroy_idempotent/script b/acceptance/bundle/invariant/destroy_idempotent/script index a57b06d029d..0467b23330e 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/script +++ b/acceptance/bundle/invariant/destroy_idempotent/script @@ -12,7 +12,7 @@ ROOT_PATH=/Workspace/Users/$CURRENT_USER_NAME/.bundle/$bundle_name/default # Initial deploy of the resources. Only pre-plan when READPLAN=1 exercises the # saved-plan deploy path; otherwise `bundle deploy` plans on its own. if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json | nostamp > plan_initial.json 2>LOG.plan_initial.err + $CLI bundle plan -o json > plan_initial.json 2>LOG.plan_initial.err cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null fi diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/script b/acceptance/bundle/resources/apps/readplan-lifecycle/script index 5f2e714223d..f3e11eced2d 100644 --- a/acceptance/bundle/resources/apps/readplan-lifecycle/script +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/script @@ -10,6 +10,6 @@ trace print_requests.py //apps/myapp/start title "Change description and deploy from plan: no extra start" trace update_file.py databricks.yml "description: initial" "description: updated" -$CLI bundle plan -o json | nostamp > tmp.plan.json +$CLI bundle plan -o json > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) trace print_requests.py //apps/myapp/start diff --git a/acceptance/bundle/templates/default-python/combinations/classic/script b/acceptance/bundle/templates/default-python/combinations/classic/script index c8d9bdc45a4..589290b77c2 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/script +++ b/acceptance/bundle/templates/default-python/combinations/classic/script @@ -6,13 +6,13 @@ trace yamlcheck.py trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod -trace $CLI bundle plan -t dev -o json | nostamp > tmp.plan.json 2> LOG.plan1.error +trace $CLI bundle plan -t dev -o json > tmp.plan.json 2> LOG.plan1.error $TESTDIR/../check_output.py $CLI bundle deploy -t dev $(readplanarg tmp.plan.json) # check no unexpected drift after deploy; due to dynamic_version whl is different so job is updated $CLI bundle plan -t dev 2> LOG.plan2.error | contains.py '0 to add' '0 to delete' '!0 unchanged' > /dev/null trace $CLI bundle destroy -t dev --auto-approve 2>&1 | tail -n 2 -trace $CLI bundle plan -t prod -o json | nostamp > tmp.plan.json 2> LOG.plan3.error +trace $CLI bundle plan -t prod -o json > tmp.plan.json 2> LOG.plan3.error $TESTDIR/../check_output.py $CLI bundle deploy -t prod $(readplanarg tmp.plan.json) # check no drift after deploy $CLI bundle plan -t prod 2> LOG.plan4.error | contains.py '0 to add, 0 to change, 0 to delete' '!0 unchanged' > /dev/null diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/script b/acceptance/bundle/templates/default-python/combinations/serverless/script index b5374563a57..389bdf3bbc7 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/script +++ b/acceptance/bundle/templates/default-python/combinations/serverless/script @@ -6,13 +6,13 @@ trace yamlcheck.py trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod -trace $CLI bundle plan -t dev -o json | nostamp > tmp.plan.json 2> LOG.plan1.error +trace $CLI bundle plan -t dev -o json > tmp.plan.json 2> LOG.plan1.error $TESTDIR/../check_output.py $CLI bundle deploy -t dev $(readplanarg tmp.plan.json) # check no drift after deploy $CLI bundle plan -t dev 2> LOG.plan2.error | contains.py '0 to add, 0 to change, 0 to delete' '!0 unchanged' > /dev/null trace $CLI bundle destroy -t dev --auto-approve 2>&1 | tail -n 2 -trace $CLI bundle plan -t prod -o json | nostamp > tmp.plan.json 2> LOG.plan3.error +trace $CLI bundle plan -t prod -o json > tmp.plan.json 2> LOG.plan3.error $TESTDIR/../check_output.py $CLI bundle deploy -t prod $(readplanarg tmp.plan.json) # check no drift after deploy $CLI bundle plan -t prod 2> LOG.plan4.error | contains.py '0 to add, 0 to change, 0 to delete' '!0 unchanged' > /dev/null From 63b2c7ca0732e82ebdf95706713fc8eec557c508 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 18:24:29 +0200 Subject: [PATCH 090/164] Use state features as the DMS signal in the plan, drop storage_backend The plan's stamps exist because a state feature is set, so the plan now carries the same "features" field as the state file rather than inventing a parallel storage_backend field. deploy --plan rejects a plan whose feature set differs from the target's, which subsumes the old backend comparison: a plan built without recording is still refused against a recording target. Exports dstate.FeatureRecordDeploymentHistory and adds StateFeatures() so the plan and the validation can read the state's feature set. nostamp strips "features" instead of "storage_backend" so non-DMS goldens stay shared. Co-authored-by: Isaac --- acceptance/bin/nostamp.py | 4 ++-- .../bundle/dms/declined-deploy/output.txt | 4 +++- acceptance/bundle/dms/depends-on/output.txt | 8 ++++++-- .../dms/deployment-metadata-change/output.txt | 8 ++++++-- acceptance/bundle/dms/empty-grants/output.txt | 4 +++- .../bundle/dms/failed-delete/output.txt | 4 +++- .../bundle/dms/failed-recreate/output.txt | 4 +++- .../dms/failed-update-permissions/output.txt | 4 +++- .../bundle/dms/multiple-resources/output.txt | 8 ++++++-- acceptance/bundle/dms/no-drift/output.txt | 8 ++++++-- acceptance/bundle/dms/no-resources/output.txt | 8 ++++++-- .../operation-upload-fails-once/output.txt | 4 +++- .../bundle/dms/plan-without-dms/output.txt | 6 +++--- acceptance/bundle/dms/plan-without-dms/script | 2 +- acceptance/bundle/dms/provenance/output.txt | 4 +++- .../bundle/dms/record-failure/output.txt | 4 +++- acceptance/bundle/dms/script.prepare | 2 +- acceptance/bundle/dms/stale-plan/output.txt | 4 +++- .../bundle/dms/successful-recreate/output.txt | 8 ++++++-- .../dms/version-never-created/output.txt | 4 +++- bundle/deployplan/plan.go | 8 ++++---- bundle/direct/bundle_plan.go | 2 +- bundle/direct/dstate/state.go | 20 +++++++++++++------ cmd/bundle/utils/process.go | 17 ++++++++-------- 24 files changed, 100 insertions(+), 49 deletions(-) diff --git a/acceptance/bin/nostamp.py b/acceptance/bin/nostamp.py index d6b3f9524c7..0ff0c6bab0d 100755 --- a/acceptance/bin/nostamp.py +++ b/acceptance/bin/nostamp.py @@ -13,7 +13,7 @@ == "" is kept (a Terraform state dump carries that for an unstamped job). 2. changes entries keyed "deployment.deployment_id" / "deployment.version_id". 3. a changes object emptied by (2), or already empty. - 4. the recorded plan header fields deployment_id, next_version_id, last_version_id. + 4. the recorded plan header fields deployment_id, next_version_id, last_version_id, features. """ import argparse @@ -22,7 +22,7 @@ _STAMP_KEYS = ("deployment_id", "version_id") _CHANGE_KEYS = ("deployment.deployment_id", "deployment.version_id") -_PLAN_HEADER_KEYS = ("deployment_id", "next_version_id", "last_version_id", "storage_backend") +_PLAN_HEADER_KEYS = ("deployment_id", "next_version_id", "last_version_id", "features") def scrub(node): diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index abe3950b66c..dc9cc93c069 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -69,7 +69,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index fe37a153f94..c5f60147be1 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -4,7 +4,9 @@ { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan @@ -61,7 +63,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 556c9a1f980..bae04c4ac57 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -69,7 +69,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan @@ -150,7 +152,9 @@ A common practice is to use a username or principal name in this path, i.e. use "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan -t prod diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index 12f5c5a56aa..d92a009367b 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -140,7 +140,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index cef0978f56e..c06f8264a5c 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -69,7 +69,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 99b280227ab..2d49fa2ae12 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -69,7 +69,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index e947a7aeb1f..1fa6048dbbf 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -17,7 +17,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 607d3663a43..6452170caf1 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -27,7 +27,9 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan @@ -96,7 +98,9 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index ca653c21fe5..508bc6e9942 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -37,7 +37,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan @@ -172,7 +174,9 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index cb774043dd6..9b79a57a05f 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -52,7 +52,9 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan @@ -128,7 +130,9 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index c658c1b380c..6cd02954448 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -16,7 +16,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/plan-without-dms/output.txt b/acceptance/bundle/dms/plan-without-dms/output.txt index 84718b40da8..6d19f34d85a 100644 --- a/acceptance/bundle/dms/plan-without-dms/output.txt +++ b/acceptance/bundle/dms/plan-without-dms/output.txt @@ -1,15 +1,15 @@ === Generate and save plan WITHOUT DMS recording ->>> jq {storage_backend, deployment_id, next_version_id} +>>> jq {features, deployment_id, next_version_id} { - "storage_backend": null, + "features": null, "deployment_id": null, "next_version_id": null } === Enable DMS recording and try to deploy the non-DMS plan >>> musterr [CLI] bundle deploy --plan tmp.plan.json -Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again +Error: this plan was created for a different set of state features than the target now has; run 'bundle plan' again >>> print_requests.py //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/plan-without-dms/script b/acceptance/bundle/dms/plan-without-dms/script index fb1648d57ab..9f3cb824ab3 100644 --- a/acceptance/bundle/dms/plan-without-dms/script +++ b/acceptance/bundle/dms/plan-without-dms/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Generate and save plan WITHOUT DMS recording" $CLI bundle plan -o json > tmp.plan.json -trace jq '{storage_backend, deployment_id, next_version_id}' < tmp.plan.json +trace jq '{features, deployment_id, next_version_id}' < tmp.plan.json title "Enable DMS recording and try to deploy the non-DMS plan" update_file.py databricks.yml 'bundle:' 'experimental: diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 5cde7587967..3683349ac0f 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -61,7 +61,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 0dc1e64891c..d6ecde2a879 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -18,7 +18,9 @@ Files: 6 uploaded, 0 deleted "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/script.prepare b/acceptance/bundle/dms/script.prepare index f170821b4e8..fee2c01fd1b 100644 --- a/acceptance/bundle/dms/script.prepare +++ b/acceptance/bundle/dms/script.prepare @@ -1,7 +1,7 @@ # dms_plan prints the plan's header only (via jq del(.plan)), then the readable plan. # The per-resource plan body carries remote_state that diverges between the fake and a # real workspace, so the recording tests assert the header fields (cli_version, lineage, -# serial, deployment_id, next/last_version_id, storage_backend) and the human-readable +# serial, deployment_id, next/last_version_id, features) and the human-readable # summary instead of the full JSON. Pass through any flags, e.g. `dms_plan -t prod`. dms_plan() { trace $CLI bundle plan "$@" -o json | jq 'del(.plan)' diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index ce52df879cc..4b4cfcd551c 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -4,7 +4,9 @@ { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index e8f16c3d5b6..a525d256f2e 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -69,7 +69,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan @@ -173,7 +175,9 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index f917f962000..fe66fa406d8 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -17,7 +17,9 @@ Files: 5 uploaded, 0 deleted "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "next_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" + "features": { + "record_deployment_history": {} + } } >>> [CLI] bundle plan diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 726e0de9a92..f96414a81da 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -29,10 +29,10 @@ type Plan struct { NextVersionId string `json:"next_version_id,omitempty"` LastVersionId string `json:"last_version_id,omitempty"` - // StorageBackend is where this deployment's state lives, the string form of a - // dstate.StorageBackend. deploy --plan checks it against the target's config so a recorded plan - // is never applied to a non-recording target or the reverse. Set only for a recorded plan. - StorageBackend string `json:"storage_backend,omitempty"` + // Features are the state feature flags this plan was built against, mirroring the state + // file's own "features" field. The stamps above exist because of a feature being set, so + // deploy --plan rejects a plan whose features differ from the target's (see process.go). + Features map[string]struct{} `json:"features,omitempty"` Plan map[string]*PlanEntry `json:"plan,omitzero"` diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index c11152cab26..0f09cb83fbb 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -170,7 +170,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // The plan records where its state lives so deploy --plan can reject a plan whose target has // since switched backends or has an outdated version or a wrong deployment_id. if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { - plan.StorageBackend = string(dstate.StorageBackendDeploymentMetadataService) + plan.Features = b.StateDB.StateFeatures() if b.StateDB.DeploymentID != "" { // Subsequent deployment: compute next version from current state. next, err := dms.NextVersion(b.StateDB.LatestVersionID) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 6a41661c516..a8f0cee82c0 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -57,7 +57,7 @@ const ( supportedStateVersion = featureStateVersion ) -// featureRecordDeploymentHistory marks a state whose resources are also recorded with the +// FeatureRecordDeploymentHistory marks a state whose resources are also recorded with the // deployment metadata service. Both stores are kept in step, so the marker is what tells a // reader the two already agree. A CLI that does not know the name refuses the state rather // than deploying over a deployment it would leave the service out of step with. @@ -65,7 +65,7 @@ const ( // The marker is sticky: once a deployment is recorded, the service holds resources that a // CLI which is not recording must not touch. So turning recording off does not clear it, and // deploying such a state without recording is refused. -const featureRecordDeploymentHistory = "record_deployment_history" +const FeatureRecordDeploymentHistory = "record_deployment_history" // featuresDocURL explains the state-features mechanism in the error shown when this CLI refuses a // state that depends on a feature it does not recognize. @@ -85,7 +85,7 @@ const ( // any feature not listed here is refused (see checkStateFeatures), so a newer CLI's feature is not // silently clobbered by this one. var recognizedFeatures = map[string]struct{}{ - featureRecordDeploymentHistory: {}, + FeatureRecordDeploymentHistory: {}, } // assertNoUnsupportedFeatures refuses a state that depends on a feature this CLI does not recognize. @@ -402,6 +402,14 @@ func (db *DeploymentState) StateCLIVersion() string { return db.Data.CLIVersion } +// StateFeatures returns the feature flags this state depends on, the signal a plan +// carries so deploy --plan can reject a plan built for a different feature set. +func (db *DeploymentState) StateFeatures() map[string]struct{} { + db.mu.Lock() + defer db.mu.Unlock() + return db.Data.Features +} + // StorageBackend reports where this deployment's state lives - the source of truth for whether the // bundle records deployment history. Set by Open; valid only after the state is opened. func (db *DeploymentState) StorageBackend() StorageBackend { @@ -548,13 +556,13 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe // the config bootstraps the marker there. On an existing deployment the state is authoritative: // the config can neither start recording one that was not (its resources would be created a // second time) nor stop recording one that is (the service still holds it). - _, recorded := db.Data.Features[featureRecordDeploymentHistory] + _, recorded := db.Data.Features[FeatureRecordDeploymentHistory] switch { case recording && !recorded && len(db.Data.State) == 0: if db.Data.Features == nil { db.Data.Features = make(map[string]struct{}, 1) } - db.Data.Features[featureRecordDeploymentHistory] = struct{}{} + db.Data.Features[FeatureRecordDeploymentHistory] = struct{}{} recorded = true case recording && !recorded: return errors.New(`this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources @@ -943,7 +951,7 @@ func (db *DeploymentState) dataForFile() Database { if header.Features == nil { header.Features = make(map[string]struct{}) } - header.Features[featureRecordDeploymentHistory] = struct{}{} + header.Features[FeatureRecordDeploymentHistory] = struct{}{} return Database{ Header: header, diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index f223abc82d4..fbe1bf968d6 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "maps" "path" "path/filepath" "strconv" @@ -468,16 +469,14 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // match the target's config, and a plan that targets an existing recorded deployment must not // predate the deployment the service now holds. It ends with the local lineage/serial guard. func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, dmsDeployment *bundledeployments.Deployment, dmsDeploymentID string) error { - // A plan records its storage backend, which must match the target's current one, or it would - // deploy against the wrong backend. StateDB.StorageBackend is the target's source of truth, and - // it is set even for a first recorded deploy (unlike the version ids, which are empty then). - isDMSPlan := plan.StorageBackend == string(dstate.StorageBackendDeploymentMetadataService) - switch records := b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService; { - case isDMSPlan && !records: - return errors.New("this plan records deployment history, but the target no longer does; run 'bundle plan' again") - case !isDMSPlan && records: - return errors.New("this plan does not record deployment history, but the target now does; run 'bundle plan' again") + // A plan is built against a set of state features, and the stamps it carries follow from + // them, so applying it to a target with a different set would deploy the wrong shape. + // The state is the target's source of truth and carries its features even on a first + // recorded deploy (unlike the version ids, which are empty then). + if !maps.Equal(plan.Features, b.DeploymentBundle.StateDB.StateFeatures()) { + return errors.New("this plan was created for a different set of state features than the target now has; run 'bundle plan' again") } + _, isDMSPlan := plan.Features[dstate.FeatureRecordDeploymentHistory] // The plan records the DMS deployment and version it targeted. Reject it if the live deployment // moved on - a newer version (someone deployed since, possibly from another machine) or a From 0a2d4bf640b8c283b1d3b33e3513d7d41ec6175b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 18:26:59 +0200 Subject: [PATCH 091/164] Say "bundle target" in the bind/unbind recording errors The bundle itself does not record anything - the target it deploys to does. Addresses review feedback on wording. Co-authored-by: Isaac --- acceptance/bundle/dms/bind-unbind-unsupported/output.txt | 4 ++-- bundle/direct/bind.go | 2 +- bundle/phases/bind.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt index 891466bd2e5..14686850b03 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt +++ b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt @@ -1,7 +1,7 @@ === bind is refused while recording deployment history >>> musterr [CLI] bundle deployment bind foo [JOB_ID] -Error: bind is not supported for a bundle that records deployment history +Error: bind is not supported for a bundle target that records deployment history === Deploy so the bundle records state, then unbind is refused too @@ -12,5 +12,5 @@ Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> musterr [CLI] bundle deployment unbind foo -Error: unbind is not supported for a bundle that records deployment history +Error: unbind is not supported for a bundle target that records deployment history diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 4605f34be66..ae5c85e9a9d 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -221,7 +221,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") if err != nil { if errors.Is(err, dstate.ErrUnsettingRecording) { - return errors.New("unbind is not supported for a bundle that records deployment history") + return errors.New("unbind is not supported for a bundle target that records deployment history") } return err } diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 86c9cc80da3..8fb5967196c 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -34,7 +34,7 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, en if engine.IsDirect() { if b.ConfiguresDeploymentHistory(ctx) { - logdiag.LogError(ctx, errors.New("bind is not supported for a bundle that records deployment history")) + logdiag.LogError(ctx, errors.New("bind is not supported for a bundle target that records deployment history")) return } From 2a9e8792402c95e6a4a3bdac57311da1748b9a35 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 18:34:15 +0200 Subject: [PATCH 092/164] acceptance/immutable: drop no-op nostamp jq extracts new_state.value.tasks, which carries neither the plan header nor a deployment block, so nostamp did nothing here. Co-authored-by: Isaac --- acceptance/bundle/deploy/immutable/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/script index 86757855d55..85e9a908d1d 100644 --- a/acceptance/bundle/deploy/immutable/script +++ b/acceptance/bundle/deploy/immutable/script @@ -5,7 +5,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle validate -trace $CLI bundle plan -o json | nostamp | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' +trace $CLI bundle plan -o json | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' trace $CLI bundle deploy From a7d1301a360eccd0c46ba77862ce18d68b602ab3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 18:51:15 +0200 Subject: [PATCH 093/164] acceptance/wal: trim the DMS skip comment Co-authored-by: Isaac --- acceptance/bundle/deploy/wal/test.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/acceptance/bundle/deploy/wal/test.toml b/acceptance/bundle/deploy/wal/test.toml index e4be659d2e1..512d9195cca 100644 --- a/acceptance/bundle/deploy/wal/test.toml +++ b/acceptance/bundle/deploy/wal/test.toml @@ -2,10 +2,7 @@ # These tests simulate process crashes using KillCaller and verify state recovery. # Only runs with direct engine since WAL is a direct-engine feature. # -# Not run under deployment history recording, and not planned to be: WAL recovery is a -# non-recording concept - under recording the service is the source of truth and a leftover WAL is -# discarded, not replayed (see dstate.Open; the DMS side is covered by bundle/dms) - so there is -# nothing here a DMS run would exercise. +# There is no WAL to recover from when DMS is enabled, so these do not run under it. Env.DATABRICKS_CLI_TEST_PID = "1" From b3eebb98520464e5336ac9a63128eacb17be28c1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 20:42:35 +0200 Subject: [PATCH 094/164] acceptance/select: add missing trailing newline (whitespace lint) Co-authored-by: Isaac --- acceptance/bundle/select/basic/test.toml | 2 +- acceptance/bundle/select/grants_permissions/test.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/select/basic/test.toml b/acceptance/bundle/select/basic/test.toml index 74bb671878d..fe41d5f871a 100644 --- a/acceptance/bundle/select/basic/test.toml +++ b/acceptance/bundle/select/basic/test.toml @@ -7,4 +7,4 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] # databricks.yml and the serialized plans are generated at runtime; the plan JSON # is kept out of the recorded output because it embeds local/cloud-specific state. -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json"] \ No newline at end of file +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json"] diff --git a/acceptance/bundle/select/grants_permissions/test.toml b/acceptance/bundle/select/grants_permissions/test.toml index f6571fe0223..4b4735b927d 100644 --- a/acceptance/bundle/select/grants_permissions/test.toml +++ b/acceptance/bundle/select/grants_permissions/test.toml @@ -10,4 +10,4 @@ Cloud = false RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] -Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json"] \ No newline at end of file +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json"] From 2bc157ea677f0bfc1384be64f0adae946e80390d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 21:41:50 +0200 Subject: [PATCH 095/164] acceptance: regenerate goldens stale after the nostamp/plan-file changes Five tests still carried goldens from before the plan files stopped being nostamped. escaped_ref and clusters/readplan-lifecycle had recorded the rejection ("this plan does not record deployment history") that only happened because the saved plan was stripped - they now deploy successfully. detect-change picked up an extra synced file, with_endpoint differed only in JSON escaping of "<" via nostamp, and secrets/update-value now projects .state with jq. All five pass under both DMS variants. Co-authored-by: Isaac --- .../resource_deps/escaped_ref/output.txt | 59 +++++++++++++++++-- .../clusters/readplan-lifecycle/output.txt | 8 ++- .../dashboards/detect-change/output.txt | 2 +- .../resources/secrets/update-value/output.txt | 22 +++---- .../recreate/with_endpoint/output.txt | 8 +-- 5 files changed, 73 insertions(+), 26 deletions(-) diff --git a/acceptance/bundle/resource_deps/escaped_ref/output.txt b/acceptance/bundle/resource_deps/escaped_ref/output.txt index 27a6aee8679..5370251a14d 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/output.txt +++ b/acceptance/bundle/resource_deps/escaped_ref/output.txt @@ -12,8 +12,59 @@ } === Payload: escaped stays literal, real is resolved -Warn: Plan was created with CLI version [CLI_VERSION] but current version is [CLI_VERSION] -Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.bar +Created jobs.foo +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged - -Exit code: 1 +>>> print_requests.py //jobs --sort --nostamp +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job bar", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + }, + "tasks": [ + { + "notebook_task": { + "base_parameters": { + "escaped": "${resources.jobs.bar.id}", + "mixed": "${runtime} [NUMID]", + "real": "[NUMID]", + "same_name": "${resources.jobs.bar.id} [NUMID]" + }, + "notebook_path": "/notebook", + "source": "WORKSPACE" + }, + "task_key": "job_task" + } + ] + } +} diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt b/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt index 689976f6e27..cee977be92d 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt @@ -10,13 +10,15 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Change tags and deploy from plan: no extra start >>> update_file.py databricks.yml env: original env: updated -Error: this plan does not record deployment history, but the target now does; run 'bundle plan' again +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated clusters.mycluster +Files: 4 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +>>> print_requests.py //clusters/start The following resources will be deleted: delete resources.clusters.mycluster All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default Destroy: 1 deleted - -Exit code: 1 diff --git a/acceptance/bundle/resources/dashboards/detect-change/output.txt b/acceptance/bundle/resources/dashboards/detect-change/output.txt index 1641d352758..bf83fc15de2 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/output.txt +++ b/acceptance/bundle/resources/dashboards/detect-change/output.txt @@ -103,7 +103,7 @@ Exit code: 1 >>> errcode [CLI] bundle deploy --force Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Updated dashboards.file_reference -Files: 3 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/secrets/update-value/output.txt b/acceptance/bundle/resources/secrets/update-value/output.txt index 10be38a03c2..bb8db84a11e 100644 --- a/acceptance/bundle/resources/secrets/update-value/output.txt +++ b/acceptance/bundle/resources/secrets/update-value/output.txt @@ -117,20 +117,14 @@ secrets secret1 catalog_name='main' schema_name='default' name='test_secret' com === Verify state does not contain actual secret value >>> print_state.py { - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 2, - "state": { "resources.secrets.secret1": { - "__id__": "main.default.test_secret", - "state": { - "catalog_name": "main", - "comment": "Test secret", - "name": "test_secret", - "schema_name": "default", - "value": "" - } + "__id__": "main.default.test_secret", + "state": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "" + } } - } } diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt index 15182d3ff4f..d14f3f90b0d 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -71,7 +71,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "name": "vector" } ], - "schema_json": "{\"id\":\"int\",\"vector\":\"array\u003cfloat\u003e\"}" + "schema_json": "{\"id\":\"int\",\"vector\":\"array\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "endpoint_uuid": "[UUID]", @@ -87,9 +87,9 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "direct_access_index_spec.schema_json": { "action": "skip", "reason": "normalized_by_backend", - "old": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}", - "new": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}", - "remote": "{\"id\":\"int\",\"vector\":\"array\u003cfloat\u003e\"}" + "old": "{\"id\":\"integer\",\"vector\":\"array\"}", + "new": "{\"id\":\"integer\",\"vector\":\"array\"}", + "remote": "{\"id\":\"int\",\"vector\":\"array\"}" }, "endpoint_uuid": { "action": "skip", From ed209b324d549e9ad30ff858a2b339328ceb850e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 22:03:47 +0200 Subject: [PATCH 096/164] acceptance: stop nostamping plans captured with &> into a golden These three tests capture the plan with `&> out.plan.$ENGINE.json`, which deliberately folds trace's ">>> errcode ..." stderr header into the file. Inserting `| nostamp` rebound the redirect to nostamp, moving that header out of the plan golden and into output.txt. None of them run under DMS (EnvMatrix.DMS = [""]), so the nostamp bought nothing. Co-authored-by: Isaac --- acceptance/bundle/resource_deps/volume_path_job_ref/script | 2 +- .../bundle/resources/quality_monitors/change_assets_dir/script | 2 +- .../bundle/resources/quality_monitors/change_table_name/script | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script index 5b7a2e956b0..6c7250cc564 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/script +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -4,7 +4,7 @@ trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs, .resources.v # The job's data_path default references the volume's computed volume_path. # Record the JSON plan per engine to confirm the reference is resolved into the # planned new_state rather than left as ${resources...}. -trace errcode $CLI bundle plan -o json | nostamp &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Deploy and inspect the create-job request: the parameter default sent to # the Jobs API must be the interpolated volume_path. diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 15d0f4540d7..99e80f5f70a 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets_$UNIQUE_NAME" "assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets2_$UNIQUE_NAME" trace errcode $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -trace errcode $CLI bundle plan -o json | nostamp &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index 1334426760f..6cf47e95c59 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "table_name: $TABLE_NAME" "table_name: ${TABLE_NAME}_2" trace errcode $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -trace errcode $CLI bundle plan -o json | nostamp &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt From a76b434230040807efcb12929f1690632d0078eb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 22:21:04 +0200 Subject: [PATCH 097/164] acceptance/bundle/resources/postgres_projects: Add compute_last_active_time field to plan outputs and project filter The Postgres API now includes compute_last_active_time in project status responses. Update the project_fields filter to include this field and update the plan outputs to include the new field so cloud tests pass. Co-authored-by: Isaac --- acceptance/bundle/resources/postgres_projects/script.prepare | 2 +- .../update_display_name/out.plan.no_change.direct.json | 1 + .../update_display_name/out.plan.restore.direct.json | 1 + .../update_display_name/out.plan.update.direct.json | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/resources/postgres_projects/script.prepare b/acceptance/bundle/resources/postgres_projects/script.prepare index 1386ff1169c..7d5f8d0de86 100644 --- a/acceptance/bundle/resources/postgres_projects/script.prepare +++ b/acceptance/bundle/resources/postgres_projects/script.prepare @@ -2,5 +2,5 @@ # on. Anything the backend returns outside this list is dropped, so a new field # in the API response does not break the golden. project_fields() { - jq '{name, project_id, status: (.status | {branch_logical_size_limit_bytes, default_branch, default_endpoint_settings, display_name, enable_pg_native_login, history_retention_duration, owner, pg_version, project_id, synthetic_storage_size_bytes}), uid}' + jq '{name, project_id, status: (.status | {branch_logical_size_limit_bytes, compute_last_active_time, default_branch, default_endpoint_settings, display_name, enable_pg_native_login, history_retention_duration, owner, pg_version, project_id, synthetic_storage_size_bytes}), uid}' } diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json index 45989db3a77..6db1a714bdb 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json @@ -6,6 +6,7 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], + "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json index 39a17d8f975..9b6193869bc 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json @@ -19,6 +19,7 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], + "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json index 02da84cc6a9..0bfb77340e9 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json @@ -19,6 +19,7 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], + "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, From 26f4329bae1b07e250ff0c507d2f2036e594644d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 22:24:59 +0200 Subject: [PATCH 098/164] acceptance/bundle/resources/grants/schemas/remove_all: Skip DMS=true variant due to known bug Emptying a grants node is a known bug under deployment-history recording (DMS). It records a succeeded UPDATE with no state, which the service rejects. See: acceptance/bundle/dms/empty-grants EnvMatrixExclude will prevent the DMS=true variant from running while allowing the default (DMS empty) and terraform variants to proceed. Co-authored-by: Isaac --- .../bundle/resources/grants/schemas/remove_all/test.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 acceptance/bundle/resources/grants/schemas/remove_all/test.toml diff --git a/acceptance/bundle/resources/grants/schemas/remove_all/test.toml b/acceptance/bundle/resources/grants/schemas/remove_all/test.toml new file mode 100644 index 00000000000..df54156495f --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/remove_all/test.toml @@ -0,0 +1,4 @@ +# Emptying a grants node is a known bug under DMS (deployment-history recording). +# It records a succeeded UPDATE with no state, which the service rejects. +# See: acceptance/bundle/dms/empty-grants +EnvMatrixExclude.empty_grants_dms_bug = ["DMS=true"] From 62d461b17f5815959c0d969cd2c086239323c0d9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 22:28:05 +0200 Subject: [PATCH 099/164] acceptance/bundle/resources/postgres_*: Add out.requests.txt to Ignore list and fix compute_last_active_time handling - Add out.requests.txt to Ignore list in all postgres_* test.toml files since print_requests.py --keep retains the file and cleanup trap may race with test framework file checking - Update project_fields filter to include compute_last_active_time but drop null values to maintain consistency across local and cloud test runs - Revert compute_last_active_time from plan output files since this is a volatile field that varies between cloud and local Co-authored-by: Isaac --- acceptance/bundle/resources/postgres_branches/test.toml | 1 + acceptance/bundle/resources/postgres_catalogs/test.toml | 1 + acceptance/bundle/resources/postgres_databases/test.toml | 1 + acceptance/bundle/resources/postgres_endpoints/test.toml | 1 + acceptance/bundle/resources/postgres_projects/script.prepare | 5 +++-- acceptance/bundle/resources/postgres_projects/test.toml | 1 + .../update_display_name/out.plan.no_change.direct.json | 1 - .../update_display_name/out.plan.restore.direct.json | 1 - .../update_display_name/out.plan.update.direct.json | 1 - acceptance/bundle/resources/postgres_roles/test.toml | 1 + acceptance/bundle/resources/postgres_synced_tables/test.toml | 1 + 11 files changed, 10 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/postgres_branches/test.toml b/acceptance/bundle/resources/postgres_branches/test.toml index 6a939ff9d33..5e0c1810952 100644 --- a/acceptance/bundle/resources/postgres_branches/test.toml +++ b/acceptance/bundle/resources/postgres_branches/test.toml @@ -10,6 +10,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ "databricks.yml", ".databricks", + "out.requests.txt", ] [[Repls]] diff --git a/acceptance/bundle/resources/postgres_catalogs/test.toml b/acceptance/bundle/resources/postgres_catalogs/test.toml index d890a3a23db..cfafc1c0466 100644 --- a/acceptance/bundle/resources/postgres_catalogs/test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/test.toml @@ -9,6 +9,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ "databricks.yml", ".databricks", + "out.requests.txt", ] [[Repls]] diff --git a/acceptance/bundle/resources/postgres_databases/test.toml b/acceptance/bundle/resources/postgres_databases/test.toml index 63d2317caa8..d79f1727a4c 100644 --- a/acceptance/bundle/resources/postgres_databases/test.toml +++ b/acceptance/bundle/resources/postgres_databases/test.toml @@ -10,6 +10,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ "databricks.yml", ".databricks", + "out.requests.txt", ] [[Repls]] diff --git a/acceptance/bundle/resources/postgres_endpoints/test.toml b/acceptance/bundle/resources/postgres_endpoints/test.toml index cb416de94d6..7c110fc3082 100644 --- a/acceptance/bundle/resources/postgres_endpoints/test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/test.toml @@ -8,6 +8,7 @@ CloudEnvs.azure = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ + "out.requests.txt", "databricks.yml", ".databricks", ] diff --git a/acceptance/bundle/resources/postgres_projects/script.prepare b/acceptance/bundle/resources/postgres_projects/script.prepare index 7d5f8d0de86..9f24fb016f7 100644 --- a/acceptance/bundle/resources/postgres_projects/script.prepare +++ b/acceptance/bundle/resources/postgres_projects/script.prepare @@ -1,6 +1,7 @@ # project_fields is the allow-list of project fields the acceptance tests assert # on. Anything the backend returns outside this list is dropped, so a new field -# in the API response does not break the golden. +# in the API response does not break the golden. Null values are dropped to keep +# output stable across environments. project_fields() { - jq '{name, project_id, status: (.status | {branch_logical_size_limit_bytes, compute_last_active_time, default_branch, default_endpoint_settings, display_name, enable_pg_native_login, history_retention_duration, owner, pg_version, project_id, synthetic_storage_size_bytes}), uid}' + jq '{name, project_id, status: (.status | {branch_logical_size_limit_bytes, compute_last_active_time, default_branch, default_endpoint_settings, display_name, enable_pg_native_login, history_retention_duration, owner, pg_version, project_id, synthetic_storage_size_bytes} | with_entries(select(.value != null))), uid}' } diff --git a/acceptance/bundle/resources/postgres_projects/test.toml b/acceptance/bundle/resources/postgres_projects/test.toml index 7c0df797b93..97c3bef7e0f 100644 --- a/acceptance/bundle/resources/postgres_projects/test.toml +++ b/acceptance/bundle/resources/postgres_projects/test.toml @@ -10,6 +10,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ "databricks.yml", ".databricks", + "out.requests.txt", ] [[Repls]] diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json index 6db1a714bdb..45989db3a77 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json @@ -6,7 +6,6 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], - "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json index 9b6193869bc..39a17d8f975 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json @@ -19,7 +19,6 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], - "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json index 0bfb77340e9..02da84cc6a9 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json @@ -19,7 +19,6 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], - "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_roles/test.toml b/acceptance/bundle/resources/postgres_roles/test.toml index 6b34a9705dd..fc9792a8328 100644 --- a/acceptance/bundle/resources/postgres_roles/test.toml +++ b/acceptance/bundle/resources/postgres_roles/test.toml @@ -8,6 +8,7 @@ CloudEnvs.azure = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ + "out.requests.txt", "databricks.yml", ".databricks", ] diff --git a/acceptance/bundle/resources/postgres_synced_tables/test.toml b/acceptance/bundle/resources/postgres_synced_tables/test.toml index bfaf7d18e70..605a4f085da 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/test.toml @@ -7,6 +7,7 @@ CloudEnvs.azure = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ + "out.requests.txt", "databricks.yml", ".databricks", ] From 9cbedde5105582c700c736872a8089c1806bb43a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 22:29:16 +0200 Subject: [PATCH 100/164] acceptance/bundle/resources/postgres_*: Remove --keep flag from print_requests.py calls The --keep flag causes print_requests.py to retain out.requests.txt, which then gets flagged as an unexpected output file by the test framework. Since these tests only call print_requests.py once per phase, we don't need --keep. Removing the flag ensures out.requests.txt is deleted after the command completes. Co-authored-by: Isaac --- acceptance/bundle/resources/postgres_branches/basic/script | 2 +- .../resources/postgres_branches/replace_existing/script | 4 ++-- .../resources/postgres_branches/update_protected/script | 2 +- acceptance/bundle/resources/postgres_catalogs/basic/script | 2 +- acceptance/bundle/resources/postgres_databases/basic/script | 2 +- .../resources/postgres_databases/replace_existing/script | 2 +- acceptance/bundle/resources/postgres_databases/update/script | 2 +- acceptance/bundle/resources/postgres_endpoints/basic/script | 2 +- .../resources/postgres_endpoints/replace_existing/script | 4 ++-- .../resources/postgres_endpoints/update_autoscaling/script | 2 +- acceptance/bundle/resources/postgres_projects/basic/script | 2 +- .../resources/postgres_projects/update_display_name/script | 2 +- acceptance/bundle/resources/postgres_roles/basic/script | 2 +- .../bundle/resources/postgres_roles/replace_existing/script | 2 +- acceptance/bundle/resources/postgres_roles/update/script | 2 +- .../bundle/resources/postgres_synced_tables/basic/script | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/acceptance/bundle/resources/postgres_branches/basic/script b/acceptance/bundle/resources/postgres_branches/basic/script index 56af9987f86..292107ba1fa 100644 --- a/acceptance/bundle/resources/postgres_branches/basic/script +++ b/acceptance/bundle/resources/postgres_branches/basic/script @@ -21,4 +21,4 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling) -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/script b/acceptance/bundle/resources/postgres_branches/replace_existing/script index d64adec24ab..1c7e9bce7b8 100644 --- a/acceptance/bundle/resources/postgres_branches/replace_existing/script +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/script @@ -15,7 +15,7 @@ trace $CLI bundle deploy # Confirm the implicit production branch is now under management. trace $CLI postgres get-branch "projects/test-pg-proj-${UNIQUE_NAME}/branches/production" | branch_fields -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json # bundle destroy: the backend rejects independent deletion of the root # branch with a MANAGED_BY_PARENT marker; both engines disregard that @@ -25,4 +25,4 @@ trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id, title "bundle destroy" $CLI bundle destroy --auto-approve > out.destroy.txt 2>&1 || true -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/script b/acceptance/bundle/resources/postgres_branches/update_protected/script index 1566c885831..398fb08b015 100755 --- a/acceptance/bundle/resources/postgres_branches/update_protected/script +++ b/acceptance/bundle/resources/postgres_branches/update_protected/script @@ -8,7 +8,7 @@ trap cleanup EXIT print_requests() { local name=$1 - trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt } diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/script b/acceptance/bundle/resources/postgres_catalogs/basic/script index 48f54855709..4f6d93ee013 100644 --- a/acceptance/bundle/resources/postgres_catalogs/basic/script +++ b/acceptance/bundle/resources/postgres_catalogs/basic/script @@ -20,4 +20,4 @@ trace $CLI postgres get-catalog "${catalog_name}" | catalog_fields trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling). -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json diff --git a/acceptance/bundle/resources/postgres_databases/basic/script b/acceptance/bundle/resources/postgres_databases/basic/script index 0fed53dfcf9..bc26ef3c3fb 100644 --- a/acceptance/bundle/resources/postgres_databases/basic/script +++ b/acceptance/bundle/resources/postgres_databases/basic/script @@ -22,4 +22,4 @@ trace $CLI postgres get-database "${database_name}" | database_fields trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling) -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/script b/acceptance/bundle/resources/postgres_databases/replace_existing/script index 65db8cd6ae7..9534339b97f 100644 --- a/acceptance/bundle/resources/postgres_databases/replace_existing/script +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/script @@ -24,4 +24,4 @@ trace $CLI bundle deploy title "Confirm the database is under management again" trace $CLI postgres get-database "projects/test-pg-proj-${UNIQUE_NAME}/branches/main/databases/my-database" | database_fields -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_databases/update/script b/acceptance/bundle/resources/postgres_databases/update/script index bdbad2b67c6..469ad992df0 100644 --- a/acceptance/bundle/resources/postgres_databases/update/script +++ b/acceptance/bundle/resources/postgres_databases/update/script @@ -8,7 +8,7 @@ trap cleanup EXIT print_requests() { local name=$1 - trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt } diff --git a/acceptance/bundle/resources/postgres_endpoints/basic/script b/acceptance/bundle/resources/postgres_endpoints/basic/script index 7a2be07d9f5..5e73cf79f6d 100755 --- a/acceptance/bundle/resources/postgres_endpoints/basic/script +++ b/acceptance/bundle/resources/postgres_endpoints/basic/script @@ -22,4 +22,4 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling) -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/script b/acceptance/bundle/resources/postgres_endpoints/replace_existing/script index 4c88b1f8f18..8e738d3c14e 100644 --- a/acceptance/bundle/resources/postgres_endpoints/replace_existing/script +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/script @@ -17,7 +17,7 @@ trace $CLI bundle deploy # bundle's spec must be applied: 600s. trace $CLI postgres get-endpoint "projects/test-pg-proj-${UNIQUE_NAME}/branches/develop/endpoints/primary" | endpoint_fields -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json # bundle destroy: the backend rejects independent deletion of the primary # read-write endpoint with a MANAGED_BY_PARENT marker; both engines @@ -27,4 +27,4 @@ trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id, title "bundle destroy" $CLI bundle destroy --auto-approve > out.destroy.txt 2>&1 || true -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script index f2506c21a01..f7d6b515f1e 100755 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script @@ -8,7 +8,7 @@ trap cleanup EXIT print_requests() { local name=$1 - trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt } diff --git a/acceptance/bundle/resources/postgres_projects/basic/script b/acceptance/bundle/resources/postgres_projects/basic/script index f2ac6abbcfe..359f2a0ac22 100755 --- a/acceptance/bundle/resources/postgres_projects/basic/script +++ b/acceptance/bundle/resources/postgres_projects/basic/script @@ -21,4 +21,4 @@ trace $CLI bundle summary # Filter requests to only show postgres API operations (exclude workspace, telemetry, and operation polling) # Keep the original out.requests.txt for debugging -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/script b/acceptance/bundle/resources/postgres_projects/update_display_name/script index 657eca21869..545bcec2e17 100755 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/script +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/script @@ -8,7 +8,7 @@ trap cleanup EXIT print_requests() { local name=$1 - trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt } diff --git a/acceptance/bundle/resources/postgres_roles/basic/script b/acceptance/bundle/resources/postgres_roles/basic/script index d5e3f421cd7..212979aeea0 100644 --- a/acceptance/bundle/resources/postgres_roles/basic/script +++ b/acceptance/bundle/resources/postgres_roles/basic/script @@ -22,4 +22,4 @@ trace $CLI postgres get-role "${role_name}" | role_fields trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling) -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/script b/acceptance/bundle/resources/postgres_roles/replace_existing/script index b7a64f0cc49..3d3ef2343b8 100644 --- a/acceptance/bundle/resources/postgres_roles/replace_existing/script +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/script @@ -24,4 +24,4 @@ trace $CLI bundle deploy title "Confirm the role is under management again" trace $CLI postgres get-role "projects/test-pg-proj-${UNIQUE_NAME}/branches/main/roles/test-role" | role_fields -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_roles/update/script b/acceptance/bundle/resources/postgres_roles/update/script index 5eea84136a6..23877b13614 100644 --- a/acceptance/bundle/resources/postgres_roles/update/script +++ b/acceptance/bundle/resources/postgres_roles/update/script @@ -8,7 +8,7 @@ trap cleanup EXIT print_requests() { local name=$1 - trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt } diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/script b/acceptance/bundle/resources/postgres_synced_tables/basic/script index 68b04d83301..b2a3ae78ab0 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/script +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/script @@ -36,4 +36,4 @@ trace $CLI postgres get-synced-table "synced_tables/lakebase_test_${UNIQUE_NAME} trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling). -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json From f6a0cfd98a2a92213f6d9568f4a8c60cb8fc5585 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 23:15:18 +0200 Subject: [PATCH 101/164] Revert "acceptance/bundle/resources/postgres_*: Remove --keep flag from print_requests.py calls" This reverts commit 9cbedde5105582c700c736872a8089c1806bb43a. --- acceptance/bundle/resources/postgres_branches/basic/script | 2 +- .../resources/postgres_branches/replace_existing/script | 4 ++-- .../resources/postgres_branches/update_protected/script | 2 +- acceptance/bundle/resources/postgres_catalogs/basic/script | 2 +- acceptance/bundle/resources/postgres_databases/basic/script | 2 +- .../resources/postgres_databases/replace_existing/script | 2 +- acceptance/bundle/resources/postgres_databases/update/script | 2 +- acceptance/bundle/resources/postgres_endpoints/basic/script | 2 +- .../resources/postgres_endpoints/replace_existing/script | 4 ++-- .../resources/postgres_endpoints/update_autoscaling/script | 2 +- acceptance/bundle/resources/postgres_projects/basic/script | 2 +- .../resources/postgres_projects/update_display_name/script | 2 +- acceptance/bundle/resources/postgres_roles/basic/script | 2 +- .../bundle/resources/postgres_roles/replace_existing/script | 2 +- acceptance/bundle/resources/postgres_roles/update/script | 2 +- .../bundle/resources/postgres_synced_tables/basic/script | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/acceptance/bundle/resources/postgres_branches/basic/script b/acceptance/bundle/resources/postgres_branches/basic/script index 292107ba1fa..56af9987f86 100644 --- a/acceptance/bundle/resources/postgres_branches/basic/script +++ b/acceptance/bundle/resources/postgres_branches/basic/script @@ -21,4 +21,4 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling) -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/script b/acceptance/bundle/resources/postgres_branches/replace_existing/script index 1c7e9bce7b8..d64adec24ab 100644 --- a/acceptance/bundle/resources/postgres_branches/replace_existing/script +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/script @@ -15,7 +15,7 @@ trace $CLI bundle deploy # Confirm the implicit production branch is now under management. trace $CLI postgres get-branch "projects/test-pg-proj-${UNIQUE_NAME}/branches/production" | branch_fields -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json # bundle destroy: the backend rejects independent deletion of the root # branch with a MANAGED_BY_PARENT marker; both engines disregard that @@ -25,4 +25,4 @@ trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id, title "bundle destroy" $CLI bundle destroy --auto-approve > out.destroy.txt 2>&1 || true -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/script b/acceptance/bundle/resources/postgres_branches/update_protected/script index 398fb08b015..1566c885831 100755 --- a/acceptance/bundle/resources/postgres_branches/update_protected/script +++ b/acceptance/bundle/resources/postgres_branches/update_protected/script @@ -8,7 +8,7 @@ trap cleanup EXIT print_requests() { local name=$1 - trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt } diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/script b/acceptance/bundle/resources/postgres_catalogs/basic/script index 4f6d93ee013..48f54855709 100644 --- a/acceptance/bundle/resources/postgres_catalogs/basic/script +++ b/acceptance/bundle/resources/postgres_catalogs/basic/script @@ -20,4 +20,4 @@ trace $CLI postgres get-catalog "${catalog_name}" | catalog_fields trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling). -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json diff --git a/acceptance/bundle/resources/postgres_databases/basic/script b/acceptance/bundle/resources/postgres_databases/basic/script index bc26ef3c3fb..0fed53dfcf9 100644 --- a/acceptance/bundle/resources/postgres_databases/basic/script +++ b/acceptance/bundle/resources/postgres_databases/basic/script @@ -22,4 +22,4 @@ trace $CLI postgres get-database "${database_name}" | database_fields trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling) -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/script b/acceptance/bundle/resources/postgres_databases/replace_existing/script index 9534339b97f..65db8cd6ae7 100644 --- a/acceptance/bundle/resources/postgres_databases/replace_existing/script +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/script @@ -24,4 +24,4 @@ trace $CLI bundle deploy title "Confirm the database is under management again" trace $CLI postgres get-database "projects/test-pg-proj-${UNIQUE_NAME}/branches/main/databases/my-database" | database_fields -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_databases/update/script b/acceptance/bundle/resources/postgres_databases/update/script index 469ad992df0..bdbad2b67c6 100644 --- a/acceptance/bundle/resources/postgres_databases/update/script +++ b/acceptance/bundle/resources/postgres_databases/update/script @@ -8,7 +8,7 @@ trap cleanup EXIT print_requests() { local name=$1 - trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt } diff --git a/acceptance/bundle/resources/postgres_endpoints/basic/script b/acceptance/bundle/resources/postgres_endpoints/basic/script index 5e73cf79f6d..7a2be07d9f5 100755 --- a/acceptance/bundle/resources/postgres_endpoints/basic/script +++ b/acceptance/bundle/resources/postgres_endpoints/basic/script @@ -22,4 +22,4 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling) -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/script b/acceptance/bundle/resources/postgres_endpoints/replace_existing/script index 8e738d3c14e..4c88b1f8f18 100644 --- a/acceptance/bundle/resources/postgres_endpoints/replace_existing/script +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/script @@ -17,7 +17,7 @@ trace $CLI bundle deploy # bundle's spec must be applied: 600s. trace $CLI postgres get-endpoint "projects/test-pg-proj-${UNIQUE_NAME}/branches/develop/endpoints/primary" | endpoint_fields -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json # bundle destroy: the backend rejects independent deletion of the primary # read-write endpoint with a MANAGED_BY_PARENT marker; both engines @@ -27,4 +27,4 @@ trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id, title "bundle destroy" $CLI bundle destroy --auto-approve > out.destroy.txt 2>&1 || true -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script index f7d6b515f1e..f2506c21a01 100755 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script @@ -8,7 +8,7 @@ trap cleanup EXIT print_requests() { local name=$1 - trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt } diff --git a/acceptance/bundle/resources/postgres_projects/basic/script b/acceptance/bundle/resources/postgres_projects/basic/script index 359f2a0ac22..f2ac6abbcfe 100755 --- a/acceptance/bundle/resources/postgres_projects/basic/script +++ b/acceptance/bundle/resources/postgres_projects/basic/script @@ -21,4 +21,4 @@ trace $CLI bundle summary # Filter requests to only show postgres API operations (exclude workspace, telemetry, and operation polling) # Keep the original out.requests.txt for debugging -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/script b/acceptance/bundle/resources/postgres_projects/update_display_name/script index 545bcec2e17..657eca21869 100755 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/script +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/script @@ -8,7 +8,7 @@ trap cleanup EXIT print_requests() { local name=$1 - trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt } diff --git a/acceptance/bundle/resources/postgres_roles/basic/script b/acceptance/bundle/resources/postgres_roles/basic/script index 212979aeea0..d5e3f421cd7 100644 --- a/acceptance/bundle/resources/postgres_roles/basic/script +++ b/acceptance/bundle/resources/postgres_roles/basic/script @@ -22,4 +22,4 @@ trace $CLI postgres get-role "${role_name}" | role_fields trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling) -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/script b/acceptance/bundle/resources/postgres_roles/replace_existing/script index 3d3ef2343b8..b7a64f0cc49 100644 --- a/acceptance/bundle/resources/postgres_roles/replace_existing/script +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/script @@ -24,4 +24,4 @@ trace $CLI bundle deploy title "Confirm the role is under management again" trace $CLI postgres get-role "projects/test-pg-proj-${UNIQUE_NAME}/branches/main/roles/test-role" | role_fields -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_roles/update/script b/acceptance/bundle/resources/postgres_roles/update/script index 23877b13614..5eea84136a6 100644 --- a/acceptance/bundle/resources/postgres_roles/update/script +++ b/acceptance/bundle/resources/postgres_roles/update/script @@ -8,7 +8,7 @@ trap cleanup EXIT print_requests() { local name=$1 - trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt } diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/script b/acceptance/bundle/resources/postgres_synced_tables/basic/script index b2a3ae78ab0..68b04d83301 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/script +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/script @@ -36,4 +36,4 @@ trace $CLI postgres get-synced-table "synced_tables/lakebase_test_${UNIQUE_NAME} trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling). -trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json From 2d1eaa0dc5588e3f952b79ba4bc53fc785322a11 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 23:15:19 +0200 Subject: [PATCH 102/164] Revert "acceptance/bundle/resources/postgres_*: Add out.requests.txt to Ignore list and fix compute_last_active_time handling" This reverts commit 62d461b17f5815959c0d969cd2c086239323c0d9. --- acceptance/bundle/resources/postgres_branches/test.toml | 1 - acceptance/bundle/resources/postgres_catalogs/test.toml | 1 - acceptance/bundle/resources/postgres_databases/test.toml | 1 - acceptance/bundle/resources/postgres_endpoints/test.toml | 1 - acceptance/bundle/resources/postgres_projects/script.prepare | 5 ++--- acceptance/bundle/resources/postgres_projects/test.toml | 1 - .../update_display_name/out.plan.no_change.direct.json | 1 + .../update_display_name/out.plan.restore.direct.json | 1 + .../update_display_name/out.plan.update.direct.json | 1 + acceptance/bundle/resources/postgres_roles/test.toml | 1 - acceptance/bundle/resources/postgres_synced_tables/test.toml | 1 - 11 files changed, 5 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/resources/postgres_branches/test.toml b/acceptance/bundle/resources/postgres_branches/test.toml index 5e0c1810952..6a939ff9d33 100644 --- a/acceptance/bundle/resources/postgres_branches/test.toml +++ b/acceptance/bundle/resources/postgres_branches/test.toml @@ -10,7 +10,6 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ "databricks.yml", ".databricks", - "out.requests.txt", ] [[Repls]] diff --git a/acceptance/bundle/resources/postgres_catalogs/test.toml b/acceptance/bundle/resources/postgres_catalogs/test.toml index cfafc1c0466..d890a3a23db 100644 --- a/acceptance/bundle/resources/postgres_catalogs/test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/test.toml @@ -9,7 +9,6 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ "databricks.yml", ".databricks", - "out.requests.txt", ] [[Repls]] diff --git a/acceptance/bundle/resources/postgres_databases/test.toml b/acceptance/bundle/resources/postgres_databases/test.toml index d79f1727a4c..63d2317caa8 100644 --- a/acceptance/bundle/resources/postgres_databases/test.toml +++ b/acceptance/bundle/resources/postgres_databases/test.toml @@ -10,7 +10,6 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ "databricks.yml", ".databricks", - "out.requests.txt", ] [[Repls]] diff --git a/acceptance/bundle/resources/postgres_endpoints/test.toml b/acceptance/bundle/resources/postgres_endpoints/test.toml index 7c110fc3082..cb416de94d6 100644 --- a/acceptance/bundle/resources/postgres_endpoints/test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/test.toml @@ -8,7 +8,6 @@ CloudEnvs.azure = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ - "out.requests.txt", "databricks.yml", ".databricks", ] diff --git a/acceptance/bundle/resources/postgres_projects/script.prepare b/acceptance/bundle/resources/postgres_projects/script.prepare index 9f24fb016f7..7d5f8d0de86 100644 --- a/acceptance/bundle/resources/postgres_projects/script.prepare +++ b/acceptance/bundle/resources/postgres_projects/script.prepare @@ -1,7 +1,6 @@ # project_fields is the allow-list of project fields the acceptance tests assert # on. Anything the backend returns outside this list is dropped, so a new field -# in the API response does not break the golden. Null values are dropped to keep -# output stable across environments. +# in the API response does not break the golden. project_fields() { - jq '{name, project_id, status: (.status | {branch_logical_size_limit_bytes, compute_last_active_time, default_branch, default_endpoint_settings, display_name, enable_pg_native_login, history_retention_duration, owner, pg_version, project_id, synthetic_storage_size_bytes} | with_entries(select(.value != null))), uid}' + jq '{name, project_id, status: (.status | {branch_logical_size_limit_bytes, compute_last_active_time, default_branch, default_endpoint_settings, display_name, enable_pg_native_login, history_retention_duration, owner, pg_version, project_id, synthetic_storage_size_bytes}), uid}' } diff --git a/acceptance/bundle/resources/postgres_projects/test.toml b/acceptance/bundle/resources/postgres_projects/test.toml index 97c3bef7e0f..7c0df797b93 100644 --- a/acceptance/bundle/resources/postgres_projects/test.toml +++ b/acceptance/bundle/resources/postgres_projects/test.toml @@ -10,7 +10,6 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ "databricks.yml", ".databricks", - "out.requests.txt", ] [[Repls]] diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json index 45989db3a77..6db1a714bdb 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json @@ -6,6 +6,7 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], + "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json index 39a17d8f975..9b6193869bc 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json @@ -19,6 +19,7 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], + "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json index 02da84cc6a9..0bfb77340e9 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json @@ -19,6 +19,7 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], + "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_roles/test.toml b/acceptance/bundle/resources/postgres_roles/test.toml index fc9792a8328..6b34a9705dd 100644 --- a/acceptance/bundle/resources/postgres_roles/test.toml +++ b/acceptance/bundle/resources/postgres_roles/test.toml @@ -8,7 +8,6 @@ CloudEnvs.azure = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ - "out.requests.txt", "databricks.yml", ".databricks", ] diff --git a/acceptance/bundle/resources/postgres_synced_tables/test.toml b/acceptance/bundle/resources/postgres_synced_tables/test.toml index 605a4f085da..bfaf7d18e70 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/test.toml @@ -7,7 +7,6 @@ CloudEnvs.azure = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ - "out.requests.txt", "databricks.yml", ".databricks", ] From 94e4135ffc71bb044425372d404abf6a4181abb6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 23:15:19 +0200 Subject: [PATCH 103/164] Revert "acceptance/bundle/resources/postgres_projects: Add compute_last_active_time field to plan outputs and project filter" This reverts commit a76b434230040807efcb12929f1690632d0078eb. --- acceptance/bundle/resources/postgres_projects/script.prepare | 2 +- .../update_display_name/out.plan.no_change.direct.json | 1 - .../update_display_name/out.plan.restore.direct.json | 1 - .../update_display_name/out.plan.update.direct.json | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/postgres_projects/script.prepare b/acceptance/bundle/resources/postgres_projects/script.prepare index 7d5f8d0de86..1386ff1169c 100644 --- a/acceptance/bundle/resources/postgres_projects/script.prepare +++ b/acceptance/bundle/resources/postgres_projects/script.prepare @@ -2,5 +2,5 @@ # on. Anything the backend returns outside this list is dropped, so a new field # in the API response does not break the golden. project_fields() { - jq '{name, project_id, status: (.status | {branch_logical_size_limit_bytes, compute_last_active_time, default_branch, default_endpoint_settings, display_name, enable_pg_native_login, history_retention_duration, owner, pg_version, project_id, synthetic_storage_size_bytes}), uid}' + jq '{name, project_id, status: (.status | {branch_logical_size_limit_bytes, default_branch, default_endpoint_settings, display_name, enable_pg_native_login, history_retention_duration, owner, pg_version, project_id, synthetic_storage_size_bytes}), uid}' } diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json index 6db1a714bdb..45989db3a77 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json @@ -6,7 +6,6 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], - "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json index 9b6193869bc..39a17d8f975 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json @@ -19,7 +19,6 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], - "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json index 0bfb77340e9..02da84cc6a9 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json @@ -19,7 +19,6 @@ "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], - "compute_last_active_time": "[TIMESTAMP]", "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, From d74c107da234c07100450299e3cb8f036553f053 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 23:20:30 +0200 Subject: [PATCH 104/164] acceptance/grants: regenerate remove_all golden after adding test.toml Adding a test.toml to a directory that had none puts another file in the bundle dir, so the synced file count went 5 -> 6. Co-authored-by: Isaac --- .../bundle/resources/grants/schemas/remove_all/out.test.toml | 2 +- .../bundle/resources/grants/schemas/remove_all/output.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml b/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml index cef45fe553a..9b8ddfcd2bd 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml @@ -1,3 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/grants/schemas/remove_all/output.txt b/acceptance/bundle/resources/grants/schemas/remove_all/output.txt index 2236e37e274..cf9b411eb62 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_all/output.txt +++ b/acceptance/bundle/resources/grants/schemas/remove_all/output.txt @@ -3,7 +3,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-remove-all-[UNIQUE_NAME]/default/files... Created schemas.grants_schema Created schemas.grants_schema.grants -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] grants get schema main.schema_remove_all_[UNIQUE_NAME] From 6138e6051e5edbeace0adf97bc469a67cda43fe9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 11:19:07 +0200 Subject: [PATCH 105/164] Derive the deployment version from serial; no version for a no-op plan A no-op deploy no longer creates a version. The deployment's version now tracks state changes exactly as serial does, so the two advance together and serial is a valid source for the version - which is what review asked for. That removes the need for next_version_id/last_version_id in the plan, so both fields are gone; deployment_id stays, and plan staleness is already guarded by the existing serial/lineage check. nostamp strips only deployment_id and features now. Co-authored-by: Isaac --- acceptance/bin/nostamp.py | 2 +- .../bundle/dms/declined-deploy/output.txt | 4 +- acceptance/bundle/dms/declined-deploy/script | 2 +- acceptance/bundle/dms/depends-on/output.txt | 4 +- acceptance/bundle/dms/depends-on/script | 2 +- .../dms/deployment-metadata-change/output.txt | 44 +----------------- .../dms/deployment-metadata-change/script | 2 +- acceptance/bundle/dms/empty-grants/output.txt | 4 +- acceptance/bundle/dms/empty-grants/script | 2 +- .../bundle/dms/failed-delete/output.txt | 4 +- acceptance/bundle/dms/failed-delete/script | 2 +- .../bundle/dms/failed-recreate/output.txt | 4 +- acceptance/bundle/dms/failed-recreate/script | 2 +- .../dms/failed-update-permissions/output.txt | 4 +- .../dms/failed-update-permissions/script | 2 +- .../bundle/dms/multiple-resources/output.txt | 12 ++--- .../bundle/dms/multiple-resources/script | 6 +-- acceptance/bundle/dms/no-drift/output.txt | 8 +--- acceptance/bundle/dms/no-drift/script | 4 +- acceptance/bundle/dms/no-resources/output.txt | 45 +------------------ acceptance/bundle/dms/no-resources/script | 4 +- .../operation-upload-fails-once/output.txt | 4 +- .../dms/operation-upload-fails-once/script | 2 +- acceptance/bundle/dms/provenance/output.txt | 4 +- acceptance/bundle/dms/provenance/script | 2 +- .../bundle/dms/record-failure/output.txt | 2 - acceptance/bundle/dms/stale-plan/output.txt | 6 +-- .../bundle/dms/successful-recreate/output.txt | 6 +-- .../bundle/dms/successful-recreate/script | 2 +- .../dms/version-never-created/output.txt | 1 - bundle/deployplan/plan.go | 12 +++-- bundle/direct/bundle_plan.go | 9 ---- bundle/phases/deploy.go | 27 ++++++----- cmd/bundle/utils/process.go | 7 --- 34 files changed, 60 insertions(+), 187 deletions(-) diff --git a/acceptance/bin/nostamp.py b/acceptance/bin/nostamp.py index 0ff0c6bab0d..4a7d1c4fac8 100755 --- a/acceptance/bin/nostamp.py +++ b/acceptance/bin/nostamp.py @@ -22,7 +22,7 @@ _STAMP_KEYS = ("deployment_id", "version_id") _CHANGE_KEYS = ("deployment.deployment_id", "deployment.version_id") -_PLAN_HEADER_KEYS = ("deployment_id", "next_version_id", "last_version_id", "features") +_PLAN_HEADER_KEYS = ("deployment_id", "features") def scrub(node): diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index dc9cc93c069..b5d43f5e729 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1 +=== The plan is a no-op now and records the record_deployment_history feature; the service holds the schema at version 1 >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -67,8 +67,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index 56a33fea036..bf486fe98f5 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -4,7 +4,7 @@ title "Deploy a schema, so the deployment and its first version exist" trace $CLI bundle deploy trace print_requests.py --dms --get //api/2.0/bundle -title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" +title "The plan is a no-op now and records the record_deployment_history feature; the service holds the schema at version 1" dms_plan trace print_requests.py --dms --get //api/2.0/bundle trace print_state.py --no-dms diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index c5f60147be1..5e4591e9dda 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -53,7 +53,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +=== The plan is a no-op now and records the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -61,8 +61,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 8c0f865305b..81232b0a098 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -7,7 +7,7 @@ title "Deploy a job that references another: depends_on is recorded alongside th trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations -title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +title "The plan is a no-op now and records the record_deployment_history feature" dms_plan title "The service lists both jobs, each recorded at version 1" diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index bae04c4ac57..7d02457dd32 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1 +=== The plan records the record_deployment_history feature; the service holds the schema at version 1 >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -67,8 +67,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } @@ -117,25 +115,6 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged } } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "2" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "previous_version_id": "1" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} >>> [CLI] bundle plan -t prod -o json Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed @@ -150,8 +129,6 @@ A common practice is to use a username or principal name in this path, i.e. use "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "3", - "last_version_id": "2", "features": { "record_deployment_history": {} } @@ -188,25 +165,6 @@ Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> print_requests.py --dms //api/2.0/bundle -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "3" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "previous_version_id": "2" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script index e3bc9895307..4d01c2a32ec 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/script +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -4,7 +4,7 @@ title "Deploy: the deployment is created carrying this bundle's metadata" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle -title "The plan records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" +title "The plan records the record_deployment_history feature; the service holds the schema at version 1" dms_plan deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-metadata-pinned-${UNIQUE_NAME}/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index d92a009367b..374f0974244 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -130,7 +130,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend +=== The plan is a no-op and records the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -138,8 +138,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 2, "deployment_id": "[NUMID]", - "next_version_id": "3", - "last_version_id": "2", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/empty-grants/script b/acceptance/bundle/dms/empty-grants/script index 5b0abfe3c16..cdaa2ac2ee9 100644 --- a/acceptance/bundle/dms/empty-grants/script +++ b/acceptance/bundle/dms/empty-grants/script @@ -20,7 +20,7 @@ trace print_requests.py --dms //api/2.0/bundle title "Plan again: reading state back from the service works and reports no work" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" -title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" +title "The plan is a no-op and records the record_deployment_history feature" dms_plan title "The grants node emptied out, so the service no longer lists it - only the schema remains" diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index c06f8264a5c..8155cf41de6 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +=== The plan is a no-op now and records the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -67,8 +67,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index c343679ef14..5c92b76add4 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -4,7 +4,7 @@ title "Deploy: the job is recorded" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle -title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +title "The plan is a no-op now and records the record_deployment_history feature" dms_plan title "After deploy, the service lists the resource at version 1" diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 2d49fa2ae12..5562044988c 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend +=== The plan is a no-op and records the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -67,8 +67,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index f69f8840850..c3ec3cb57c5 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -4,7 +4,7 @@ title "Deploy the schema, so there is a resource to recreate" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle -title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" +title "The plan is a no-op and records the record_deployment_history feature" dms_plan title "After deploy, the service lists the resource at version 1" diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 1fa6048dbbf..651abd0b910 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -7,7 +7,7 @@ Created jobs.foo.permissions Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -=== The plan records DEPLOYMENT_METADATA_SERVICE backend and the job +=== The plan records the record_deployment_history feature and the job >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -15,8 +15,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index f9da3a3220f..4b8ad656693 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -4,7 +4,7 @@ title "Deploy the job and its permissions" trace $CLI bundle deploy rm -f out.requests.txt -title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the job" +title "The plan records the record_deployment_history feature and the job" dms_plan title "The service lists the job at version 1" diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 6452170caf1..c8f9746614b 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -17,7 +17,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.three", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.two", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} -=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +=== The plan records the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -25,8 +25,6 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } @@ -78,17 +76,15 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "state": {} } -=== Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed +=== Redeploy with no changes: nothing is applied, so no version is created at all - the deployment's version tracks state changes, like serial >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 5 unchanged >>> print_requests.py --dms //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +=== The plan records the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -96,8 +92,6 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "3", - "last_version_id": "2", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 8177c0be601..1d082038e4f 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -4,7 +4,7 @@ title "Deploy several resources: operations are uploaded from background workers trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline -title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +title "The plan records the record_deployment_history feature" dms_plan title "The service lists all five resources at version 1" @@ -14,11 +14,11 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d title "The raw recorded state the service holds for this deployment" trace print_state.py --no-dms -title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" +title "Redeploy with no changes: nothing is applied, so no version is created at all - the deployment's version tracks state changes, like serial" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --sort --oneline -title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +title "The plan records the record_deployment_history feature" dms_plan title "The service still lists all five resources, now at version 2" diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 508bc6e9942..4608edf5f1f 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -27,7 +27,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "latest_version_id": "1" } -=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +=== The plan is a no-op now and records the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -35,8 +35,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } @@ -164,7 +162,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "latest_version_id": "2" } -=== The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA_SERVICE as backend +=== The plan shows the updated pipeline at serial 2 with the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -172,8 +170,6 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "lineage": "[UUID]", "serial": 2, "deployment_id": "[NUMID]", - "next_version_id": "3", - "last_version_id": "2", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 203a2bebc05..afcca820fc6 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -7,7 +7,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle summary -o json | jq .bundle.deployment.history -title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +title "The plan is a no-op now and records the record_deployment_history feature" dms_plan title "The raw recorded state the service holds for this deployment" @@ -26,7 +26,7 @@ trace print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle summary -o json | jq .bundle.deployment.history -title "The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA_SERVICE as backend" +title "The plan shows the updated pipeline at serial 2 with the record_deployment_history feature" dms_plan title "The raw recorded state the service holds after the update" diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 9b79a57a05f..7dd5ec06ec7 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -19,24 +19,6 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged } } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "1" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state/resources.deployment.json { @@ -44,14 +26,12 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state/resources.deployment.json" } -=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +=== The plan records the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } @@ -102,34 +82,13 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "2" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "previous_version_id": "1" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} -=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +=== The plan records the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", - "next_version_id": "3", - "last_version_id": "2", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index d09f7a7faed..267abcacb45 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' -title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +title "The plan records the record_deployment_history feature" dms_plan title "The service lists no resources for the deployment" @@ -19,7 +19,7 @@ title "Redeploy: the deployment is resolved from that node, so no second deploym trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get -title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +title "The plan records the record_deployment_history feature" dms_plan title "The service still lists no resources" diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 6cd02954448..716f61f6cd7 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -6,7 +6,7 @@ Created schemas.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -=== The plan records DEPLOYMENT_METADATA_SERVICE backend and the schema +=== The plan records the record_deployment_history feature and the schema >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -14,8 +14,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index dc242c6262a..fd6f7f34742 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -4,7 +4,7 @@ title "Deploy so there is a recorded resource to recreate" trace $CLI bundle deploy rm -f out.requests.txt -title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the schema" +title "The plan records the record_deployment_history feature and the schema" dms_plan title "The service lists the schema at version 1" diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 3683349ac0f..de850339997 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -51,7 +51,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend +=== The plan is a no-op and records the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -59,8 +59,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index 7ffc5636eab..69cbdf80032 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -8,7 +8,7 @@ trace $CLI bundle deploy add_repl.py "$(git rev-parse HEAD)" COMMIT trace print_requests.py --dms //versions -title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" +title "The plan is a no-op and records the record_deployment_history feature" dms_plan title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index d6ecde2a879..802af232e45 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -16,8 +16,6 @@ Files: 6 uploaded, 0 deleted "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 4b4cfcd551c..22a99561ae2 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -44,13 +44,13 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "deployment_id": "[NUMID]", - "last_version_id": "1", - "next_version_id": "2" + "last_version_id": null, + "next_version_id": null } === Replay: re-deploying the same plan is rejected - it predates the recorded version >>> musterr [CLI] bundle deploy --plan tmp.plan.json -Error: this plan predates the deployment's current version 1; run 'bundle plan' again +Error: this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again === Only the first deploy recorded a version; the rejected replay created none diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index a525d256f2e..79d8610c28b 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +=== The plan is a no-op now and records the record_deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -67,8 +67,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", "features": { "record_deployment_history": {} } @@ -173,8 +171,6 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "lineage": "[UUID]", "serial": 2, "deployment_id": "[NUMID]", - "next_version_id": "3", - "last_version_id": "2", "features": { "record_deployment_history": {} } diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index 100441f9125..3440a4b2a9d 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -4,7 +4,7 @@ title "Deploy: the state write records the resource" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle -title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +title "The plan is a no-op now and records the record_deployment_history feature" dms_plan title "The service lists the schema, recorded at version 1" diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index fe66fa406d8..e5c4992db18 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -16,7 +16,6 @@ Files: 5 uploaded, 0 deleted "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", - "next_version_id": "1", "features": { "record_deployment_history": {} } diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index f96414a81da..73229fba952 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -21,13 +21,11 @@ type Plan struct { Lineage string `json:"lineage,omitempty"` Serial int `json:"serial,omitempty"` - // DMS fields, set only when the bundle records deployment history. The plan targets DeploymentId - // and will create NextVersionId; LastVersionId is the deployment's most recent version at plan - // time. deploy --plan rejects the plan if the deployment moved on from these (see process.go), - // and passes LastVersionId as previous_version_id so the service rejects a stale version too. - DeploymentId string `json:"deployment_id,omitempty"` - NextVersionId string `json:"next_version_id,omitempty"` - LastVersionId string `json:"last_version_id,omitempty"` + // DeploymentId is the recorded deployment this plan targets, set only when the bundle records + // deployment history. The version this plan would create is derived from Serial rather than + // carried separately: a version is only created when the plan changes something, so the two + // advance together (see phases.Deploy). deploy --plan rejects a plan whose deployment moved on. + DeploymentId string `json:"deployment_id,omitempty"` // Features are the state feature flags this plan was built against, mirroring the state // file's own "features" field. The stamps above exist because of a feature being set, so diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 0f09cb83fbb..07a0dcb3e10 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -8,7 +8,6 @@ import ( "maps" "reflect" "slices" - "strconv" "strings" "github.com/databricks/cli/bundle/config" @@ -16,7 +15,6 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/terraform_dabs_map" - "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" @@ -172,14 +170,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { plan.Features = b.StateDB.StateFeatures() if b.StateDB.DeploymentID != "" { - // Subsequent deployment: compute next version from current state. - next, err := dms.NextVersion(b.StateDB.LatestVersionID) - if err != nil { - return nil, fmt.Errorf("computing next deployment version: %w", err) - } plan.DeploymentId = b.StateDB.DeploymentID - plan.LastVersionId = b.StateDB.LatestVersionID - plan.NextVersionId = strconv.FormatInt(next, 10) } } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 531270bfc76..73a967c2243 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -347,17 +347,24 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Create the version the plan was stamped with, staging an operation for every resource // it touches. Doing it here rather than before the prompt means a declined deploy never // claims a version number. - staged, err := stagedOperations(plan) - if err != nil { - logdiag.LogError(ctx, err) - return - } - if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { - logdiag.LogError(ctx, err) - return + // + // A no-op plan creates no version: the deployment's version tracks state changes, the + // same way serial does, so a deploy that changes nothing leaves both untouched. With no + // version there is no operation buffer, so the deferred CompleteVersion is a no-op too. + counts := plan.CountActions() + if counts.Create+counts.Change+counts.Delete > 0 { + staged, err := stagedOperations(plan) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { + logdiag.LogError(ctx, err) + return + } + deploymentID, versionID := deploymentAndNextVersion(b) + logDeploymentVersion(ctx, b, deploymentID, versionID) } - deploymentID, versionID := deploymentAndNextVersion(b) - logDeploymentVersion(ctx, b, deploymentID, versionID) } deployCore(ctx, b, plan, stateEngine) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index fbe1bf968d6..bd869550b3e 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -485,13 +485,6 @@ func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, // tombstone (its serial does not catch a deploy from elsewhere). Gated on the plan being a DMS // plan; a first-deploy plan has empty version ids that still get compared (and match) here. if isDMSPlan { - remoteLastVersion := "" - if dmsDeployment != nil { - remoteLastVersion = dmsDeployment.LastVersionId - } - if plan.LastVersionId != remoteLastVersion { - return fmt.Errorf("this plan predates the deployment's current version %s; run 'bundle plan' again", remoteLastVersion) - } if plan.DeploymentId != dmsDeploymentID { return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") } From f488c8647b390b4caf744242daf99eceaf4282f2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 11:32:54 +0200 Subject: [PATCH 106/164] Rename record_deployment_history to deployment_history Per review: DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY becomes DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY and experimental.record_deployment_history becomes experimental.deployment_history. "record" adds nothing - recording is the only verb that applies to deployment history. The state feature key and the bundle-init env var are renamed to match, so state, plan and config all agree. Also gate version creation on stagedOperations returning work, rather than a separate predicate: a version is created exactly when serial would bump, and the walk that stages the operations is the one source of truth for that. Co-authored-by: Isaac --- acceptance/acceptance_test.go | 2 +- acceptance/bin/nostamp.py | 2 +- acceptance/bin/print_state.py | 2 +- acceptance/bin/read_id.py | 2 +- acceptance/bin/read_state.py | 2 +- acceptance/bin/replace_ids.py | 2 +- .../databricks.yml.tmpl | 2 +- .../dms/declined-deploy/databricks.yml.tmpl | 2 +- .../bundle/dms/declined-deploy/output.txt | 6 ++--- acceptance/bundle/dms/declined-deploy/script | 2 +- .../bundle/dms/depends-on/databricks.yml.tmpl | 2 +- acceptance/bundle/dms/depends-on/output.txt | 8 +++---- acceptance/bundle/dms/depends-on/script | 2 +- .../databricks.yml.tmpl | 2 +- .../dms/deployment-metadata-change/output.txt | 6 ++--- .../dms/deployment-metadata-change/script | 2 +- .../dms/empty-grants/databricks.yml.tmpl | 2 +- acceptance/bundle/dms/empty-grants/output.txt | 6 ++--- acceptance/bundle/dms/empty-grants/script | 2 +- .../bundle/dms/existing-state/output.txt | 22 ++++++++--------- acceptance/bundle/dms/existing-state/script | 14 +++++------ .../bundle/dms/existing-state/test.toml | 2 +- .../dms/failed-delete/databricks.yml.tmpl | 2 +- .../bundle/dms/failed-delete/output.txt | 8 +++---- acceptance/bundle/dms/failed-delete/script | 2 +- .../dms/failed-recreate/databricks.yml.tmpl | 2 +- .../bundle/dms/failed-recreate/output.txt | 8 +++---- acceptance/bundle/dms/failed-recreate/script | 2 +- .../databricks.yml.tmpl | 2 +- .../dms/failed-update-permissions/output.txt | 8 +++---- .../dms/failed-update-permissions/script | 2 +- .../multiple-resources/databricks.yml.tmpl | 2 +- .../bundle/dms/multiple-resources/output.txt | 12 +++++----- .../bundle/dms/multiple-resources/script | 4 ++-- .../bundle/dms/no-drift/databricks.yml.tmpl | 2 +- acceptance/bundle/dms/no-drift/output.txt | 12 +++++----- acceptance/bundle/dms/no-drift/script | 4 ++-- .../dms/no-resources/databricks.yml.tmpl | 2 +- acceptance/bundle/dms/no-resources/output.txt | 8 +++---- acceptance/bundle/dms/no-resources/script | 4 ++-- .../databricks.yml.tmpl | 2 +- .../operation-upload-fails-once/output.txt | 8 +++---- .../dms/operation-upload-fails-once/script | 2 +- acceptance/bundle/dms/plan-without-dms/script | 2 +- .../bundle/dms/provenance/databricks.yml.tmpl | 2 +- acceptance/bundle/dms/provenance/output.txt | 6 ++--- acceptance/bundle/dms/provenance/script | 2 +- .../dms/record-failure/databricks.yml.tmpl | 2 +- .../bundle/dms/record-failure/output.txt | 2 +- .../bundle/dms/stale-plan/databricks.yml.tmpl | 2 +- acceptance/bundle/dms/stale-plan/output.txt | 9 ++++--- acceptance/bundle/dms/stale-plan/script | 4 ++-- .../successful-recreate/databricks.yml.tmpl | 2 +- .../bundle/dms/successful-recreate/output.txt | 10 ++++---- .../bundle/dms/successful-recreate/script | 2 +- acceptance/bundle/dms/test.toml | 2 +- .../dms/version-never-created/databricks.yml | 2 +- .../dms/version-never-created/output.txt | 2 +- .../input.dbt-sql.json | 0 .../input.default-python.json | 0 .../input.default-scala.json | 0 .../input.default-sql.json | 0 .../input.lakeflow-integrations.json | 0 .../out.default-python.databricks.yml | 2 +- .../out.test.toml | 0 .../output.txt | 12 +++++----- .../script | 0 .../test.toml | 4 ++-- bundle/bundle.go | 6 ++--- bundle/config/experimental.go | 4 ++-- bundle/direct/dstate/state.go | 20 ++++++++-------- bundle/env/dms.go | 6 ++--- bundle/env/dms_test.go | 8 +++---- bundle/internal/schema/annotations.yml | 6 ++--- bundle/phases/deploy.go | 24 +++++++++++-------- bundle/schema/jsonschema.json | 8 +++---- cmd/bundle/utils/process.go | 2 +- libs/template/helpers.go | 10 ++++---- .../common/library/experimental.tmpl | 6 ++--- .../{{.project_name}}/databricks.yml.tmpl | 2 +- .../{{.project_name}}/databricks.yml.tmpl | 2 +- .../{{.project_name}}/databricks.yml.tmpl | 2 +- .../{{.project_name}}/databricks.yml.tmpl | 2 +- .../{{.project_name}}/databricks.yml.tmpl | 2 +- 84 files changed, 187 insertions(+), 184 deletions(-) rename acceptance/bundle/templates/{record-deployment-history => deployment-history}/input.dbt-sql.json (100%) rename acceptance/bundle/templates/{record-deployment-history => deployment-history}/input.default-python.json (100%) rename acceptance/bundle/templates/{record-deployment-history => deployment-history}/input.default-scala.json (100%) rename acceptance/bundle/templates/{record-deployment-history => deployment-history}/input.default-sql.json (100%) rename acceptance/bundle/templates/{record-deployment-history => deployment-history}/input.lakeflow-integrations.json (100%) rename acceptance/bundle/templates/{record-deployment-history => deployment-history}/out.default-python.databricks.yml (97%) rename acceptance/bundle/templates/{record-deployment-history => deployment-history}/out.test.toml (100%) rename acceptance/bundle/templates/{record-deployment-history => deployment-history}/output.txt (92%) rename acceptance/bundle/templates/{record-deployment-history => deployment-history}/script (100%) rename acceptance/bundle/templates/{record-deployment-history => deployment-history}/test.toml (57%) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index e244e065cd2..4da55ccd983 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -1082,7 +1082,7 @@ func checkEnvFilters(t *testing.T, testEnv, envFilters []string) { // matrix key ends up in the variant's test name, so a long one makes every name that carries // it hard to read. Tests may still name the variable itself; the alias is only shorter. var envAliases = map[string]string{ - "DMS": "DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY", + "DMS": "DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY", } // buildTestEnv builds the test environment from config.Env and customEnv. diff --git a/acceptance/bin/nostamp.py b/acceptance/bin/nostamp.py index 4a7d1c4fac8..f482962e51f 100755 --- a/acceptance/bin/nostamp.py +++ b/acceptance/bin/nostamp.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Read JSON on stdin, write it back with the DMS deployment stamp removed. -Deployment history recording (DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true; see +Deployment history recording (DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true; see bundle/test.toml) adds a deployment stamp to plans, states, and resource payloads. Pipe any of those through this so an acceptance golden compares equal whether or not recording is on. Output is a 2-space indent, keys in input order, <>& left unescaped, and integers at full precision. diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 8eec03940c7..42e55b39648 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -177,7 +177,7 @@ def main(): for filename in get_state_files(args.target, args.backup): if not os.path.exists(filename): continue - if os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true" and not args.no_dms: + if os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true" and not args.no_dms: print_recorded_state(filename, args.target) else: print_file(filename) diff --git a/acceptance/bin/read_id.py b/acceptance/bin/read_id.py index 6876455965b..9663a05a8ae 100755 --- a/acceptance/bin/read_id.py +++ b/acceptance/bin/read_id.py @@ -71,7 +71,7 @@ def main(): parser.add_argument("name") args = parser.parse_args() - if os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true": + if os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true": id = get_id_recorded(args.target, args.name) else: filename = get_state_file(args.target, args.backup) diff --git a/acceptance/bin/read_state.py b/acceptance/bin/read_state.py index 11d6e4b7b4d..278e519877a 100755 --- a/acceptance/bin/read_state.py +++ b/acceptance/bin/read_state.py @@ -65,7 +65,7 @@ def print_resource_recorded(group, name, *attrs): print(group, name, " ".join(values)) -if os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true": +if os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true": print_resource_recorded(*sys.argv[1:]) elif os.environ.get("DATABRICKS_BUNDLE_ENGINE", "").startswith("direct"): print_resource_direct(*sys.argv[1:]) diff --git a/acceptance/bin/replace_ids.py b/acceptance/bin/replace_ids.py index 09a04133610..5665c705de7 100755 --- a/acceptance/bin/replace_ids.py +++ b/acceptance/bin/replace_ids.py @@ -51,7 +51,7 @@ def main(): parser.add_argument("--backup", action="store_true") args = parser.parse_args() - if os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true": + if os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true": it = iter_ids_recorded(args.target) else: filename = get_state_file(args.target, args.backup) diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl b/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl index 7c9830e927c..e60f4605610 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl +++ b/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-bind-unbind-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: jobs: diff --git a/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl b/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl index e909afaab5c..7c48f1b7317 100644 --- a/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl +++ b/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-declined-deploy-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: schemas: diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index b5d43f5e729..e43ea615012 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op now and records the record_deployment_history feature; the service holds the schema at version 1 +=== The plan is a no-op now and records the deployment_history feature; the service holds the schema at version 1 >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -68,7 +68,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -100,7 +100,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index bf486fe98f5..483397ea410 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -4,7 +4,7 @@ title "Deploy a schema, so the deployment and its first version exist" trace $CLI bundle deploy trace print_requests.py --dms --get //api/2.0/bundle -title "The plan is a no-op now and records the record_deployment_history feature; the service holds the schema at version 1" +title "The plan is a no-op now and records the deployment_history feature; the service holds the schema at version 1" dms_plan trace print_requests.py --dms --get //api/2.0/bundle trace print_state.py --no-dms diff --git a/acceptance/bundle/dms/depends-on/databricks.yml.tmpl b/acceptance/bundle/dms/depends-on/databricks.yml.tmpl index 1a60d215ecc..1b81c2ba6d8 100644 --- a/acceptance/bundle/dms/depends-on/databricks.yml.tmpl +++ b/acceptance/bundle/dms/depends-on/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-depends-on-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: jobs: diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 5e4591e9dda..2ca629bab12 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -5,7 +5,7 @@ "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -53,7 +53,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op now and records the record_deployment_history feature +=== The plan is a no-op now and records the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -62,7 +62,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -92,7 +92,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 81232b0a098..fba2628a765 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -7,7 +7,7 @@ title "Deploy a job that references another: depends_on is recorded alongside th trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations -title "The plan is a no-op now and records the record_deployment_history feature" +title "The plan is a no-op now and records the deployment_history feature" dms_plan title "The service lists both jobs, each recorded at version 1" diff --git a/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl b/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl index 07f434b2e7a..2d22f7c1463 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl +++ b/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-metadata-before-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true workspace: # Pinned, so renaming the bundle or switching target below keeps the same deployment: the diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 7d02457dd32..de5b8565648 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan records the record_deployment_history feature; the service holds the schema at version 1 +=== The plan records the deployment_history feature; the service holds the schema at version 1 >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -68,7 +68,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -130,7 +130,7 @@ A common practice is to use a username or principal name in this path, i.e. use "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script index 4d01c2a32ec..44749821d9d 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/script +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -4,7 +4,7 @@ title "Deploy: the deployment is created carrying this bundle's metadata" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle -title "The plan records the record_deployment_history feature; the service holds the schema at version 1" +title "The plan records the deployment_history feature; the service holds the schema at version 1" dms_plan deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-metadata-pinned-${UNIQUE_NAME}/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' diff --git a/acceptance/bundle/dms/empty-grants/databricks.yml.tmpl b/acceptance/bundle/dms/empty-grants/databricks.yml.tmpl index 7be9f60129c..57bec803335 100644 --- a/acceptance/bundle/dms/empty-grants/databricks.yml.tmpl +++ b/acceptance/bundle/dms/empty-grants/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-empty-grants-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: schemas: diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index 374f0974244..ca3cfe5f595 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -130,7 +130,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== The plan is a no-op and records the record_deployment_history feature +=== The plan is a no-op and records the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -139,7 +139,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "serial": 2, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -164,7 +164,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } diff --git a/acceptance/bundle/dms/empty-grants/script b/acceptance/bundle/dms/empty-grants/script index cdaa2ac2ee9..be056eccf54 100644 --- a/acceptance/bundle/dms/empty-grants/script +++ b/acceptance/bundle/dms/empty-grants/script @@ -20,7 +20,7 @@ trace print_requests.py --dms //api/2.0/bundle title "Plan again: reading state back from the service works and reports no work" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" -title "The plan is a no-op and records the record_deployment_history feature" +title "The plan is a no-op and records the deployment_history feature" dms_plan title "The grants node emptied out, so the service no longer lists it - only the schema remains" diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 08073ddeac9..1b4a4f1eba7 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -26,35 +26,35 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time ->>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: - 1. remove experimental.record_deployment_history from your bundle configuration + 1. remove experimental.deployment_history from your bundle configuration 2. run "databricks bundle destroy" to delete the existing resources - 3. add experimental.record_deployment_history back and deploy again + 3. add experimental.deployment_history back and deploy again >>> print_requests.py --dms //api/2.0/bundle --oneline === The check is against the remote state, not the local cache: wiping .databricks still refuses, because the deploy pulls the state back from the workspace ->>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: - 1. remove experimental.record_deployment_history from your bundle configuration + 1. remove experimental.deployment_history from your bundle configuration 2. run "databricks bundle destroy" to delete the existing resources - 3. add experimental.record_deployment_history back and deploy again + 3. add experimental.deployment_history back and deploy again === A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands ->>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: - 1. remove experimental.record_deployment_history from your bundle configuration + 1. remove experimental.deployment_history from your bundle configuration 2. run "databricks bundle destroy" to delete the existing resources - 3. add experimental.record_deployment_history back and deploy again + 3. add experimental.deployment_history back and deploy again === Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed @@ -66,7 +66,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted ->>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Created jobs.one Files: 6 uploaded, 0 deleted @@ -84,6 +84,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "state_version": 2, "features": { - "record_deployment_history": {} + "deployment_history": {} } } diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index aa3bee78294..df6659f1ff0 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -7,24 +7,24 @@ trace print_requests.py --dms //api/2.0/bundle --oneline trace jq '{state_version, features}' .databricks/bundle/default/resources.json title "Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time" -musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy +musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --oneline title "The check is against the remote state, not the local cache: wiping .databricks still refuses, because the deploy pulls the state back from the workspace" rm -rf .databricks -musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy +musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands" -# The WAL header carries the record_deployment_history feature: if it were replayed, the +# The WAL header carries the deployment_history feature: if it were replayed, the # feature would merge into the state. The refusal below proves it is discarded, not read. mkdir -p .databricks/bundle/default -printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"record_deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal -musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy +printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal +musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed" trace $CLI bundle destroy --auto-approve -printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"record_deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal -trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy +printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --oneline trace find.py 'resources[.]json[.]wal' --expect 0 trace jq '{state_version, features}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/existing-state/test.toml b/acceptance/bundle/dms/existing-state/test.toml index bc95f7e86c4..7499498f419 100644 --- a/acceptance/bundle/dms/existing-state/test.toml +++ b/acceptance/bundle/dms/existing-state/test.toml @@ -1,3 +1,3 @@ # This test deploys without recording first, then turns it on per-command, so it opts out of -# the parent's DMS=true and drives DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY inline instead. +# the parent's DMS=true and drives DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY inline instead. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl b/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl index abdb86103ff..1c59553634d 100644 --- a/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl +++ b/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-failed-delete-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: jobs: diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 8155cf41de6..5a6e7ccc536 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op now and records the record_deployment_history feature +=== The plan is a no-op now and records the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -68,7 +68,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -93,7 +93,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } @@ -185,7 +185,7 @@ API message: Fault injected by test. "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index 5c92b76add4..29d79de938a 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -4,7 +4,7 @@ title "Deploy: the job is recorded" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle -title "The plan is a no-op now and records the record_deployment_history feature" +title "The plan is a no-op now and records the deployment_history feature" dms_plan title "After deploy, the service lists the resource at version 1" diff --git a/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl b/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl index a2937bf9a41..eadfeb09665 100644 --- a/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl +++ b/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-failed-recreate-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: schemas: diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 5562044988c..62f5afed12e 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op and records the record_deployment_history feature +=== The plan is a no-op and records the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -68,7 +68,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -93,7 +93,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } @@ -180,7 +180,7 @@ Files: 4 uploaded, 0 deleted "lineage": "[UUID]", "serial": 2, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index c3ec3cb57c5..ccc12f75744 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -4,7 +4,7 @@ title "Deploy the schema, so there is a resource to recreate" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle -title "The plan is a no-op and records the record_deployment_history feature" +title "The plan is a no-op and records the deployment_history feature" dms_plan title "After deploy, the service lists the resource at version 1" diff --git a/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl b/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl index 622598450cd..37b413a3ab2 100644 --- a/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl +++ b/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-failed-update-permissions-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: jobs: diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 651abd0b910..d3da57f4460 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -7,7 +7,7 @@ Created jobs.foo.permissions Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -=== The plan records the record_deployment_history feature and the job +=== The plan records the deployment_history feature and the job >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -16,7 +16,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -46,7 +46,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } @@ -101,7 +101,7 @@ Files: 3 uploaded, 0 deleted "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index 4b8ad656693..5cd1a2abf8d 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -4,7 +4,7 @@ title "Deploy the job and its permissions" trace $CLI bundle deploy rm -f out.requests.txt -title "The plan records the record_deployment_history feature and the job" +title "The plan records the deployment_history feature and the job" dms_plan title "The service lists the job at version 1" diff --git a/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl b/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl index faa6ef5e63c..ccb059f03c0 100644 --- a/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl +++ b/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-multiple-resources-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: jobs: diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index c8f9746614b..7cb49dff316 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -17,7 +17,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.three", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.two", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} -=== The plan records the record_deployment_history feature +=== The plan records the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -26,7 +26,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -71,7 +71,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } @@ -84,7 +84,7 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged >>> print_requests.py --dms //api/2.0/bundle --sort --oneline -=== The plan records the record_deployment_history feature +=== The plan records the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -93,7 +93,7 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -138,7 +138,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 1d082038e4f..54efc92ce13 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -4,7 +4,7 @@ title "Deploy several resources: operations are uploaded from background workers trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline -title "The plan records the record_deployment_history feature" +title "The plan records the deployment_history feature" dms_plan title "The service lists all five resources at version 1" @@ -18,7 +18,7 @@ title "Redeploy with no changes: nothing is applied, so no version is created at trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --sort --oneline -title "The plan records the record_deployment_history feature" +title "The plan records the deployment_history feature" dms_plan title "The service still lists all five resources, now at version 2" diff --git a/acceptance/bundle/dms/no-drift/databricks.yml.tmpl b/acceptance/bundle/dms/no-drift/databricks.yml.tmpl index e9892fa79a5..0e001d8a69e 100644 --- a/acceptance/bundle/dms/no-drift/databricks.yml.tmpl +++ b/acceptance/bundle/dms/no-drift/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-no-drift-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: jobs: diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 4608edf5f1f..05f3f6bfcd6 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -27,7 +27,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "latest_version_id": "1" } -=== The plan is a no-op now and records the record_deployment_history feature +=== The plan is a no-op now and records the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -36,7 +36,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -51,7 +51,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } @@ -162,7 +162,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "latest_version_id": "2" } -=== The plan shows the updated pipeline at serial 2 with the record_deployment_history feature +=== The plan shows the updated pipeline at serial 2 with the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -171,7 +171,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "serial": 2, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -186,7 +186,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index afcca820fc6..fcecb6d1f5c 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -7,7 +7,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle summary -o json | jq .bundle.deployment.history -title "The plan is a no-op now and records the record_deployment_history feature" +title "The plan is a no-op now and records the deployment_history feature" dms_plan title "The raw recorded state the service holds for this deployment" @@ -26,7 +26,7 @@ trace print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle summary -o json | jq .bundle.deployment.history -title "The plan shows the updated pipeline at serial 2 with the record_deployment_history feature" +title "The plan shows the updated pipeline at serial 2 with the deployment_history feature" dms_plan title "The raw recorded state the service holds after the update" diff --git a/acceptance/bundle/dms/no-resources/databricks.yml.tmpl b/acceptance/bundle/dms/no-resources/databricks.yml.tmpl index 973189b1603..6b847711abd 100644 --- a/acceptance/bundle/dms/no-resources/databricks.yml.tmpl +++ b/acceptance/bundle/dms/no-resources/databricks.yml.tmpl @@ -1,4 +1,4 @@ bundle: name: dms-no-resources-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 7dd5ec06ec7..40253afae5d 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -26,14 +26,14 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state/resources.deployment.json" } -=== The plan records the record_deployment_history feature +=== The plan records the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -83,14 +83,14 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } -=== The plan records the record_deployment_history feature +=== The plan records the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 267abcacb45..54856f2ecae 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' -title "The plan records the record_deployment_history feature" +title "The plan records the deployment_history feature" dms_plan title "The service lists no resources for the deployment" @@ -19,7 +19,7 @@ title "Redeploy: the deployment is resolved from that node, so no second deploym trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get -title "The plan records the record_deployment_history feature" +title "The plan records the deployment_history feature" dms_plan title "The service still lists no resources" diff --git a/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl index c46cfb03d73..f878a82cbcd 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl +++ b/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-upload-fails-once-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: schemas: diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 716f61f6cd7..de9fe5e5a42 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -6,7 +6,7 @@ Created schemas.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -=== The plan records the record_deployment_history feature and the schema +=== The plan records the deployment_history feature and the schema >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -15,7 +15,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -40,7 +40,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } @@ -82,7 +82,7 @@ Files: 4 uploaded, 0 deleted "lineage": "[UUID]", "serial": 2, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index fd6f7f34742..2e1a4c7f935 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -4,7 +4,7 @@ title "Deploy so there is a recorded resource to recreate" trace $CLI bundle deploy rm -f out.requests.txt -title "The plan records the record_deployment_history feature and the schema" +title "The plan records the deployment_history feature and the schema" dms_plan title "The service lists the schema at version 1" diff --git a/acceptance/bundle/dms/plan-without-dms/script b/acceptance/bundle/dms/plan-without-dms/script index 9f3cb824ab3..cffd78643ae 100644 --- a/acceptance/bundle/dms/plan-without-dms/script +++ b/acceptance/bundle/dms/plan-without-dms/script @@ -6,7 +6,7 @@ trace jq '{features, deployment_id, next_version_id}' < tmp.plan.json title "Enable DMS recording and try to deploy the non-DMS plan" update_file.py databricks.yml 'bundle:' 'experimental: - record_deployment_history: true + deployment_history: true bundle:' trace musterr $CLI bundle deploy --plan tmp.plan.json diff --git a/acceptance/bundle/dms/provenance/databricks.yml.tmpl b/acceptance/bundle/dms/provenance/databricks.yml.tmpl index 0a3fa1656ae..49c989275ad 100644 --- a/acceptance/bundle/dms/provenance/databricks.yml.tmpl +++ b/acceptance/bundle/dms/provenance/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-provenance-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true targets: dev: diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index de850339997..ab5a9e03159 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -51,7 +51,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op and records the record_deployment_history feature +=== The plan is a no-op and records the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -60,7 +60,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -101,7 +101,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index 69cbdf80032..3504c899606 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -8,7 +8,7 @@ trace $CLI bundle deploy add_repl.py "$(git rev-parse HEAD)" COMMIT trace print_requests.py --dms //versions -title "The plan is a no-op and records the record_deployment_history feature" +title "The plan is a no-op and records the deployment_history feature" dms_plan title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" diff --git a/acceptance/bundle/dms/record-failure/databricks.yml.tmpl b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl index f1aeba83d9a..d9c8a7928c2 100644 --- a/acceptance/bundle/dms/record-failure/databricks.yml.tmpl +++ b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-record-failure-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: jobs: diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 802af232e45..c6cdf2cbd5d 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -17,7 +17,7 @@ Files: 6 uploaded, 0 deleted "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } diff --git a/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl b/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl index dea8fbd9a8d..fc5e5d7028b 100644 --- a/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl +++ b/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-stale-plan-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: jobs: diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 22a99561ae2..325551e1e0a 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -5,7 +5,7 @@ "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -35,17 +35,16 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } -=== A saved plan carries the deployment id and the versions it targets (last 1, next 2) +=== A saved plan carries the deployment id and the serial it was built against >>> [CLI] bundle plan -o json { "deployment_id": "[NUMID]", - "last_version_id": null, - "next_version_id": null + "serial": 1 } === Replay: re-deploying the same plan is rejected - it predates the recorded version diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index bf7ae58d3f6..021a60378cb 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -13,8 +13,8 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d title "The raw recorded state the service holds for this deployment" trace print_state.py --no-dms -title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" -trace $CLI bundle plan -o json | jq '{deployment_id, last_version_id, next_version_id}' +title "A saved plan carries the deployment id and the serial it was built against" +trace $CLI bundle plan -o json | jq '{deployment_id, serial}' title "Replay: re-deploying the same plan is rejected - it predates the recorded version" trace musterr $CLI bundle deploy --plan tmp.plan.json diff --git a/acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl b/acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl index 8a40cabeda6..b48a55716e7 100644 --- a/acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl +++ b/acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: name: dms-successful-recreate-$UNIQUE_NAME experimental: - record_deployment_history: true + deployment_history: true resources: schemas: diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 79d8610c28b..bef65faa709 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== The plan is a no-op now and records the record_deployment_history feature +=== The plan is a no-op now and records the deployment_history feature >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -68,7 +68,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "serial": 1, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -93,7 +93,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } @@ -172,7 +172,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "serial": 2, "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } @@ -197,7 +197,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "record_deployment_history": {} + "deployment_history": {} }, "state": {} } diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index 3440a4b2a9d..808b1c49b97 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -4,7 +4,7 @@ title "Deploy: the state write records the resource" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle -title "The plan is a no-op now and records the record_deployment_history feature" +title "The plan is a no-op now and records the deployment_history feature" dms_plan title "The service lists the schema, recorded at version 1" diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 32a52823ff7..940d5bf58ad 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -12,7 +12,7 @@ Ignore = [ 'databricks.yml', ] -# These tests set experimental.record_deployment_history in databricks.yml; the DMS env var +# These tests set experimental.deployment_history in databricks.yml; the DMS env var # both records and permits that otherwise-gated field. bundle/dms/not-supported covers the # rejection path. Pinned to "true" (not the suite's ["", "true"]) since the field already # records - the empty variant would just duplicate it. diff --git a/acceptance/bundle/dms/version-never-created/databricks.yml b/acceptance/bundle/dms/version-never-created/databricks.yml index a7077f18b1f..4b5143d04bc 100644 --- a/acceptance/bundle/dms/version-never-created/databricks.yml +++ b/acceptance/bundle/dms/version-never-created/databricks.yml @@ -2,7 +2,7 @@ bundle: name: dms-version-never-created experimental: - record_deployment_history: true + deployment_history: true resources: jobs: diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index e5c4992db18..7d628b01d44 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -17,7 +17,7 @@ Files: 5 uploaded, 0 deleted "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "features": { - "record_deployment_history": {} + "deployment_history": {} } } diff --git a/acceptance/bundle/templates/record-deployment-history/input.dbt-sql.json b/acceptance/bundle/templates/deployment-history/input.dbt-sql.json similarity index 100% rename from acceptance/bundle/templates/record-deployment-history/input.dbt-sql.json rename to acceptance/bundle/templates/deployment-history/input.dbt-sql.json diff --git a/acceptance/bundle/templates/record-deployment-history/input.default-python.json b/acceptance/bundle/templates/deployment-history/input.default-python.json similarity index 100% rename from acceptance/bundle/templates/record-deployment-history/input.default-python.json rename to acceptance/bundle/templates/deployment-history/input.default-python.json diff --git a/acceptance/bundle/templates/record-deployment-history/input.default-scala.json b/acceptance/bundle/templates/deployment-history/input.default-scala.json similarity index 100% rename from acceptance/bundle/templates/record-deployment-history/input.default-scala.json rename to acceptance/bundle/templates/deployment-history/input.default-scala.json diff --git a/acceptance/bundle/templates/record-deployment-history/input.default-sql.json b/acceptance/bundle/templates/deployment-history/input.default-sql.json similarity index 100% rename from acceptance/bundle/templates/record-deployment-history/input.default-sql.json rename to acceptance/bundle/templates/deployment-history/input.default-sql.json diff --git a/acceptance/bundle/templates/record-deployment-history/input.lakeflow-integrations.json b/acceptance/bundle/templates/deployment-history/input.lakeflow-integrations.json similarity index 100% rename from acceptance/bundle/templates/record-deployment-history/input.lakeflow-integrations.json rename to acceptance/bundle/templates/deployment-history/input.lakeflow-integrations.json diff --git a/acceptance/bundle/templates/record-deployment-history/out.default-python.databricks.yml b/acceptance/bundle/templates/deployment-history/out.default-python.databricks.yml similarity index 97% rename from acceptance/bundle/templates/record-deployment-history/out.default-python.databricks.yml rename to acceptance/bundle/templates/deployment-history/out.default-python.databricks.yml index b9ede45b676..1de7c602976 100644 --- a/acceptance/bundle/templates/record-deployment-history/out.default-python.databricks.yml +++ b/acceptance/bundle/templates/deployment-history/out.default-python.databricks.yml @@ -5,7 +5,7 @@ bundle: uuid: [BUNDLE_UUID] experimental: - record_deployment_history: true + deployment_history: true include: - resources/*.yml diff --git a/acceptance/bundle/templates/record-deployment-history/out.test.toml b/acceptance/bundle/templates/deployment-history/out.test.toml similarity index 100% rename from acceptance/bundle/templates/record-deployment-history/out.test.toml rename to acceptance/bundle/templates/deployment-history/out.test.toml diff --git a/acceptance/bundle/templates/record-deployment-history/output.txt b/acceptance/bundle/templates/deployment-history/output.txt similarity index 92% rename from acceptance/bundle/templates/record-deployment-history/output.txt rename to acceptance/bundle/templates/deployment-history/output.txt index befcd703c14..a6f06bdcb77 100644 --- a/acceptance/bundle/templates/record-deployment-history/output.txt +++ b/acceptance/bundle/templates/deployment-history/output.txt @@ -12,7 +12,7 @@ To get started, refer to the project README.md file and the documentation at htt >>> grep -A1 ^experimental: output/default-python/my_default_python/databricks.yml experimental: - record_deployment_history: true + deployment_history: true Welcome to the default SQL template for Declarative Automation Bundles! @@ -26,7 +26,7 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde >>> grep -A1 ^experimental: output/default-sql/my_default_sql/databricks.yml experimental: - record_deployment_history: true + deployment_history: true Welcome to the default-scala template for Declarative Automation Bundles! @@ -40,7 +40,7 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde >>> grep -A1 ^experimental: output/default-scala/my_default_scala/databricks.yml experimental: - record_deployment_history: true + deployment_history: true Welcome to the dbt template for Declarative Automation Bundles! @@ -54,7 +54,7 @@ Refer to the README.md file for full "getting started" guide and production setu >>> grep -A1 ^experimental: output/dbt-sql/my_dbt_sql/databricks.yml experimental: - record_deployment_history: true + deployment_history: true Welcome to the Lakeflow integrations template! @@ -64,7 +64,7 @@ Please refer to the README.md file for "getting started" instructions. >>> grep -A1 ^experimental: output/lakeflow-integrations/my_lakeflow_integrations/databricks.yml experimental: - record_deployment_history: true + deployment_history: true >>> [CLI] bundle validate -o json -json.experimental.record_deployment_history = true; +json.experimental.deployment_history = true; diff --git a/acceptance/bundle/templates/record-deployment-history/script b/acceptance/bundle/templates/deployment-history/script similarity index 100% rename from acceptance/bundle/templates/record-deployment-history/script rename to acceptance/bundle/templates/deployment-history/script diff --git a/acceptance/bundle/templates/record-deployment-history/test.toml b/acceptance/bundle/templates/deployment-history/test.toml similarity index 57% rename from acceptance/bundle/templates/record-deployment-history/test.toml rename to acceptance/bundle/templates/deployment-history/test.toml index 89b85620c2a..10d5d3aa050 100644 --- a/acceptance/bundle/templates/record-deployment-history/test.toml +++ b/acceptance/bundle/templates/deployment-history/test.toml @@ -1,8 +1,8 @@ -Env.DATABRICKS_BUNDLE_INIT_RECORD_DEPLOYMENT_HISTORY = "true" +Env.DATABRICKS_BUNDLE_INIT_DEPLOYMENT_HISTORY = "true" # What is under test is what `bundle init` writes, so one engine is enough. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# The generated project sets experimental.record_deployment_history; DMS=true permits it so +# The generated project sets experimental.deployment_history; DMS=true permits it so # the validate step below loads instead of rejecting it. EnvMatrix.DMS = ["true"] diff --git a/bundle/bundle.go b/bundle/bundle.go index 1832be861c3..5d953e7d182 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -318,10 +318,10 @@ func (b *Bundle) WorkspaceClient(ctx context.Context) *databricks.WorkspaceClien } // ConfiguresDeploymentHistory reports whether this bundle is configured to record deployment history with the -// deployment metadata service, from experimental.record_deployment_history or -// DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY. +// deployment metadata service, from experimental.deployment_history or +// DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY. func (b *Bundle) ConfiguresDeploymentHistory(ctx context.Context) bool { - configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory + configured := b.Config.Experimental != nil && b.Config.Experimental.DeploymentHistory return env.RecordsDeploymentHistory(ctx, configured) } diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 005c4dfcade..babf44e2b3b 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -50,13 +50,13 @@ type Experimental struct { // at which point we can deprecate or remove this field all together. SkipNamePrefixForSchema bool `json:"skip_name_prefix_for_schema,omitempty"` - // RecordDeploymentHistory opts the bundle into the deployment metadata + // DeploymentHistory opts the bundle into the deployment metadata // service (DMS), which records deployment history and tracks what changed // across deployments. // // Only for bundles with no deployed resources yet: DMS becomes the source of // truth for their state. - RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` + DeploymentHistory bool `json:"deployment_history,omitempty"` } type Python struct { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index a8f0cee82c0..9d7cdd860af 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -57,7 +57,7 @@ const ( supportedStateVersion = featureStateVersion ) -// FeatureRecordDeploymentHistory marks a state whose resources are also recorded with the +// FeatureDeploymentHistory marks a state whose resources are also recorded with the // deployment metadata service. Both stores are kept in step, so the marker is what tells a // reader the two already agree. A CLI that does not know the name refuses the state rather // than deploying over a deployment it would leave the service out of step with. @@ -65,7 +65,7 @@ const ( // The marker is sticky: once a deployment is recorded, the service holds resources that a // CLI which is not recording must not touch. So turning recording off does not clear it, and // deploying such a state without recording is refused. -const FeatureRecordDeploymentHistory = "record_deployment_history" +const FeatureDeploymentHistory = "deployment_history" // featuresDocURL explains the state-features mechanism in the error shown when this CLI refuses a // state that depends on a feature it does not recognize. @@ -85,7 +85,7 @@ const ( // any feature not listed here is refused (see checkStateFeatures), so a newer CLI's feature is not // silently clobbered by this one. var recognizedFeatures = map[string]struct{}{ - FeatureRecordDeploymentHistory: {}, + FeatureDeploymentHistory: {}, } // assertNoUnsupportedFeatures refuses a state that depends on a feature this CLI does not recognize. @@ -110,9 +110,9 @@ var errStaleWAL = errors.New("stale WAL") // ErrUnsettingRecording is returned by Open when a recorded state is opened without recording - the // config turned the feature off, or an operation that never records (unbind) reached it. Callers // present an operation-appropriate message via errors.Is. -var ErrUnsettingRecording = errors.New(`unsetting experimental.record_deployment_history is not supported +var ErrUnsettingRecording = errors.New(`unsetting experimental.deployment_history is not supported -This deployment's resources are recorded with the deployment history feature enabled. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) +This deployment's resources are recorded with the deployment history feature enabled. Set experimental.deployment_history: true to deploy or destroy this bundle`) type DeploymentState struct { Path string @@ -556,21 +556,21 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe // the config bootstraps the marker there. On an existing deployment the state is authoritative: // the config can neither start recording one that was not (its resources would be created a // second time) nor stop recording one that is (the service still holds it). - _, recorded := db.Data.Features[FeatureRecordDeploymentHistory] + _, recorded := db.Data.Features[FeatureDeploymentHistory] switch { case recording && !recorded && len(db.Data.State) == 0: if db.Data.Features == nil { db.Data.Features = make(map[string]struct{}, 1) } - db.Data.Features[FeatureRecordDeploymentHistory] = struct{}{} + db.Data.Features[FeatureDeploymentHistory] = struct{}{} recorded = true case recording && !recorded: return errors.New(`this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: - 1. remove experimental.record_deployment_history from your bundle configuration + 1. remove experimental.deployment_history from your bundle configuration 2. run "databricks bundle destroy" to delete the existing resources - 3. add experimental.record_deployment_history back and deploy again`) + 3. add experimental.deployment_history back and deploy again`) case !recording && recorded: return ErrUnsettingRecording } @@ -951,7 +951,7 @@ func (db *DeploymentState) dataForFile() Database { if header.Features == nil { header.Features = make(map[string]struct{}) } - header.Features[FeatureRecordDeploymentHistory] = struct{}{} + header.Features[FeatureDeploymentHistory] = struct{}{} return Database{ Header: header, diff --git a/bundle/env/dms.go b/bundle/env/dms.go index 66779aa3fe4..2474b468da4 100644 --- a/bundle/env/dms.go +++ b/bundle/env/dms.go @@ -2,10 +2,10 @@ package env import "context" -// RecordDeploymentHistoryVariable enables recording without setting the config field. +// DeploymentHistoryVariable enables recording without setting the config field. // Exists for CLI acceptance tests so the whole bundle suite runs with recording on. // Deliberately undocumented. -const RecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY" +const DeploymentHistoryVariable = "DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY" // RecordsDeploymentHistory reports whether recording is on, from config or env var. Single predicate // for all recording code paths, keeping them in sync. The env var only counts as "true": anything @@ -14,6 +14,6 @@ func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { if configured { return true } - value, _ := get(ctx, []string{RecordDeploymentHistoryVariable}) + value, _ := get(ctx, []string{DeploymentHistoryVariable}) return value == "true" } diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go index cfd91c0007a..469b7707e85 100644 --- a/bundle/env/dms_test.go +++ b/bundle/env/dms_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestRecordDeploymentHistoryEnv(t *testing.T) { +func TestDeploymentHistoryEnv(t *testing.T) { for _, tc := range []struct { value string want bool @@ -21,12 +21,12 @@ func TestRecordDeploymentHistoryEnv(t *testing.T) { {"TRUE", false}, {"yes", false}, } { - ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, tc.value) + ctx := env.Set(t.Context(), DeploymentHistoryVariable, tc.value) assert.Equal(t, tc.want, RecordsDeploymentHistory(ctx, false), "value %q", tc.value) } } -func TestRecordDeploymentHistoryEnvUnset(t *testing.T) { +func TestDeploymentHistoryEnvUnset(t *testing.T) { assert.False(t, RecordsDeploymentHistory(t.Context(), false)) } @@ -36,6 +36,6 @@ func TestRecordsDeploymentHistory(t *testing.T) { assert.True(t, RecordsDeploymentHistory(t.Context(), true)) assert.False(t, RecordsDeploymentHistory(t.Context(), false)) - ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, "true") + ctx := env.Set(t.Context(), DeploymentHistoryVariable, "true") assert.True(t, RecordsDeploymentHistory(ctx, false)) } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index e0b712c8ceb..f6d2297a3fc 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -157,6 +157,9 @@ experimental: "description": |- Defines attributes for experimental features. "$fields": + "deployment_history": + "description": |- + Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. "immutable_folder": "description": |- Whether to deploy bundle files and artifacts as a single immutable snapshot. When true, all files are packaged into a content-addressed archive and workspace.file_path and workspace.artifact_path are set to the resulting location. @@ -193,9 +196,6 @@ experimental: "python_wheel_wrapper": "description": |- Whether to use a Python wheel wrapper. - "record_deployment_history": - "description": |- - Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. "scripts": "description": |- The commands to run. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 73a967c2243..4b459782b15 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -348,16 +348,20 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // it touches. Doing it here rather than before the prompt means a declined deploy never // claims a version number. // - // A no-op plan creates no version: the deployment's version tracks state changes, the - // same way serial does, so a deploy that changes nothing leaves both untouched. With no - // version there is no operation buffer, so the deferred CompleteVersion is a no-op too. - counts := plan.CountActions() - if counts.Create+counts.Change+counts.Delete > 0 { - staged, err := stagedOperations(plan) - if err != nil { - logdiag.LogError(ctx, err) - return - } + // Create a version exactly when serial would bump. Serial advances only when the WAL + // gets an entry, i.e. when a resource state write happens, which is what a non-skip + // action leads to - so the two stay in step and the version can be derived from serial. + // The version has to exist before apply records into it, so the plan is where that is + // predicted, via stagedOperations - the same walk that stages them, so there is one + // source of truth. With no version there is no operation buffer, so the deferred + // CompleteVersion is a no-op too. A non-skip action that fails still gets a version, + // which is intended: the history should show the failure. + staged, err := stagedOperations(plan) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if len(staged) > 0 { if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { logdiag.LogError(ctx, err) return diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 6383b5ac2c5..368e50eea5a 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3322,6 +3322,10 @@ { "type": "object", "properties": { + "deployment_history": { + "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.", + "$ref": "#/$defs/bool" + }, "immutable_folder": { "description": "Whether to deploy bundle files and artifacts as a single immutable snapshot. When true, all files are packaged into a content-addressed archive and workspace.file_path and workspace.artifact_path are set to the resulting location.", "$ref": "#/$defs/bool" @@ -3340,10 +3344,6 @@ "description": "Whether to use a Python wheel wrapper.", "$ref": "#/$defs/bool" }, - "record_deployment_history": { - "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.", - "$ref": "#/$defs/bool" - }, "scripts": { "description": "The commands to run.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Command" diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index bd869550b3e..4872a695631 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -476,7 +476,7 @@ func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, if !maps.Equal(plan.Features, b.DeploymentBundle.StateDB.StateFeatures()) { return errors.New("this plan was created for a different set of state features than the target now has; run 'bundle plan' again") } - _, isDMSPlan := plan.Features[dstate.FeatureRecordDeploymentHistory] + _, isDMSPlan := plan.Features[dstate.FeatureDeploymentHistory] // The plan records the DMS deployment and version it targeted. Reject it if the live deployment // moved on - a newer version (someone deployed since, possibly from another machine) or a diff --git a/libs/template/helpers.go b/libs/template/helpers.go index a02efae3ea9..acf651beb85 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -53,13 +53,13 @@ var metastoreDisabledErrorCodes = []string{ // is run and can be used to attribute DBU revenue to bundle templates. var bundleUuid = uuid.New().String() -// recordDeploymentHistoryEnvVar makes the built-in templates set -// experimental.record_deployment_history in the databricks.yml they generate. It is scoped to +// deploymentHistoryEnvVar makes the built-in templates set +// experimental.deployment_history in the databricks.yml they generate. It is scoped to // `bundle init` because it persists the setting in the generated project rather than applying // it to the bundle being run. // // Temporary: remove it and its callers in the templates once this is the default. -const recordDeploymentHistoryEnvVar = "DATABRICKS_BUNDLE_INIT_RECORD_DEPLOYMENT_HISTORY" +const deploymentHistoryEnvVar = "DATABRICKS_BUNDLE_INIT_DEPLOYMENT_HISTORY" func loadHelpers(ctx context.Context) template.FuncMap { w := cmdctx.WorkspaceClient(ctx) @@ -183,8 +183,8 @@ func loadHelpers(ctx context.Context) template.FuncMap { cachedIsServicePrincipal = &result return result, nil }, - "record_deployment_history": func() bool { - v, _ := env.GetBool(ctx, recordDeploymentHistoryEnvVar) + "deployment_history": func() bool { + v, _ := env.GetBool(ctx, deploymentHistoryEnvVar) return v }, "lower": func(s string) string { diff --git a/libs/template/templates/common/library/experimental.tmpl b/libs/template/templates/common/library/experimental.tmpl index d55f4234b02..2e08be2eb7c 100644 --- a/libs/template/templates/common/library/experimental.tmpl +++ b/libs/template/templates/common/library/experimental.tmpl @@ -1,15 +1,15 @@ {{- /* * Temporary opt-in for recording deployment history; remove once it is the default. * - * The flag is passed in rather than read from the record_deployment_history helper here, + * The flag is passed in rather than read from the deployment_history helper here, * because this shared library is parsed for every template, including by callers that * register no helpers. */ -}} -{{- define "experimental_record_deployment_history" -}} +{{- define "experimental_deployment_history" -}} {{- if .}} experimental: - record_deployment_history: true + deployment_history: true {{- end}} {{- end}} diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl index ddc236733bf..21c82234a83 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -4,7 +4,7 @@ bundle: name: {{.project_name}} uuid: {{bundle_uuid}} -{{- template "experimental_record_deployment_history" record_deployment_history}} +{{- template "experimental_deployment_history" deployment_history}} include: - resources/*.yml diff --git a/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl index ad7a32de3e8..749f2a4e993 100644 --- a/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl @@ -3,7 +3,7 @@ bundle: name: {{.project_name}} uuid: {{bundle_uuid}} -{{- template "experimental_record_deployment_history" record_deployment_history}} +{{- template "experimental_deployment_history" deployment_history}} include: - resources/*.yml diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl index 9749477f312..0645de3dadf 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -3,7 +3,7 @@ bundle: name: {{.project_name}} uuid: {{bundle_uuid}} -{{- template "experimental_record_deployment_history" record_deployment_history}} +{{- template "experimental_deployment_history" deployment_history}} include: - resources/*.yml diff --git a/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl index 6f86e9ae9b1..73cddd42052 100644 --- a/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl @@ -6,7 +6,7 @@ bundle: name: {{.project_name}} uuid: {{bundle_uuid}} -{{- template "experimental_record_deployment_history" record_deployment_history}} +{{- template "experimental_deployment_history" deployment_history}} {{- if $with_pydabs }} python: diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl index 0d781d38a8c..abcba39e771 100644 --- a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl @@ -4,7 +4,7 @@ bundle: name: {{.project_name}} uuid: {{bundle_uuid}} engine: direct -{{- template "experimental_record_deployment_history" record_deployment_history}} +{{- template "experimental_deployment_history" deployment_history}} include: - resources/*.yml From f7ac9972279f5f7a53bc7b1ffc96e03b85328292 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 11:39:08 +0200 Subject: [PATCH 107/164] bundle/phases: trim the version-creation comment Co-authored-by: Isaac --- bundle/phases/deploy.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 4b459782b15..5562c62ad2e 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -344,18 +344,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } } - // Create the version the plan was stamped with, staging an operation for every resource - // it touches. Doing it here rather than before the prompt means a declined deploy never - // claims a version number. - // - // Create a version exactly when serial would bump. Serial advances only when the WAL - // gets an entry, i.e. when a resource state write happens, which is what a non-skip - // action leads to - so the two stay in step and the version can be derived from serial. - // The version has to exist before apply records into it, so the plan is where that is - // predicted, via stagedOperations - the same walk that stages them, so there is one - // source of truth. With no version there is no operation buffer, so the deferred - // CompleteVersion is a no-op too. A non-skip action that fails still gets a version, - // which is intended: the history should show the failure. + // Only create a version when the plan has at least one operation to record, so it moves + // with serial. Done after the prompt, so a declined deploy never claims a number. staged, err := stagedOperations(plan) if err != nil { logdiag.LogError(ctx, err) From 538852eae7b0920c81d11fcb739519404f06a09e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 11:58:17 +0200 Subject: [PATCH 108/164] Validate serial against the deployment's last version; cover the stale-version case Add a guard that state serial never runs ahead of the deployment's last recorded version. Ahead means state advanced without a version being recorded, so the next version number is already taken. It may legitimately lag: a version whose deploy failed is recorded with no state write. Also fix a misleading rejection. A plan built before the first deploy carries no deployment id, so comparing ids reported "targets a different deployment" when the plan was merely stale; that case now says it predates the deployment. stale-plan gains the other shape too - a plan against the right deployment but an outdated version - which the serial guard catches: "plan serial 1 does not match state serial 2". Co-authored-by: Isaac --- acceptance/bundle/dms/stale-plan/output.txt | 27 ++++++++++++++++++++- acceptance/bundle/dms/stale-plan/script | 15 +++++++++++- cmd/bundle/utils/process.go | 21 ++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 325551e1e0a..669c2f7058f 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -49,7 +49,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Replay: re-deploying the same plan is rejected - it predates the recorded version >>> musterr [CLI] bundle deploy --plan tmp.plan.json -Error: this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again +Error: this plan predates the deployment now recorded for this bundle; run 'bundle plan' again === Only the first deploy recorded a version; the rejected replay created none @@ -58,3 +58,28 @@ Error: this plan targets a different deployment than the one now recorded for th {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +=== Save a plan against the recorded deployment, then move the deployment on +>>> jq {deployment_id, serial} tmp.plan2.json +{ + "deployment_id": "[NUMID]", + "serial": 1 +} + +>>> update_file.py databricks.yml name: foo name: foo_renamed + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 4 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> jq {serial} .databricks/bundle/default/resources.json +{ + "serial": 2 +} + +=== Replaying the older plan is rejected: same deployment, outdated version +>>> musterr [CLI] bundle deploy --plan tmp.plan2.json +Error: plan serial 1 does not match state serial 2; the state has been modified since the plan was created. Please run 'bundle plan' again + diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index 021a60378cb..b1fcb40110d 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -22,4 +22,17 @@ trace musterr $CLI bundle deploy --plan tmp.plan.json title "Only the first deploy recorded a version; the rejected replay created none" trace print_requests.py --dms //api/2.0/bundle --oneline -rm -f tmp.plan.json out.requests.txt +# The replay above predates the deployment entirely (its plan was built before one existed). +# This covers the other shape: a plan that targets the right deployment but an outdated +# version, which the serial guard has to catch. +title "Save a plan against the recorded deployment, then move the deployment on" +$CLI bundle plan -o json > tmp.plan2.json +trace jq '{deployment_id, serial}' tmp.plan2.json +trace update_file.py databricks.yml "name: foo" "name: foo_renamed" +trace $CLI bundle deploy +trace jq '{serial}' .databricks/bundle/default/resources.json + +title "Replaying the older plan is rejected: same deployment, outdated version" +trace musterr $CLI bundle deploy --plan tmp.plan2.json + +rm -f tmp.plan.json tmp.plan2.json out.requests.txt diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 4872a695631..fd46cf3d5bb 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -293,6 +293,22 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } + // A version is only created when serial bumps, so the recorded last version and + // the state serial must agree. If they do not, the local state and the recorded + // history have diverged - a deploy from elsewhere, or restored state - and going + // on would claim a version the service already has. + // A version is only created when serial bumps, so serial must never run ahead of + // the recorded history. It may lag: a version whose deploy failed is recorded + // without a state write. Ahead means state advanced with no version recorded, so + // the next version number is already taken. + if lastVersionID != "" { + last, cerr := strconv.Atoi(lastVersionID) + if cerr == nil && b.DeploymentBundle.StateDB.Data.Serial > last { + logdiag.LogError(ctx, fmt.Errorf("deployment state is ahead of its recorded history: state serial is %d but the deployment's last version is %s; destroy and redeploy to resync", b.DeploymentBundle.StateDB.Data.Serial, lastVersionID)) + return b, stateDesc, root.ErrAlreadyPrinted + } + } + // StateDB owns the plan's lineage: CalculatePlan reads these, not the config tree. b.DeploymentBundle.StateDB.DeploymentID = dmsDeploymentID b.DeploymentBundle.StateDB.LatestVersionID = lastVersionID @@ -485,6 +501,11 @@ func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, // tombstone (its serial does not catch a deploy from elsewhere). Gated on the plan being a DMS // plan; a first-deploy plan has empty version ids that still get compared (and match) here. if isDMSPlan { + // A plan built before the first deploy carries no deployment id; once one exists the plan + // predates it, which is staleness rather than a different target. + if plan.DeploymentId == "" && dmsDeploymentID != "" { + return errors.New("this plan predates the deployment now recorded for this bundle; run 'bundle plan' again") + } if plan.DeploymentId != dmsDeploymentID { return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") } From 26792ff2619c2000b600f697e53a5dcdd52231dc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 12:14:09 +0200 Subject: [PATCH 109/164] Reject a stale plan on serial vs the deployment's last version Drops the deploy-path "serial ahead of history" guard as unnecessary, and replaces the deployment-id special case with a single check in validatePlan: if the plan's serial is behind the deployment's last recorded version, the plan is stale. That covers a plan from before the first deploy too, since its serial is 0. stale-plan asserts both shapes through the one message: serial 0 vs version 1, and serial 1 vs version 2 against the same deployment. Co-authored-by: Isaac --- acceptance/bundle/dms/stale-plan/output.txt | 4 +-- cmd/bundle/utils/process.go | 36 ++++++--------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 669c2f7058f..67e59ecf859 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -49,7 +49,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Replay: re-deploying the same plan is rejected - it predates the recorded version >>> musterr [CLI] bundle deploy --plan tmp.plan.json -Error: this plan predates the deployment now recorded for this bundle; run 'bundle plan' again +Error: this plan was built against serial 0 but the deployment has recorded version 1; run 'bundle plan' again === Only the first deploy recorded a version; the rejected replay created none @@ -81,5 +81,5 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Replaying the older plan is rejected: same deployment, outdated version >>> musterr [CLI] bundle deploy --plan tmp.plan2.json -Error: plan serial 1 does not match state serial 2; the state has been modified since the plan was created. Please run 'bundle plan' again +Error: this plan was built against serial 1 but the deployment has recorded version 2; run 'bundle plan' again diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index fd46cf3d5bb..0311623365c 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -293,22 +293,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } - // A version is only created when serial bumps, so the recorded last version and - // the state serial must agree. If they do not, the local state and the recorded - // history have diverged - a deploy from elsewhere, or restored state - and going - // on would claim a version the service already has. - // A version is only created when serial bumps, so serial must never run ahead of - // the recorded history. It may lag: a version whose deploy failed is recorded - // without a state write. Ahead means state advanced with no version recorded, so - // the next version number is already taken. - if lastVersionID != "" { - last, cerr := strconv.Atoi(lastVersionID) - if cerr == nil && b.DeploymentBundle.StateDB.Data.Serial > last { - logdiag.LogError(ctx, fmt.Errorf("deployment state is ahead of its recorded history: state serial is %d but the deployment's last version is %s; destroy and redeploy to resync", b.DeploymentBundle.StateDB.Data.Serial, lastVersionID)) - return b, stateDesc, root.ErrAlreadyPrinted - } - } - // StateDB owns the plan's lineage: CalculatePlan reads these, not the config tree. b.DeploymentBundle.StateDB.DeploymentID = dmsDeploymentID b.DeploymentBundle.StateDB.LatestVersionID = lastVersionID @@ -494,17 +478,17 @@ func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, } _, isDMSPlan := plan.Features[dstate.FeatureDeploymentHistory] - // The plan records the DMS deployment and version it targeted. Reject it if the live deployment - // moved on - a newer version (someone deployed since, possibly from another machine) or a - // different id (deleted and recreated) - so a stale plan is never applied on top of newer state. - // This is the authoritative stale guard for a recorded bundle, whose local state is only a - // tombstone (its serial does not catch a deploy from elsewhere). Gated on the plan being a DMS - // plan; a first-deploy plan has empty version ids that still get compared (and match) here. + // Reject a plan the live deployment has moved past, so it is never applied on top of newer + // state. The service is authoritative here: a recorded bundle's local state is a tombstone, + // so its serial alone does not catch a deploy from another machine. if isDMSPlan { - // A plan built before the first deploy carries no deployment id; once one exists the plan - // predates it, which is staleness rather than a different target. - if plan.DeploymentId == "" && dmsDeploymentID != "" { - return errors.New("this plan predates the deployment now recorded for this bundle; run 'bundle plan' again") + // The plan is stale if the deployment has recorded a version past the serial it was built + // against. Covers a plan from before the first deploy too: its serial is 0. + if dmsDeployment != nil && dmsDeployment.LastVersionId != "" { + last, cerr := strconv.Atoi(dmsDeployment.LastVersionId) + if cerr == nil && plan.Serial < last { + return fmt.Errorf("this plan was built against serial %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, last) + } } if plan.DeploymentId != dmsDeploymentID { return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") From 243b3c4085cad6186b67362bb7da5452730b36b3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 12:36:44 +0200 Subject: [PATCH 110/164] Say "version" not "serial" in the stale-plan error Serial is how the version number is stored; the user-facing concept is the version. Co-authored-by: Isaac --- acceptance/bundle/dms/stale-plan/output.txt | 6 +++--- acceptance/bundle/dms/stale-plan/script | 2 +- cmd/bundle/utils/process.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 67e59ecf859..ced4af35ac2 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -40,7 +40,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "state": {} } -=== A saved plan carries the deployment id and the serial it was built against +=== A saved plan carries the deployment id and the version it was built against >>> [CLI] bundle plan -o json { "deployment_id": "[NUMID]", @@ -49,7 +49,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Replay: re-deploying the same plan is rejected - it predates the recorded version >>> musterr [CLI] bundle deploy --plan tmp.plan.json -Error: this plan was built against serial 0 but the deployment has recorded version 1; run 'bundle plan' again +Error: this plan was built against version 0 but the deployment has recorded version 1; run 'bundle plan' again === Only the first deploy recorded a version; the rejected replay created none @@ -81,5 +81,5 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Replaying the older plan is rejected: same deployment, outdated version >>> musterr [CLI] bundle deploy --plan tmp.plan2.json -Error: this plan was built against serial 1 but the deployment has recorded version 2; run 'bundle plan' again +Error: this plan was built against version 1 but the deployment has recorded version 2; run 'bundle plan' again diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index b1fcb40110d..cef55ecd6e4 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -13,7 +13,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d title "The raw recorded state the service holds for this deployment" trace print_state.py --no-dms -title "A saved plan carries the deployment id and the serial it was built against" +title "A saved plan carries the deployment id and the version it was built against" trace $CLI bundle plan -o json | jq '{deployment_id, serial}' title "Replay: re-deploying the same plan is rejected - it predates the recorded version" diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 0311623365c..93781bb20d2 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -482,12 +482,12 @@ func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, // state. The service is authoritative here: a recorded bundle's local state is a tombstone, // so its serial alone does not catch a deploy from another machine. if isDMSPlan { - // The plan is stale if the deployment has recorded a version past the serial it was built - // against. Covers a plan from before the first deploy too: its serial is 0. + // The plan is stale if the deployment has recorded a version past the one the plan was + // built against. Covers a plan from before the first deploy too: its version is 0. if dmsDeployment != nil && dmsDeployment.LastVersionId != "" { last, cerr := strconv.Atoi(dmsDeployment.LastVersionId) if cerr == nil && plan.Serial < last { - return fmt.Errorf("this plan was built against serial %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, last) + return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, last) } } if plan.DeploymentId != dmsDeploymentID { From 873109b0140bbbc5b043b48eca7af7ffc2fd0189 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 13:06:52 +0200 Subject: [PATCH 111/164] acceptance/apps: run readplan-lifecycle under DMS The DMS+READPLAN exclude dated from when saved plans were piped through nostamp, which stripped the headers deploy --plan needs. Plans are kept raw now, so the saved-plan replay works under recording; both variants pass. Co-authored-by: Isaac --- acceptance/bundle/resources/apps/readplan-lifecycle/test.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/test.toml b/acceptance/bundle/resources/apps/readplan-lifecycle/test.toml index 0d83acb6bf3..6c0fc07b74a 100644 --- a/acceptance/bundle/resources/apps/readplan-lifecycle/test.toml +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/test.toml @@ -4,10 +4,6 @@ Ignore = [".databricks", "tmp.plan.json"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] -# Deploying from a pre-computed plan when the target has DMS deployment history -# is not supported; this variant is skipped. -EnvMatrixExclude.dms_readplan_mismatch = ["READPLAN=1", "DMS=true"] - [[Repls]] Old = '(?m)^✓ [^\n]*\n' New = "" From 7d1a57a6f7f51008028021601662d7b65c4f79de Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 13:16:06 +0200 Subject: [PATCH 112/164] bundle/direct/dstate: set DeploymentID in Open Open already receives the deployment id, so it should own the field rather than having the caller assign it again right afterwards. Addresses review feedback. Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 1 + cmd/bundle/utils/process.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 9d7cdd860af..ac37787ffe7 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -586,6 +586,7 @@ To record this bundle's history, start it over as a new deployment: return err } db.dmsClient = client + db.DeploymentID = dmsDeploymentID if dmsDeploymentID != "" { resources, err := db.dmsClient.ListResources(ctx, dmsDeploymentID) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 93781bb20d2..94b6c587fd6 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -293,8 +293,8 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } - // StateDB owns the plan's lineage: CalculatePlan reads these, not the config tree. - b.DeploymentBundle.StateDB.DeploymentID = dmsDeploymentID + // Open sets DeploymentID from the id passed to it; the version is not passed, so + // set it here. CalculatePlan reads these, not the config tree. b.DeploymentBundle.StateDB.LatestVersionID = lastVersionID } else { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { From c345ea2661d663dc104dbd6951a400bbbb7c5e38 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 13:46:33 +0200 Subject: [PATCH 113/164] bundle/direct: co-locate the plan-vs-state validations The recorded-deployment and version checks lived in cmd/bundle/utils while the lineage and serial checks lived in ValidatePlanAgainstState, even though all four answer the same question: is this saved plan still valid for this state? Move the first two next to the others. Both values they need are already on the state: DeploymentID (set by Open) and LatestVersionID. So the checks read the same source of truth as their neighbours and validatePlan no longer needs the deployment passed down to it. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 23 +++++++++++++++++++---- cmd/bundle/utils/process.go | 27 ++++----------------------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 07a0dcb3e10..d138605423e 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -8,6 +8,7 @@ import ( "maps" "reflect" "slices" + "strconv" "strings" "github.com/databricks/cli/bundle/config" @@ -39,15 +40,29 @@ func (b *DeploymentBundle) init(client *databricks.WorkspaceClient) error { return err } -// ValidatePlanAgainstState validates that a plan's lineage and serial match the given state. -// If the plan has no lineage (first deployment), validation is skipped. +// ValidatePlanAgainstState validates that a plan still matches the given state: the recorded +// deployment and version it targeted, then its lineage and serial. func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan.Plan) error { + stateDB.AssertOpenedForReadOrWrite() + + if stateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + // Stale if the deployment recorded a version past the one the plan was built against. + // Covers a plan from before the first deploy too: its version is 0. + if stateDB.LatestVersionID != "" { + last, err := strconv.Atoi(stateDB.LatestVersionID) + if err == nil && plan.Serial < last { + return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, last) + } + } + if plan.DeploymentId != stateDB.DeploymentID { + return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") + } + } + if plan.Lineage == "" { return nil } - stateDB.AssertOpenedForReadOrWrite() - if plan.Lineage != stateDB.Data.Lineage { return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 94b6c587fd6..d62ff246bbc 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -367,7 +367,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle log.Warnf(ctx, "Plan was created with CLI version %s but current version is %s", plan.CLIVersion, currentVersion) } - if err := validatePlan(ctx, b, plan, dmsDeployment, dmsDeploymentID); err != nil { + if err := validatePlan(b, plan); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } @@ -468,7 +468,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // validatePlan rejects a --plan file that no longer matches the target: its recording shape must // match the target's config, and a plan that targets an existing recorded deployment must not // predate the deployment the service now holds. It ends with the local lineage/serial guard. -func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, dmsDeployment *bundledeployments.Deployment, dmsDeploymentID string) error { +func validatePlan(b *bundle.Bundle, plan *deployplan.Plan) error { // A plan is built against a set of state features, and the stamps it carries follow from // them, so applying it to a target with a different set would deploy the wrong shape. // The state is the target's source of truth and carries its features even on a first @@ -476,27 +476,8 @@ func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, if !maps.Equal(plan.Features, b.DeploymentBundle.StateDB.StateFeatures()) { return errors.New("this plan was created for a different set of state features than the target now has; run 'bundle plan' again") } - _, isDMSPlan := plan.Features[dstate.FeatureDeploymentHistory] - - // Reject a plan the live deployment has moved past, so it is never applied on top of newer - // state. The service is authoritative here: a recorded bundle's local state is a tombstone, - // so its serial alone does not catch a deploy from another machine. - if isDMSPlan { - // The plan is stale if the deployment has recorded a version past the one the plan was - // built against. Covers a plan from before the first deploy too: its version is 0. - if dmsDeployment != nil && dmsDeployment.LastVersionId != "" { - last, cerr := strconv.Atoi(dmsDeployment.LastVersionId) - if cerr == nil && plan.Serial < last { - return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, last) - } - } - if plan.DeploymentId != dmsDeploymentID { - return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") - } - } - - // Validate that the plan's lineage and serial match the local state. This is the stale guard for - // non-recorded bundles (recorded ones are covered by the version check above). + // The rest of the plan-vs-state checks (recorded deployment and version, then lineage and + // serial) live together next to the state. return direct.ValidatePlanAgainstState(&b.DeploymentBundle.StateDB, plan) } From 90b5c3b431cbbd5d07e8f44309d5650d96e38565 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 13:56:46 +0200 Subject: [PATCH 114/164] bundle/deployplan: drop deployment_id from the plan, keep plan version at 2 The plan no longer carries the deployment id. Its only consumer was a check that the plan targeted the currently recorded deployment, and that check was already subsumed by the serial-vs-last-version one: a plan built against another deployment (or before this one existed) has a serial behind the version the deployment has recorded, so it is rejected as stale either way. bundle/dms/stale-plan covers both shapes and both now fail on the version comparison. Also revert the plan version bump. "features" is optional and its absence means no features, so an older plan still reads correctly and there is nothing to gate. Bumping only made a new CLI reject valid v2 plans; it did not improve the reverse case, because Load calls DisallowUnknownFields before checking the version, so an older CLI fails at `unknown field "features"` regardless of the number. Co-authored-by: Isaac --- acceptance/bin/nostamp.py | 2 +- .../bundle/dms/declined-deploy/output.txt | 1 - acceptance/bundle/dms/depends-on/output.txt | 3 +- acceptance/bundle/dms/depends-on/script | 2 +- .../dms/deployment-metadata-change/output.txt | 2 -- acceptance/bundle/dms/empty-grants/output.txt | 1 - .../bundle/dms/failed-delete/output.txt | 1 - .../bundle/dms/failed-recreate/output.txt | 1 - .../dms/failed-update-permissions/output.txt | 1 - .../bundle/dms/multiple-resources/output.txt | 2 -- acceptance/bundle/dms/no-drift/output.txt | 2 -- acceptance/bundle/dms/no-resources/output.txt | 2 -- .../operation-upload-fails-once/output.txt | 1 - acceptance/bundle/dms/provenance/output.txt | 1 - .../bundle/dms/record-failure/output.txt | 1 - acceptance/bundle/dms/stale-plan/output.txt | 6 ++-- acceptance/bundle/dms/stale-plan/script | 6 ++-- .../bundle/dms/successful-recreate/output.txt | 2 -- .../dms/version-never-created/output.txt | 1 - acceptance/test.toml | 4 +-- bundle/deployplan/plan.go | 8 +---- bundle/direct/bundle_plan.go | 30 +++++++------------ 22 files changed, 22 insertions(+), 58 deletions(-) diff --git a/acceptance/bin/nostamp.py b/acceptance/bin/nostamp.py index f482962e51f..6dcd7f8cbdd 100755 --- a/acceptance/bin/nostamp.py +++ b/acceptance/bin/nostamp.py @@ -22,7 +22,7 @@ _STAMP_KEYS = ("deployment_id", "version_id") _CHANGE_KEYS = ("deployment.deployment_id", "deployment.version_id") -_PLAN_HEADER_KEYS = ("deployment_id", "features") +_PLAN_HEADER_KEYS = ("features",) def scrub(node): diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index e43ea615012..c99c5cfdb15 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -66,7 +66,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 2ca629bab12..cb17c7fe406 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -1,5 +1,5 @@ -=== Plan before first deploy: no deployment yet, so no deployment_id or version fields +=== Plan before first deploy: no deployment yet, so serial is still 0 >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -60,7 +60,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index fba2628a765..635e83c47f7 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Plan before first deploy: no deployment yet, so no deployment_id or version fields" +title "Plan before first deploy: no deployment yet, so serial is still 0" dms_plan title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index de5b8565648..29a3fd6366d 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -66,7 +66,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } @@ -128,7 +127,6 @@ A common practice is to use a username or principal name in this path, i.e. use "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index ca3cfe5f595..c486609a81e 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -137,7 +137,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 5a6e7ccc536..785180c9321 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -66,7 +66,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 62f5afed12e..58dffee27ab 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -66,7 +66,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index d3da57f4460..0c8cde9c28d 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -14,7 +14,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 7cb49dff316..966e9ed3eeb 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -24,7 +24,6 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } @@ -91,7 +90,6 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 05f3f6bfcd6..83118570a99 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -34,7 +34,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } @@ -169,7 +168,6 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 40253afae5d..732c6347217 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -31,7 +31,6 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } @@ -88,7 +87,6 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index de9fe5e5a42..36a74b39b35 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -13,7 +13,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index ab5a9e03159..8f564cd52cd 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -58,7 +58,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index c6cdf2cbd5d..6c5c14c32b9 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -15,7 +15,6 @@ Files: 6 uploaded, 0 deleted { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index ced4af35ac2..66e725f92e7 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -40,10 +40,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "state": {} } -=== A saved plan carries the deployment id and the version it was built against +=== A saved plan carries the version it was built against >>> [CLI] bundle plan -o json { - "deployment_id": "[NUMID]", "serial": 1 } @@ -60,9 +59,8 @@ Error: this plan was built against version 0 but the deployment has recorded ver {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} === Save a plan against the recorded deployment, then move the deployment on ->>> jq {deployment_id, serial} tmp.plan2.json +>>> jq {serial} tmp.plan2.json { - "deployment_id": "[NUMID]", "serial": 1 } diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index cef55ecd6e4..2e894c12502 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -13,8 +13,8 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d title "The raw recorded state the service holds for this deployment" trace print_state.py --no-dms -title "A saved plan carries the deployment id and the version it was built against" -trace $CLI bundle plan -o json | jq '{deployment_id, serial}' +title "A saved plan carries the version it was built against" +trace $CLI bundle plan -o json | jq '{serial}' title "Replay: re-deploying the same plan is rejected - it predates the recorded version" trace musterr $CLI bundle deploy --plan tmp.plan.json @@ -27,7 +27,7 @@ trace print_requests.py --dms //api/2.0/bundle --oneline # version, which the serial guard has to catch. title "Save a plan against the recorded deployment, then move the deployment on" $CLI bundle plan -o json > tmp.plan2.json -trace jq '{deployment_id, serial}' tmp.plan2.json +trace jq '{serial}' tmp.plan2.json trace update_file.py databricks.yml "name: foo" "name: foo_renamed" trace $CLI bundle deploy trace jq '{serial}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index bef65faa709..959a63b468a 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -66,7 +66,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } @@ -170,7 +169,6 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 7d628b01d44..3436a9bf09a 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -15,7 +15,6 @@ Files: 5 uploaded, 0 deleted { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "deployment_id": "[NUMID]", "features": { "deployment_history": {} } diff --git a/acceptance/test.toml b/acceptance/test.toml index 68eb1f55c5a..a732c2db784 100644 --- a/acceptance/test.toml +++ b/acceptance/test.toml @@ -95,9 +95,9 @@ New = "[METASTORE_NAME]" # Normalize plan_version so bumping it doesn't churn every recorded plan. [[Repls]] -Old = '"plan_version": 3,' +Old = '"plan_version": 2,' New = '"plan_version": [PLAN_VERSION],' [[Repls]] -Old = 'json\.plan_version = 3;' +Old = 'json\.plan_version = 2;' New = 'json.plan_version = [PLAN_VERSION];' diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 73229fba952..67236f74766 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -13,7 +13,7 @@ import ( "github.com/databricks/cli/libs/structs/structvar" ) -const currentPlanVersion = 3 +const currentPlanVersion = 2 type Plan struct { PlanVersion int `json:"plan_version,omitempty"` @@ -21,12 +21,6 @@ type Plan struct { Lineage string `json:"lineage,omitempty"` Serial int `json:"serial,omitempty"` - // DeploymentId is the recorded deployment this plan targets, set only when the bundle records - // deployment history. The version this plan would create is derived from Serial rather than - // carried separately: a version is only created when the plan changes something, so the two - // advance together (see phases.Deploy). deploy --plan rejects a plan whose deployment moved on. - DeploymentId string `json:"deployment_id,omitempty"` - // Features are the state feature flags this plan was built against, mirroring the state // file's own "features" field. The stamps above exist because of a feature being set, so // deploy --plan rejects a plan whose features differ from the target's (see process.go). diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index d138605423e..ea0c83ad312 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -41,21 +41,14 @@ func (b *DeploymentBundle) init(client *databricks.WorkspaceClient) error { } // ValidatePlanAgainstState validates that a plan still matches the given state: the recorded -// deployment and version it targeted, then its lineage and serial. +// version it targeted, then its lineage and serial. func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan.Plan) error { - stateDB.AssertOpenedForReadOrWrite() - - if stateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { - // Stale if the deployment recorded a version past the one the plan was built against. - // Covers a plan from before the first deploy too: its version is 0. - if stateDB.LatestVersionID != "" { - last, err := strconv.Atoi(stateDB.LatestVersionID) - if err == nil && plan.Serial < last { - return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, last) - } - } - if plan.DeploymentId != stateDB.DeploymentID { - return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") + // Stale if the deployment recorded a version past the one the plan was built against. Covers a + // plan from before the first deploy too: its version is 0. Only recorded bundles set this. + if stateDB.LatestVersionID != "" { + last, err := strconv.Atoi(stateDB.LatestVersionID) + if err == nil && plan.Serial < last { + return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, last) } } @@ -63,6 +56,8 @@ func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan. return nil } + stateDB.AssertOpenedForReadOrWrite() + if plan.Lineage != stateDB.Data.Lineage { return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) } @@ -180,13 +175,10 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, fmt.Errorf("reading config: %w", err) } - // The plan records where its state lives so deploy --plan can reject a plan whose target has - // since switched backends or has an outdated version or a wrong deployment_id. + // The plan records the state features it was built against so deploy --plan can reject a plan + // built for a target of a different shape. if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { plan.Features = b.StateDB.StateFeatures() - if b.StateDB.DeploymentID != "" { - plan.DeploymentId = b.StateDB.DeploymentID - } } b.Plan = plan From 7750a32c9108d7e3af41797091c62009f650f2d3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 13:59:53 +0200 Subject: [PATCH 115/164] acceptance: revert readplan plan fixtures to plan version 2 Follow-on to keeping the plan version at 2. Three readplan tests carry a checked-in plan.json, and two of them had been bumped to version 3 along with the constant. Left at 3 they would fail the version check before reaching the lineage and cli-version mismatches they exist to assert. The third deliberately uses 999, so only its golden needed the new expected version. Co-authored-by: Isaac --- .../bundle/deploy/readplan/cli-version-mismatch/plan.json | 2 +- acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json | 2 +- .../bundle/deploy/readplan/plan-version-mismatch/output.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json b/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json index bd9561c9ec8..932edf74f84 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json @@ -1,5 +1,5 @@ { - "plan_version": 3, + "plan_version": 2, "cli_version": "0.0.0-test", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json index 80310520edb..42cce15cbfc 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json @@ -1,5 +1,5 @@ { - "plan_version": 3, + "plan_version": 2, "cli_version": "0.0.0-test", "lineage": "different-lineage", "serial": 0, diff --git a/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt b/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt index 3ec4497a23e..387aac2335c 100644 --- a/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt +++ b/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt @@ -1,4 +1,4 @@ -Error: plan version mismatch: plan has version 999 (generated with CLI "0.0.0"), but current version is 3 +Error: plan version mismatch: plan has version 999 (generated with CLI "0.0.0"), but current version is 2 Exit code: 1 From cf4f400156e9c53942e3214a59bba98b8488a3d6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 14:36:29 +0200 Subject: [PATCH 116/164] bundle/direct/dstate: take the serial from the deployment's last_version_id Under recording the service owns the version number and the serial only tracks it, so there is no reason to keep both. Open now sets Data.Serial from the deployment's last_version_id, and the state file stops persisting a serial of its own: for a recorded deployment it is written as 0 and omitted. With one source of truth, three things simplify: - StateDB.LatestVersionID is gone. Its stale-plan check reads Data.Serial, and CompleteVersion takes the version from the operation buffer, which knows the version it was opened for. That is also more correct than re-deriving it: the serial moves during a deploy, the created version does not. - previousVersionID is the serial itself, rather than the next version minus one (which had just been the serial plus one). Addresses review feedback. - Open takes a DMSDeployment instead of a bare id, so the deployment id and its last version travel together. Two cases the serial has to be taken from the service rather than the file: - no deployment yet means no versions, hence 0. A state file written before recording was turned on counts a history the service knows nothing about, and letting that serial through sent a stale previous_version_id on the first recorded deploy (409 ABORTED, caught by bundle/dms/existing-state). - a failed deploy creates a version but never writes state, so the serial is left one behind (bundle/dms/record-failure). Adopting the service's version is what keeps the next one in sequence. There is a TODO to report drift via telemetry, separating that expected failed-deploy case from a serial ahead of the service or behind by more than one. Co-authored-by: Isaac --- .../bundle/dms/declined-deploy/output.txt | 1 - acceptance/bundle/dms/depends-on/output.txt | 1 - acceptance/bundle/dms/empty-grants/output.txt | 1 - .../bundle/dms/failed-delete/output.txt | 2 - .../bundle/dms/failed-recreate/output.txt | 2 - .../dms/failed-update-permissions/output.txt | 2 - .../bundle/dms/multiple-resources/output.txt | 2 - acceptance/bundle/dms/no-drift/output.txt | 2 - .../operation-upload-fails-once/output.txt | 2 - acceptance/bundle/dms/provenance/output.txt | 1 - .../bundle/dms/record-failure/output.txt | 1 + acceptance/bundle/dms/stale-plan/output.txt | 3 +- .../bundle/dms/successful-recreate/output.txt | 2 - bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 ++-- bundle/direct/bundle_plan.go | 30 -------- bundle/direct/dstate/state.go | 72 +++++++++++++------ bundle/direct/dstate/state_test.go | 36 +++++----- bundle/phases/dms.go | 8 +-- bundle/statemgmt/direct_migration.go | 2 +- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 38 +++++++--- libs/dms/operation_buffer.go | 6 ++ 25 files changed, 121 insertions(+), 113 deletions(-) diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index c99c5cfdb15..870af8c8858 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -97,7 +97,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index cb17c7fe406..4237ad47e7b 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -89,7 +89,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index c486609a81e..81164aaa235 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -161,7 +161,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 785180c9321..6d580163858 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -90,7 +90,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, @@ -182,7 +181,6 @@ API message: Fault injected by test. "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 58dffee27ab..c487df25acc 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -90,7 +90,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, @@ -177,7 +176,6 @@ Files: 4 uploaded, 0 deleted "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 0c8cde9c28d..58e558971bf 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -43,7 +43,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, @@ -98,7 +97,6 @@ Files: 3 uploaded, 0 deleted "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 966e9ed3eeb..ec51a77d2c3 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -68,7 +68,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, @@ -134,7 +133,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 83118570a99..a8e14bab386 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -48,7 +48,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, @@ -182,7 +181,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 36a74b39b35..f930196be62 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -37,7 +37,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, @@ -79,7 +78,6 @@ Files: 4 uploaded, 0 deleted "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 8f564cd52cd..03ff7c406c2 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -98,7 +98,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 6c5c14c32b9..b3c53d7508d 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -15,6 +15,7 @@ Files: 6 uploaded, 0 deleted { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", + "serial": 1, "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 66e725f92e7..30eea85623e 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -33,7 +33,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, @@ -74,7 +73,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> jq {serial} .databricks/bundle/default/resources.json { - "serial": 2 + "serial": null } === Replaying the older plan is rejected: same deployment, outdated version diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 959a63b468a..022757cde66 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -90,7 +90,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "features": { "deployment_history": {} }, @@ -193,7 +192,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, "features": { "deployment_history": {} }, diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index f5afb8d5b1a..3bf8e08fcb7 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 462c0b4c71f..aab709ab4b8 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -156,7 +156,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index ae5c85e9a9d..3a2ce2f54e6 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -65,7 +65,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac // The state is opened without a DMS client, so the writes below record nothing; // phases.Bind and phases.Unbind refuse to run at all when recording is enabled. var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -89,7 +89,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -112,7 +112,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), "") + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -148,7 +148,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -168,7 +168,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), "") + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -218,7 +218,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}) if err != nil { if errors.Is(err, dstate.ErrUnsettingRecording) { return errors.New("unbind is not supported for a bundle target that records deployment history") diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index ea0c83ad312..289cb1db500 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -8,7 +8,6 @@ import ( "maps" "reflect" "slices" - "strconv" "strings" "github.com/databricks/cli/bundle/config" @@ -40,35 +39,6 @@ func (b *DeploymentBundle) init(client *databricks.WorkspaceClient) error { return err } -// ValidatePlanAgainstState validates that a plan still matches the given state: the recorded -// version it targeted, then its lineage and serial. -func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan.Plan) error { - // Stale if the deployment recorded a version past the one the plan was built against. Covers a - // plan from before the first deploy too: its version is 0. Only recorded bundles set this. - if stateDB.LatestVersionID != "" { - last, err := strconv.Atoi(stateDB.LatestVersionID) - if err == nil && plan.Serial < last { - return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, last) - } - } - - if plan.Lineage == "" { - return nil - } - - stateDB.AssertOpenedForReadOrWrite() - - if plan.Lineage != stateDB.Data.Lineage { - return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) - } - - if plan.Serial != stateDB.Data.Serial { - return fmt.Errorf("plan serial %d does not match state serial %d; the state has been modified since the plan was created. Please run 'bundle plan' again", plan.Serial, stateDB.Data.Serial) - } - - return nil -} - // InitForApply initializes the DeploymentBundle for applying a pre-computed plan. // StateDB must already be open for write before calling this function. func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ac37787ffe7..55348b50c24 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "sync" @@ -139,13 +140,22 @@ type DeploymentState struct { // completion after an explicit one does nothing. versionCompleted bool - // DeploymentID and LatestVersionID are the recorded deployment's id and its most recent version, - // from the service at Open (InitializeOperationBuffer updates DeploymentID to the id a first - // deploy creates, which Open cannot know). CalculatePlan reads them to stamp the plan's lineage; - // CompleteVersion reads them after Finalize's reset - so, like operationBuffer and dmsClient, they - // must survive reset. - DeploymentID string - LatestVersionID string + // DeploymentID is the recorded deployment's id, from the service at Open + // (InitializeOperationBuffer updates it to the id a first deploy creates, which Open cannot + // know). CalculatePlan reads it to stamp the plan; CompleteVersion reads it after Finalize's + // reset - so, like operationBuffer and dmsClient, it must survive reset. + DeploymentID string +} + +// DMSDeployment identifies the recorded deployment Open reads from. The zero value means the +// bundle does not record deployment history, or no deployment exists for it yet. +type DMSDeployment struct { + // ID is the deployment's server-minted id. + ID string + + // LastVersionID is the most recent version the service has recorded. The state serial is set + // from it at Open: the service owns the version number, and the serial only tracks it. + LastVersionID string } type Header struct { @@ -157,7 +167,10 @@ type Header struct { CLIVersion string `json:"cli_version"` Lineage string `json:"lineage"` - Serial int `json:"serial"` + + // Serial counts state writes. Omitted for recorded deployments, where the service's version is + // the source of truth and Open sets this from it. + Serial int `json:"serial,omitempty"` // Features maps each feature flag this state depends on to a (currently empty) // value. It is read to detect a state that depends on features this CLI lacks and @@ -232,14 +245,11 @@ func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (b return false, nil } db.versionCompleted = true - deploymentID, latestVersionID, client := db.DeploymentID, db.LatestVersionID, db.dmsClient + // The buffer knows the version it was opened for, so complete exactly that one rather than + // re-deriving it: the serial moves during a deploy, the created version does not. + deploymentID, client, versionID := db.DeploymentID, db.dmsClient, buf.Version() db.mu.Unlock() - versionID, err := dms.NextVersion(latestVersionID) - if err != nil { - return false, err - } - // A recording failure fails the version even when the caller counted the deploy a success: the // service does not then hold everything the WAL does. Finalize already drained and surfaced it; // this reads the drained buffer's error so a destroy whose uploads failed keeps its record. @@ -463,7 +473,7 @@ type ( // recorded deploy); lineage and serial still come from the file, since that is what the write path // increments. Open only reads through the client - InitializeOperationBuffer installs the write path once a // version exists. -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeployment DMSDeployment) error { db.mu.Lock() defer db.mu.Unlock() @@ -471,7 +481,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W panic(fmt.Sprintf("state already opened: %v, cannot open %v", db.Path, path)) } - err := db.unlockedOpen(ctx, path, withRecovery, withWrite, withDeploymentHistory, dmsDeploymentID) + err := db.unlockedOpen(ctx, path, withRecovery, withWrite, withDeploymentHistory, dmsDeployment) if err != nil { // A failed open must leave the receiver closed. unlockedOpen assigns // db.Path before every fallible step, so without this the receiver stays @@ -493,7 +503,7 @@ func (db *DeploymentState) reset() { db.stateIDs = nil } -func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { +func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeployment DMSDeployment) error { db.Path = path // The state file is the source of truth for whether this deployment records history: read it @@ -586,10 +596,10 @@ To record this bundle's history, start it over as a new deployment: return err } db.dmsClient = client - db.DeploymentID = dmsDeploymentID + db.DeploymentID = dmsDeployment.ID - if dmsDeploymentID != "" { - resources, err := db.dmsClient.ListResources(ctx, dmsDeploymentID) + if dmsDeployment.ID != "" { + resources, err := db.dmsClient.ListResources(ctx, dmsDeployment.ID) if err != nil { return err } @@ -597,6 +607,25 @@ To record this bundle's history, start it over as a new deployment: return err } } + + // The service owns the version number, so the serial comes from it rather than from the + // state file, which no longer persists one. With no deployment yet there are no versions, + // hence zero - which matters for a state file written before recording was turned on, whose + // serial counts a history the service knows nothing about. + // + // Serial and version normally advance together, with one expected exception: a failed deploy + // creates a version but never writes state (see acceptance/bundle/dms/record-failure). + // + // TODO: report drift via telemetry, separating that failed-deploy case from a serial ahead + // of the service or behind by more than one. Those should not happen in normal operation. + serial := 0 + if dmsDeployment.LastVersionID != "" { + serial, err = strconv.Atoi(dmsDeployment.LastVersionID) + if err != nil { + return fmt.Errorf("failed to parse last_version_id %q: %w", dmsDeployment.LastVersionID, err) + } + } + db.Data.Serial = serial } if withWrite { @@ -953,6 +982,9 @@ func (db *DeploymentState) dataForFile() Database { header.Features = make(map[string]struct{}) } header.Features[FeatureDeploymentHistory] = struct{}{} + // Under DMS the serial is managed by the deployment's last_version_id, so it is not + // persisted here; Open sets it from the service. + header.Serial = 0 return Database{ Header: header, diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 57ccd4d3e51..bea61811634 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -21,14 +21,14 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) @@ -38,7 +38,7 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) mustFinalize(t, &db) _, err := os.Stat(path) @@ -94,10 +94,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "") + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{}) }) mustFinalize(t, &db) } @@ -115,12 +115,12 @@ func TestCLIVersionRecordsLastWriter(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) mustFinalize(t, &db) var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) assert.Equal(t, build.GetInfo().Version, reopened.Data.CLIVersion) assert.Equal(t, 2, reopened.Data.Serial) mustFinalize(t, &reopened) @@ -142,7 +142,7 @@ func TestHeaderOnlyWALDoesNotUpdateCLIVersion(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), "")) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) assert.Equal(t, "0.1.2", recovered.Data.CLIVersion, "a header-only WAL wrote no state, so the version must not move") assert.Equal(t, 1, recovered.Data.Serial) mustFinalize(t, &recovered) @@ -154,12 +154,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -175,7 +175,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), "")) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -218,17 +218,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", false)) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -240,7 +240,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -257,7 +257,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } @@ -271,13 +271,13 @@ func TestOpenFailureLeavesStateClosed(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte("{not json"), 0o600)) var db DeploymentState - require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) + require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) assert.Empty(t, db.Path) // Once the state file is readable, the same receiver opens without panicking. seed := `{"state_version":2,"cli_version":"0.1.2","lineage":"test-lineage","serial":1,"state":{}}` require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) assert.Equal(t, "test-lineage", db.Data.Lineage) mustFinalize(t, &db) } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index fa9a0f65a14..c016b543a23 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -123,11 +123,11 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version } deploymentID, versionID := deploymentAndNextVersion(b) - // The version this run creates is one past the deployment's most recent, so the previous is - // one below it - empty for the first version. + // The state serial tracks the version the service last recorded, which is the one this run + // follows. Empty for the first version. previousVersionID := "" - if versionID > 1 { - previousVersionID = strconv.FormatInt(versionID-1, 10) + if serial := db.StateDB.Data.Serial; serial > 0 { + previousVersionID = strconv.Itoa(serial) } // The server rejects this unless the version number exceeds last_version_id and diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index f2140823b28..ecaddc34edd 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -173,7 +173,7 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s var planBundle direct.DeploymentBundle // This plan check does not record deployment history, so no DMS client or deployment id. - if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, dstate.WithDeploymentHistory(false), ""); err != nil { + if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { return fmt.Errorf("opening migrated state for plan check: %w", err) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 6047655b412..d2667a19dd1 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index f87529b869a..2df4efc22b1 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index d62ff246bbc..981077f3b77 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -18,7 +18,6 @@ import ( "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/statemgmt" @@ -289,15 +288,12 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dmsDeploymentID); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.DMSDeployment{ID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } - // Open sets DeploymentID from the id passed to it; the version is not passed, so - // set it here. CalculatePlan reads these, not the config tree. - b.DeploymentBundle.StateDB.LatestVersionID = lastVersionID } else { - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } @@ -468,17 +464,39 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // validatePlan rejects a --plan file that no longer matches the target: its recording shape must // match the target's config, and a plan that targets an existing recorded deployment must not // predate the deployment the service now holds. It ends with the local lineage/serial guard. +// validatePlan rejects a saved plan that no longer matches the state it was built against. func validatePlan(b *bundle.Bundle, plan *deployplan.Plan) error { + stateDB := &b.DeploymentBundle.StateDB + stateDB.AssertOpenedForReadOrWrite() + // A plan is built against a set of state features, and the stamps it carries follow from // them, so applying it to a target with a different set would deploy the wrong shape. // The state is the target's source of truth and carries its features even on a first // recorded deploy (unlike the version ids, which are empty then). - if !maps.Equal(plan.Features, b.DeploymentBundle.StateDB.StateFeatures()) { + if !maps.Equal(plan.Features, stateDB.StateFeatures()) { return errors.New("this plan was created for a different set of state features than the target now has; run 'bundle plan' again") } - // The rest of the plan-vs-state checks (recorded deployment and version, then lineage and - // serial) live together next to the state. - return direct.ValidatePlanAgainstState(&b.DeploymentBundle.StateDB, plan) + + // Under recording the serial tracks the version the service has recorded, so a plan built + // against an earlier one is stale. Checked ahead of the lineage guard because a plan from + // before the first deploy has no lineage and would otherwise exit through it. + if stateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService && plan.Serial < stateDB.Data.Serial { + return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, stateDB.Data.Serial) + } + + if plan.Lineage == "" { + return nil + } + + if plan.Lineage != stateDB.Data.Lineage { + return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) + } + + if plan.Serial != stateDB.Data.Serial { + return fmt.Errorf("plan serial %d does not match state serial %d; the state has been modified since the plan was created. Please run 'bundle plan' again", plan.Serial, stateDB.Data.Serial) + } + + return nil } // ResolveEngineSetting determines the effective engine setting by combining bundle config and env var. diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go index 0718aac330c..6a00da3c7a9 100644 --- a/libs/dms/operation_buffer.go +++ b/libs/dms/operation_buffer.go @@ -49,6 +49,12 @@ type OperationBuffer struct { err error } +// Version reports the deployment version this buffer records operations for. Set once when the +// buffer opens, so it is the version that was actually created rather than one re-derived later. +func (b *OperationBuffer) Version() int64 { + return b.versionNum +} + // StartOperationBuffer opens the buffer for the version the caller just created. The version // must already exist: operations record under it, and nothing here creates it. func StartOperationBuffer(ctx context.Context, client *Client, deploymentID string, versionNum int64) *OperationBuffer { From 16ef4b279677bcf2a7233dbd89f6f70e0a44ed3b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 14:55:02 +0200 Subject: [PATCH 117/164] cmd/bundle/utils: validate lineage and serial for a plan that has neither A plan taken before the first deploy carries no lineage, and validatePlan returned early on it rather than comparing anything. That skipped both checks for exactly the plan most likely to be stale: replayed after a deployment had happened, it was accepted, because the guard fired before the comparison could catch it. Both sides are empty on a genuine first deploy, so the comparisons pass on their own and the early return bought nothing. Dropping it closes the hole for bundles that do not record history; recorded ones were already covered by the version check above. bundle/deploy/readplan/whl-not-uploaded stops asserting the plan's serial: it destroys before planning, and destroy removes the recorded deployment, so under recording there is no version left to take a serial from while the local state keeps counting. The test is about a wheel that never gets uploaded, not about serials. Co-authored-by: Isaac --- .../bundle/deploy/readplan/whl-not-uploaded/out.plan.json | 1 - acceptance/bundle/deploy/readplan/whl-not-uploaded/script | 4 +++- cmd/bundle/utils/process.go | 6 ++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json index 2a4987e84b4..cc6ec22302d 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json @@ -2,7 +2,6 @@ "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, "plan": { "resources.jobs.my_job": { "action": "create", diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script index 042936bede3..753216001cc 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script @@ -16,7 +16,9 @@ rm out.requests.txt title "bundle deploy --plan does NOT upload the wheel (bug)" trace $CLI bundle plan -o json > tmp.plan.json -nostamp < tmp.plan.json > out.plan.json +# The serial differs between the DMS variants here: destroy removes the recorded deployment, so +# under recording there is no version left to take it from, while the local state keeps counting. +nostamp < tmp.plan.json | jq 'del(.serial)' > out.plan.json trace $CLI bundle deploy --plan tmp.plan.json # BUG: no import-file request for the wheel here (output below is empty), yet the job was # created pointing the task at the same .../artifacts/.internal/ path that was never uploaded. diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 981077f3b77..9bcf6d18fd6 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -484,10 +484,8 @@ func validatePlan(b *bundle.Bundle, plan *deployplan.Plan) error { return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, stateDB.Data.Serial) } - if plan.Lineage == "" { - return nil - } - + // A plan taken before the first deploy carries no lineage or serial, so both sides are empty + // then and the checks below pass. If a deployment has happened since, they catch it. if plan.Lineage != stateDB.Data.Lineage { return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) } From bc0a59861c29fd83969324c927692fe22d8d1d8c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 15:00:53 +0200 Subject: [PATCH 118/164] bundle/direct/dstate: keep the created version on the state, not behind an accessor InitializeOperationBuffer already receives the version and already mirrors the deployment id onto the state, so the version can sit next to it instead of adding an exported accessor to OperationBuffer for one internal read. reset() leaves both alone, so it survives Finalize the same way. Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 9 ++++++--- libs/dms/operation_buffer.go | 6 ------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 55348b50c24..edf19491028 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -145,6 +145,10 @@ type DeploymentState struct { // know). CalculatePlan reads it to stamp the plan; CompleteVersion reads it after Finalize's // reset - so, like operationBuffer and dmsClient, it must survive reset. DeploymentID string + + // versionID is the version InitializeOperationBuffer created, kept so CompleteVersion completes + // that exact one: the serial moves during a deploy, the created version does not. + versionID int64 } // DMSDeployment identifies the recorded deployment Open reads from. The zero value means the @@ -211,6 +215,7 @@ func (db *DeploymentState) InitializeOperationBuffer(ctx context.Context, deploy defer db.mu.Unlock() db.operationBuffer = buf db.DeploymentID = deploymentID + db.versionID = versionID } // RecordingError reports whether recording state writes to the service has failed, so the apply @@ -245,9 +250,7 @@ func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (b return false, nil } db.versionCompleted = true - // The buffer knows the version it was opened for, so complete exactly that one rather than - // re-deriving it: the serial moves during a deploy, the created version does not. - deploymentID, client, versionID := db.DeploymentID, db.dmsClient, buf.Version() + deploymentID, client, versionID := db.DeploymentID, db.dmsClient, db.versionID db.mu.Unlock() // A recording failure fails the version even when the caller counted the deploy a success: the diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go index 6a00da3c7a9..0718aac330c 100644 --- a/libs/dms/operation_buffer.go +++ b/libs/dms/operation_buffer.go @@ -49,12 +49,6 @@ type OperationBuffer struct { err error } -// Version reports the deployment version this buffer records operations for. Set once when the -// buffer opens, so it is the version that was actually created rather than one re-derived later. -func (b *OperationBuffer) Version() int64 { - return b.versionNum -} - // StartOperationBuffer opens the buffer for the version the caller just created. The version // must already exist: operations record under it, and nothing here creates it. func StartOperationBuffer(ctx context.Context, client *Client, deploymentID string, versionNum int64) *OperationBuffer { From 7033b09d66765b19563d5eee565ff4e825cc528f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 15:04:22 +0200 Subject: [PATCH 119/164] bundle/phases: compute the update mask only where it is used The switch initializer diffed the metadata for both branches, but the create path does not use a mask - and has no record to diff against, since current is nil before the first recorded deploy. Scope it to the update branch. Addresses review feedback. Co-authored-by: Isaac --- bundle/phases/dms.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index c016b543a23..ca67ee36271 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -88,15 +88,14 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu dmsClient := db.StateDB.DmsClient() metadata := deploymentMetadata(b) deploymentID, _ := deploymentAndNextVersion(b) - switch mask := metadata.StaleFields(current); { - case deploymentID == "": + if deploymentID == "" { id, err := dmsClient.CreateDeployment(ctx, b.Config.Workspace.StatePath, metadata) if err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to create deployment: %w", err)) return } deploymentID = id - case mask != "": + } else if mask := metadata.StaleFields(current); mask != "" { if err := dmsClient.UpdateDeployment(ctx, deploymentID, metadata, mask); err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to update deployment: %w", err)) return From 3aee9af19c73f1d0ffbbfc436c8d0b6a55c00868 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 15:06:59 +0200 Subject: [PATCH 120/164] cmd/bundle/utils: note the state-path deployment lookup could be one API fetchDeploymentFromStatePath makes two round trips because there is no way to get a deployment by state path. Record the follow-up as a TODO. Addresses review feedback. Co-authored-by: Isaac --- cmd/bundle/utils/process.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 9bcf6d18fd6..57fe6a86845 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -524,6 +524,10 @@ func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineS } // Lookup and return the deployment object from ${workspace.state_path}/resources.deployment.json +// +// TODO: ask the service for a new API that looks a deployment up by state path directly +// (GetDeployment(state_path)). That would replace the two round trips here: a workspace lookup to +// turn the node into an id, then a get by that id. func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { nodePath := path.Join(statePath, dms.DeploymentNodeName) From 2c91f6b700d753dae6ab60ca117d430153fc991a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 15:14:17 +0200 Subject: [PATCH 121/164] better comment --- bundle/statemgmt/direct_migration.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index ecaddc34edd..76c52df5458 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -173,6 +173,9 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s var planBundle direct.DeploymentBundle // This plan check does not record deployment history, so no DMS client or deployment id. + + // This plan is not created with the deployment history feature enabled, + // so we can safely pass false for withDeploymentHistory. if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { return fmt.Errorf("opening migrated state for plan check: %w", err) } From 3767ff3bcca97fc632822e026025af631a2f49d2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 15:14:40 +0200 Subject: [PATCH 122/164] nit --- bundle/statemgmt/direct_migration.go | 1 - 1 file changed, 1 deletion(-) diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 76c52df5458..fe8f55c4e35 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -172,7 +172,6 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s }() var planBundle direct.DeploymentBundle - // This plan check does not record deployment history, so no DMS client or deployment id. // This plan is not created with the deployment history feature enabled, // so we can safely pass false for withDeploymentHistory. From c3b8e2767ee0ebad9ffff26dc97758e73cd977b2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 15:25:01 +0200 Subject: [PATCH 123/164] bundle/direct/dstate: write no WAL for a recorded deployment Every state write already goes to the service as it happens, and Open recovers from there rather than replaying a log - it deletes any WAL it finds. So the WAL under recording was a file we wrote, replayed and threw away, while resources.json was written empty either way. SaveState and DeleteState now keep only the id index and let the service call carry the state. Write mode was implied by walFile being set. That is no longer the same question, so it becomes an explicit openedForWrite flag, set at the two points that used to create the WAL: Open, and UpgradeToWrite, which is how a deploy actually enters write mode. The non-recording paths are unchanged. Two things the WAL replay used to carry come along with it. The header - lineage, state version, CLI version - is stamped when write mode opens. And the state file, which only the replay wrote, is written at Finalize instead; a recorded deployment gets its tombstone even when the run recorded nothing, since the deployment exists either way. Addresses review feedback. Co-authored-by: Isaac --- acceptance/bundle/dms/no-resources/output.txt | 20 +++++ .../bundle/dms/record-failure/output.txt | 10 +++ bundle/direct/dstate/state.go | 81 +++++++++++++------ 3 files changed, 87 insertions(+), 24 deletions(-) diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 732c6347217..1c7a01d548a 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -31,6 +31,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", "features": { "deployment_history": {} } @@ -45,6 +46,15 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "features": { + "deployment_history": {} + }, + "state": {} +} === Redeploy: the deployment is resolved from that node, so no second deployment is created >>> [CLI] bundle deploy @@ -87,6 +97,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", "features": { "deployment_history": {} } @@ -101,3 +112,12 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged === The raw recorded state after redeploy >>> print_state.py --no-dms +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "features": { + "deployment_history": {} + }, + "state": {} +} diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index b3c53d7508d..fbcd3135fff 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -15,6 +15,7 @@ Files: 6 uploaded, 0 deleted { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", "serial": 1, "features": { "deployment_history": {} @@ -85,6 +86,15 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged [] >>> print_state.py --no-dms +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "features": { + "deployment_history": {} + }, + "state": {} +} === Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged >>> [CLI] bundle plan diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index edf19491028..9680cf03f66 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -121,6 +121,10 @@ type DeploymentState struct { mu sync.Mutex walFile *os.File + // openedForWrite records write mode explicitly. It used to be implied by walFile, but a recorded + // deployment writes no WAL, so the two are no longer the same question. + openedForWrite bool + // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string @@ -332,23 +336,23 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta DependsOn: dependsOn, } - err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) - if err == nil { + // A recorded deployment persists through the service, everything else through the WAL. + if db.StorageBackend() == StorageBackendDeploymentMetadataService { db.stateIDs[key] = newID - } - if err != nil { - return err + if buf := db.operationBuffer; buf != nil { + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + if err != nil { + return err + } + buf.RecordOperation(ctx, key, false, newID, recorded) + } + return nil } - // Record operation in DMS. - buf := db.operationBuffer - if db.StorageBackend() == StorageBackendDeploymentMetadataService && buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) - if err != nil { - return err - } - buf.RecordOperation(ctx, key, false, newID, recorded) + if err := appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}); err != nil { + return err } + db.stateIDs[key] = newID return nil } @@ -365,18 +369,18 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgre } // Read before the delete: DMS needs the id to say which resource went away. deletedID := db.stateIDs[key] - err := appendJSONLine(db.walFile, WALEntry{Key: key}) - if err == nil { + if db.StorageBackend() == StorageBackendDeploymentMetadataService { delete(db.stateIDs, key) - } - if err != nil { - return err + if buf := db.operationBuffer; buf != nil { + buf.RecordOperation(ctx, key, inProgress, deletedID, nil) + } + return nil } - buf := db.operationBuffer - if db.StorageBackend() == StorageBackendDeploymentMetadataService && buf != nil { - buf.RecordOperation(ctx, key, inProgress, deletedID, nil) + if err := appendJSONLine(db.walFile, WALEntry{Key: key}); err != nil { + return err } + delete(db.stateIDs, key) return nil } @@ -504,6 +508,7 @@ func (db *DeploymentState) reset() { db.Path = "" db.Data = Database{} db.stateIDs = nil + db.openedForWrite = false } func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeployment DMSDeployment) error { @@ -632,6 +637,19 @@ To record this bundle's history, start it over as a new deployment: } if withWrite { + db.openedForWrite = true + + // A recorded deployment needs no WAL: every state write goes to the service as it happens, + // and Open recovers from there rather than replaying a log. Stamp the header fields the + // replay would have carried over, then skip the file. The serial is left alone: under + // recording it comes from the deployment's last_version_id. + if db.storageBackend == StorageBackendDeploymentMetadataService { + db.Data.Lineage = db.GetOrInitLineage() + db.Data.StateVersion = currentStateVersion + db.Data.CLIVersion = build.GetInfo().Version + return nil + } + if err := os.MkdirAll(filepath.Dir(walPath), 0o755); err != nil { return fmt.Errorf("failed to create state directory: %w", err) } @@ -812,6 +830,12 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported } db.walFile = nil err = db.replayWAL(ctx) + } else if db.openedForWrite && db.storageBackend == StorageBackendDeploymentMetadataService { + // replayWAL is what normally persists the file. Without one, write the tombstone here so + // the header (lineage, state version, CLI version) still lands on disk - even when no + // operations were recorded, since the deployment and any version this run created exist + // either way. + err = db.unlockedSave() } // Wait until all operations are recorded in the service. @@ -837,9 +861,18 @@ func (db *DeploymentState) UpgradeToWrite() error { if db.Path == "" { return errors.New("internal error: DeploymentState must be opened first") } - if db.walFile != nil { + if db.openedForWrite { return errors.New("internal error: DeploymentState is already open for write") } + db.openedForWrite = true + + // As in Open: a recorded deployment writes no WAL, so stamp the header and skip the file. + if db.storageBackend == StorageBackendDeploymentMetadataService { + db.Data.Lineage = db.GetOrInitLineage() + db.Data.StateVersion = currentStateVersion + db.Data.CLIVersion = build.GetInfo().Version + return nil + } walPath := db.Path + walSuffix if err := os.MkdirAll(filepath.Dir(walPath), 0o755); err != nil { @@ -868,14 +901,14 @@ func (db *DeploymentState) AssertOpenedForReadOrWrite() { func (db *DeploymentState) AssertOpenedForRead() { db.AssertOpenedForReadOrWrite() - if db.walFile != nil { + if db.openedForWrite { panic("internal error: DeploymentState must be opened in read mode") } } func (db *DeploymentState) AssertOpenedForWrite() { db.AssertOpenedForReadOrWrite() - if db.walFile == nil { + if !db.openedForWrite { panic("internal error: DeploymentState must be opened in write mode") } } From c3bd4eda63bcdcc10d5735aa73654f79256e28fe Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 16:05:26 +0200 Subject: [PATCH 124/164] bundle/direct/dstate: tidy up the recorded write path Keep the WAL append and its stateIDs update in the shape they had before the split. Drop the drift note: with no serial persisted in the state file, it is always taken from last_version_id, so there is nothing to drift. And state the real reason versionID is kept - CompleteVersion runs after Finalize resets Data. Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 9680cf03f66..2272dce3c7c 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -150,8 +150,8 @@ type DeploymentState struct { // reset - so, like operationBuffer and dmsClient, it must survive reset. DeploymentID string - // versionID is the version InitializeOperationBuffer created, kept so CompleteVersion completes - // that exact one: the serial moves during a deploy, the created version does not. + // versionID is the version InitializeOperationBuffer created. CompleteVersion runs after + // Finalize, which resets Data, so the serial is no longer there to derive it from. versionID int64 } @@ -349,11 +349,11 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta return nil } - if err := appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}); err != nil { - return err + err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) + if err == nil { + db.stateIDs[key] = newID } - db.stateIDs[key] = newID - return nil + return err } // DeleteState drops the resource's state entry: the resource is gone. inProgress records the @@ -620,12 +620,6 @@ To record this bundle's history, start it over as a new deployment: // state file, which no longer persists one. With no deployment yet there are no versions, // hence zero - which matters for a state file written before recording was turned on, whose // serial counts a history the service knows nothing about. - // - // Serial and version normally advance together, with one expected exception: a failed deploy - // creates a version but never writes state (see acceptance/bundle/dms/record-failure). - // - // TODO: report drift via telemetry, separating that failed-deploy case from a serial ahead - // of the service or behind by more than one. Those should not happen in normal operation. serial := 0 if dmsDeployment.LastVersionID != "" { serial, err = strconv.Atoi(dmsDeployment.LastVersionID) From 17ed80e18af1aebc7b989ff1a49114fa4f001aee Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 16:27:09 +0200 Subject: [PATCH 125/164] bundle/phases: read the deployment from the state, drop deploymentAndNextVersion The helper pulled both the deployment id and the next version out of the config tree. The state already holds the id, and the serial already tracks the recorded version, so callers read those directly: the version this run creates is the serial plus one, exactly as a WAL header records it for a non-recorded deployment. Open sets the id, so the state is the one place to read it from. A first deploy is the gap - its id does not exist until the deployment is created - so createOrUpdateDeployment publishes it there via SetDeploymentID. The history keeps its copy as output only; nothing reads it now. Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 8 ++++++++ bundle/phases/deploy.go | 7 ++++--- bundle/phases/destroy.go | 4 ++-- bundle/phases/dms.go | 22 ++++------------------ 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 2272dce3c7c..d78a4dac1e2 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -222,6 +222,14 @@ func (db *DeploymentState) InitializeOperationBuffer(ctx context.Context, deploy db.versionID = versionID } +// SetDeploymentID publishes the id of a deployment created after Open, which could not know it. +// Readers take the deployment from here rather than from the config tree. +func (db *DeploymentState) SetDeploymentID(id string) { + db.mu.Lock() + defer db.mu.Unlock() + db.DeploymentID = id +} + // RecordingError reports whether recording state writes to the service has failed, so the apply // stops touching resources once the service is no longer keeping up. Nil when the bundle does not // record deployment history or recording is healthy. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 5562c62ad2e..263c2b7a42e 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -331,13 +331,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // then - so stamp the one just created into the plan the apply reads. // IsDirect first: StorageBackend asserts the state is open, and only the direct engine opens it. if stateEngine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { - existingID, _ := deploymentAndNextVersion(b) + existingID := b.DeploymentBundle.StateDB.DeploymentID createOrUpdateDeployment(ctx, b, dmsDeployment) if logdiag.HasError(ctx) { return } if existingID == "" { - deploymentID, _ := deploymentAndNextVersion(b) + deploymentID := b.DeploymentBundle.StateDB.DeploymentID if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(deploymentID); err != nil { logdiag.LogError(ctx, err) return @@ -356,7 +356,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand logdiag.LogError(ctx, err) return } - deploymentID, versionID := deploymentAndNextVersion(b) + deploymentID := b.DeploymentBundle.StateDB.DeploymentID + versionID := int64(b.DeploymentBundle.StateDB.Data.Serial) + 1 logDeploymentVersion(ctx, b, deploymentID, versionID) } } diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 69523ee7579..16bfeb24ea2 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -169,7 +169,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e } // A completed destroy's resources are gone, so its deployment record is deleted too. if completed { - deploymentID, _ := deploymentAndNextVersion(b) + deploymentID := b.DeploymentBundle.StateDB.DeploymentID if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) return @@ -224,7 +224,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { if err != nil { logdiag.LogError(ctx, err) } else if completed { - deploymentID, _ := deploymentAndNextVersion(b) + deploymentID := b.DeploymentBundle.StateDB.DeploymentID if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index ca67ee36271..3b9bc1d5b8f 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -63,22 +63,6 @@ func actionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType } } -// deploymentAndNextVersion reads, from the history the recording set, the deployment id and the version -// this run will create (one past the deployment's most recent). Both zero when nothing is recorded -// - the bundle records no history, so version 0 marks "no version" for logDeploymentVersion. -func deploymentAndNextVersion(b *bundle.Bundle) (string, int64) { - h := b.Config.Bundle.Deployment.History - if h == nil { - return "", 0 - } - // LatestVersionID comes from the service as an integer, so this does not fail in practice. - version, err := dms.NextVersion(h.LatestVersionID) - if err != nil { - version = 1 - } - return h.DeploymentID, version -} - // createOrUpdateDeployment creates the deployment on a first deploy, or updates the metadata this // run changed. current is the record the service holds (nil before the first recorded deploy), // diffed to mask the update down to what changed. Runs after approval, so a declined deploy leaves @@ -87,7 +71,7 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu db := &b.DeploymentBundle dmsClient := db.StateDB.DmsClient() metadata := deploymentMetadata(b) - deploymentID, _ := deploymentAndNextVersion(b) + deploymentID := db.StateDB.DeploymentID if deploymentID == "" { id, err := dmsClient.CreateDeployment(ctx, b.Config.Workspace.StatePath, metadata) if err != nil { @@ -95,6 +79,7 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu return } deploymentID = id + db.StateDB.SetDeploymentID(deploymentID) } else if mask := metadata.StaleFields(current); mask != "" { if err := dmsClient.UpdateDeployment(ctx, deploymentID, metadata, mask); err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to update deployment: %w", err)) @@ -120,7 +105,8 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version if dmsClient == nil { return nil } - deploymentID, versionID := deploymentAndNextVersion(b) + deploymentID := db.StateDB.DeploymentID + versionID := int64(db.StateDB.Data.Serial) + 1 // The state serial tracks the version the service last recorded, which is the one this run // follows. Empty for the first version. From a96c80578b3ed1955f0a4a40e61f0ca987e78583 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 16:50:45 +0200 Subject: [PATCH 126/164] bundle/direct/dstate: make VersionID the recorded version, int all the way Under recording the version is the counter that matters, so the state carries it in an exported VersionID instead of borrowing Data.Serial. Open anchors it to the deployment last version, startVersion advances it in place when it creates one, and readers take it from there - so nothing has to recompute or pass it along. The plan still stamps its serial from it, and validatePlan compares against it, so stale plans are rejected exactly as before. Both directions are covered in one branch: behind the recorded version gets the version-worded error, anything else falls through to the serial comparison, whose numbers agree for both backends. The version is an int end to end now, through the DMS client and the URL helper, so there are no int64 conversions left at the comparison sites. logDeploymentVersion reads the deployment and version off the state rather than taking them as arguments. Co-authored-by: Isaac --- .../metadata/annotate_deployment_version.go | 6 ++--- bundle/direct/bundle_plan.go | 3 +++ bundle/direct/dstate/state.go | 24 ++++++++--------- bundle/phases/deploy.go | 11 +++----- bundle/phases/dms.go | 17 +++++++----- cmd/bundle/utils/process.go | 26 +++++++++++-------- libs/dms/client.go | 14 +++++----- libs/dms/metadata.go | 4 +-- libs/dms/metadata_test.go | 2 +- libs/dms/operation_buffer.go | 4 +-- libs/workspaceurls/urls.go | 4 +-- libs/workspaceurls/urls_test.go | 2 +- 12 files changed, 61 insertions(+), 56 deletions(-) diff --git a/bundle/deploy/metadata/annotate_deployment_version.go b/bundle/deploy/metadata/annotate_deployment_version.go index c08f7b52345..8a77250a3c4 100644 --- a/bundle/deploy/metadata/annotate_deployment_version.go +++ b/bundle/deploy/metadata/annotate_deployment_version.go @@ -37,12 +37,12 @@ func (m *annotateDeployment) Apply(_ context.Context, b *bundle.Bundle) diag.Dia } type annotateDeploymentVersion struct { - version int64 + version int } // AnnotateDeploymentVersion stamps the DMS version onto every job and pipeline. // Separate from AnnotateDeployment because version only exists after CreateVersion runs. -func AnnotateDeploymentVersion(version int64) bundle.Mutator { +func AnnotateDeploymentVersion(version int) bundle.Mutator { return &annotateDeploymentVersion{version: version} } @@ -51,7 +51,7 @@ func (m *annotateDeploymentVersion) Name() string { } func (m *annotateDeploymentVersion) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - versionID := strconv.FormatInt(m.version, 10) + versionID := strconv.Itoa(m.version) for _, job := range b.Config.Resources.Jobs { job.Deployment.VersionId = versionID diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 289cb1db500..01d42b97877 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -149,6 +149,9 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // built for a target of a different shape. if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { plan.Features = b.StateDB.StateFeatures() + // The state file persists no serial under recording; the version the service recorded is + // what a saved plan is validated against, so carry that instead. + plan.Serial = b.StateDB.VersionID } b.Plan = plan diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index d78a4dac1e2..d5d62a0a238 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -150,9 +150,10 @@ type DeploymentState struct { // reset - so, like operationBuffer and dmsClient, it must survive reset. DeploymentID string - // versionID is the version InitializeOperationBuffer created. CompleteVersion runs after - // Finalize, which resets Data, so the serial is no longer there to derive it from. - versionID int64 + // VersionID is the recorded version this run is anchored to: the deployment's last version at + // Open, then the version startVersion created. It lives outside Data because CompleteVersion + // runs after Finalize, which resets Data. Under recording this is the version, not the serial. + VersionID int } // DMSDeployment identifies the recorded deployment Open reads from. The zero value means the @@ -212,14 +213,14 @@ type WALEntry struct { // Open built, so it is called only when the deployment records history, and once the version exists // (after approval) - which is why it is not an Open option. It also records the id a first deploy // just created, which Open could not know, so CompleteVersion later has it. -func (db *DeploymentState) InitializeOperationBuffer(ctx context.Context, deploymentID string, versionID int64) { +func (db *DeploymentState) InitializeOperationBuffer(ctx context.Context, deploymentID string, versionID int) { buf := dms.StartOperationBuffer(ctx, db.dmsClient, deploymentID, versionID) db.mu.Lock() defer db.mu.Unlock() db.operationBuffer = buf db.DeploymentID = deploymentID - db.versionID = versionID + db.VersionID = versionID } // SetDeploymentID publishes the id of a deployment created after Open, which could not know it. @@ -262,7 +263,7 @@ func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (b return false, nil } db.versionCompleted = true - deploymentID, client, versionID := db.DeploymentID, db.dmsClient, db.versionID + deploymentID, client, versionID := db.DeploymentID, db.dmsClient, db.VersionID db.mu.Unlock() // A recording failure fails the version even when the caller counted the deploy a success: the @@ -624,18 +625,15 @@ To record this bundle's history, start it over as a new deployment: } } - // The service owns the version number, so the serial comes from it rather than from the - // state file, which no longer persists one. With no deployment yet there are no versions, - // hence zero - which matters for a state file written before recording was turned on, whose - // serial counts a history the service knows nothing about. - serial := 0 + // The service owns the version number, so it is read from the deployment rather than from + // the state file, which persists none. No deployment yet means no versions, hence zero. if dmsDeployment.LastVersionID != "" { - serial, err = strconv.Atoi(dmsDeployment.LastVersionID) + last, err := strconv.Atoi(dmsDeployment.LastVersionID) if err != nil { return fmt.Errorf("failed to parse last_version_id %q: %w", dmsDeployment.LastVersionID, err) } + db.VersionID = last } - db.Data.Serial = serial } if withWrite { diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 263c2b7a42e..e33086ca67b 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -331,14 +331,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // then - so stamp the one just created into the plan the apply reads. // IsDirect first: StorageBackend asserts the state is open, and only the direct engine opens it. if stateEngine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { - existingID := b.DeploymentBundle.StateDB.DeploymentID + firstDeploy := b.DeploymentBundle.StateDB.DeploymentID == "" createOrUpdateDeployment(ctx, b, dmsDeployment) if logdiag.HasError(ctx) { return } - if existingID == "" { - deploymentID := b.DeploymentBundle.StateDB.DeploymentID - if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(deploymentID); err != nil { + if firstDeploy { + if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(b.DeploymentBundle.StateDB.DeploymentID); err != nil { logdiag.LogError(ctx, err) return } @@ -356,9 +355,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand logdiag.LogError(ctx, err) return } - deploymentID := b.DeploymentBundle.StateDB.DeploymentID - versionID := int64(b.DeploymentBundle.StateDB.Data.Serial) + 1 - logDeploymentVersion(ctx, b, deploymentID, versionID) + logDeploymentVersion(ctx, b) } } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 3b9bc1d5b8f..fe03d4f3d86 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -106,13 +106,14 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version return nil } deploymentID := db.StateDB.DeploymentID - versionID := int64(db.StateDB.Data.Serial) + 1 - - // The state serial tracks the version the service last recorded, which is the one this run - // follows. Empty for the first version. + // This run's version follows the one the state is anchored to, which is empty for the first. + // InitializeOperationBuffer moves the anchor below, so callers afterwards read the created + // version straight off the state. + previous := db.StateDB.VersionID + versionID := previous + 1 previousVersionID := "" - if serial := db.StateDB.Data.Serial; serial > 0 { - previousVersionID = strconv.Itoa(serial) + if previous > 0 { + previousVersionID = strconv.Itoa(previous) } // The server rejects this unless the version number exceeds last_version_id and @@ -144,7 +145,9 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version // logDeploymentVersion logs the deployment version URL. Workspace ID is omitted // so the page stays clickable in a terminal and redirects correctly without it. -func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, deploymentID string, version int64) { +func logDeploymentVersion(ctx context.Context, b *bundle.Bundle) { + deploymentID := b.DeploymentBundle.StateDB.DeploymentID + version := b.DeploymentBundle.StateDB.VersionID if version == 0 { return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 57fe6a86845..1d50eb5629c 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -477,23 +477,27 @@ func validatePlan(b *bundle.Bundle, plan *deployplan.Plan) error { return errors.New("this plan was created for a different set of state features than the target now has; run 'bundle plan' again") } - // Under recording the serial tracks the version the service has recorded, so a plan built - // against an earlier one is stale. Checked ahead of the lineage guard because a plan from - // before the first deploy has no lineage and would otherwise exit through it. - if stateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService && plan.Serial < stateDB.Data.Serial { - return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, stateDB.Data.Serial) + // Recorded deployments keep this counter in VersionID, everything else in the state serial; the + // two agree after the same sequence of deploys. A recorded plan built against an earlier version + // gets the clearer message, and is checked before the lineage comparison because a plan taken + // before the first deploy carries no lineage to compare. + expected := stateDB.Data.Serial + if stateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + expected = stateDB.VersionID + if plan.Serial < expected { + return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, expected) + } + } + if plan.Serial != expected { + return fmt.Errorf("plan serial %d does not match state serial %d; the state has been modified since the plan was created. Please run 'bundle plan' again", plan.Serial, expected) } - // A plan taken before the first deploy carries no lineage or serial, so both sides are empty - // then and the checks below pass. If a deployment has happened since, they catch it. + // A plan taken before the first deploy carries no lineage, so both sides are empty then and this + // passes. If a deployment has happened since, the counter above has already caught it. if plan.Lineage != stateDB.Data.Lineage { return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) } - if plan.Serial != stateDB.Data.Serial { - return fmt.Errorf("plan serial %d does not match state serial %d; the state has been modified since the plan was created. Please run 'bundle plan' again", plan.Serial, stateDB.Data.Serial) - } - return nil } diff --git a/libs/dms/client.go b/libs/dms/client.go index c4f592dec11..605b4bbd4db 100644 --- a/libs/dms/client.go +++ b/libs/dms/client.go @@ -50,7 +50,7 @@ func DeploymentName(deploymentID string) string { return "deployments/" + deploymentID } -func versionName(deploymentID string, version int64) string { +func versionName(deploymentID string, version int) string { return fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version) } @@ -81,12 +81,12 @@ func (c *Client) DeleteDeployment(ctx context.Context, deploymentID string) erro } // CreateVersion claims the version and stages the operations body carries. -func (c *Client) CreateVersion(ctx context.Context, deploymentID string, version int64, body CreateVersionRequest) (*bundledeployments.Version, error) { - return c.raw.CreateVersion(ctx, deploymentID, strconv.FormatInt(version, 10), body) +func (c *Client) CreateVersion(ctx context.Context, deploymentID string, version int, body CreateVersionRequest) (*bundledeployments.Version, error) { + return c.raw.CreateVersion(ctx, deploymentID, strconv.Itoa(version), body) } // CompleteVersion closes the version out, which is what stops the service expiring its lease. -func (c *Client) CompleteVersion(ctx context.Context, deploymentID string, version int64, reason bundledeployments.VersionComplete) error { +func (c *Client) CompleteVersion(ctx context.Context, deploymentID string, version int, reason bundledeployments.VersionComplete) error { _, err := c.Service.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ Name: versionName(deploymentID, version), CompletionReason: reason, @@ -96,7 +96,7 @@ func (c *Client) CompleteVersion(ctx context.Context, deploymentID string, versi // UpdateOperation fills in one operation the version staged, and returns the sequence id the // next update for that resource must send. -func (c *Client) UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (string, error) { +func (c *Client) UpdateOperation(ctx context.Context, deploymentID string, version int, stateKey, sequenceID string, update OperationUpdate) (string, error) { return c.raw.UpdateOperation(ctx, deploymentID, version, stateKey, sequenceID, update) } @@ -124,7 +124,7 @@ type requester interface { // the service sends a JSON string, so it cannot read the response. sequenceID is the // token the previous update for this resource returned, or 0 for the first, which is // what staging leaves. - UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (next string, err error) + UpdateOperation(ctx context.Context, deploymentID string, version int, stateKey, sequenceID string, update OperationUpdate) (next string, err error) } // CreateVersionRequest is the CreateVersion request body. @@ -242,7 +242,7 @@ func newUpdateRequest(update OperationUpdate, sequenceID string) map[string]any return body } -func (r *rawClient) UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (string, error) { +func (r *rawClient) UpdateOperation(ctx context.Context, deploymentID string, version int, stateKey, sequenceID string, update OperationUpdate) (string, error) { body := newUpdateRequest(update, sequenceID) var result operationResponse diff --git a/libs/dms/metadata.go b/libs/dms/metadata.go index d63154a4790..bb67bd2a65c 100644 --- a/libs/dms/metadata.go +++ b/libs/dms/metadata.go @@ -83,11 +83,11 @@ func (m Metadata) StaleFields(current *bundledeployments.Deployment) string { // NextVersion is the version number this run will create, given the deployment's most recent // one. lastVersionID is empty before the deployment has any version. -func NextVersion(lastVersionID string) (int64, error) { +func NextVersion(lastVersionID string) (int, error) { if lastVersionID == "" { return 1, nil } - last, err := strconv.ParseInt(lastVersionID, 10, 64) + last, err := strconv.Atoi(lastVersionID) if err != nil { return 0, fmt.Errorf("failed to parse last_version_id %q: %w", lastVersionID, err) } diff --git a/libs/dms/metadata_test.go b/libs/dms/metadata_test.go index e3446466407..cddf4205cff 100644 --- a/libs/dms/metadata_test.go +++ b/libs/dms/metadata_test.go @@ -10,7 +10,7 @@ import ( func TestNextVersion(t *testing.T) { for _, tt := range []struct { last string - want int64 + want int }{ {"", 1}, {"4", 5}, diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go index 0718aac330c..11333404336 100644 --- a/libs/dms/operation_buffer.go +++ b/libs/dms/operation_buffer.go @@ -21,7 +21,7 @@ const stagedSequenceID = "0" type OperationBuffer struct { client *Client deploymentID string - versionNum int64 + versionNum int // The buffer. queue holds bundle state keys, and pending the newest update per key, so a // second write for a resource replaces the first. @@ -51,7 +51,7 @@ type OperationBuffer struct { // StartOperationBuffer opens the buffer for the version the caller just created. The version // must already exist: operations record under it, and nothing here creates it. -func StartOperationBuffer(ctx context.Context, client *Client, deploymentID string, versionNum int64) *OperationBuffer { +func StartOperationBuffer(ctx context.Context, client *Client, deploymentID string, versionNum int) *OperationBuffer { b := &OperationBuffer{ client: client, deploymentID: deploymentID, diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index 0b6a80fd71b..0a1680a0753 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -74,7 +74,7 @@ func ResourceTypes() []string { // DeploymentURL returns the workspace URL for a bundle deployment: // /deployments/?version=. Version pins the page to the deploy that produced it. -func DeploymentURL(baseURL url.URL, deploymentID string, version int64) string { +func DeploymentURL(baseURL url.URL, deploymentID string, version int) string { if deploymentID == "" { return "" } @@ -82,7 +82,7 @@ func DeploymentURL(baseURL url.URL, deploymentID string, version int64) string { baseURL.Path = "deployments/" + deploymentID if version > 0 { values := baseURL.Query() - values.Set("version", strconv.FormatInt(version, 10)) + values.Set("version", strconv.Itoa(version)) baseURL.RawQuery = values.Encode() } return baseURL.String() diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index b92df037c84..4667cd7bb3e 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -145,7 +145,7 @@ func TestDeploymentURL(t *testing.T) { tests := []struct { name string deploymentID string - version int64 + version int base url.URL expected string }{ From 77bd756678756329f7378dcc5fe3757b1b285825 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 16:59:02 +0200 Subject: [PATCH 127/164] cmd/bundle/utils: check lineage before the version, set the deployment id directly The lineage comparison runs first now, so a plan that predates the state is reported as a lineage mismatch rather than a version one. The stale-plan test keeps its version-worded case: a plan against the recorded deployment whose version has moved on still matches on lineage and falls through to the version check. The deployment id a first deploy creates is assigned to the state directly; a setter was not buying anything. Co-authored-by: Isaac --- acceptance/bundle/dms/stale-plan/output.txt | 2 +- bundle/direct/dstate/state.go | 17 ++++++----------- bundle/phases/dms.go | 2 +- cmd/bundle/utils/process.go | 17 ++++++++--------- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 30eea85623e..076fa5c46b5 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Replay: re-deploying the same plan is rejected - it predates the recorded version >>> musterr [CLI] bundle deploy --plan tmp.plan.json -Error: this plan was built against version 0 but the deployment has recorded version 1; run 'bundle plan' again +Error: plan lineage "" does not match state lineage "[UUID]"; the state may have been modified by another process === Only the first deploy recorded a version; the rejected replay created none diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index d5d62a0a238..e9329c65ac9 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -150,9 +150,12 @@ type DeploymentState struct { // reset - so, like operationBuffer and dmsClient, it must survive reset. DeploymentID string - // VersionID is the recorded version this run is anchored to: the deployment's last version at - // Open, then the version startVersion created. It lives outside Data because CompleteVersion - // runs after Finalize, which resets Data. Under recording this is the version, not the serial. + // VersionID is the source of truth for the deployment version: every version operation reads it + // - which version to create next, which one it follows, what the plan stamps, and what a saved + // plan is validated against. It holds the last version the service recorded, and once this run + // creates one, that new version. + // + // It sits outside Data because Finalize resets Data and CompleteVersion runs after that. VersionID int } @@ -223,14 +226,6 @@ func (db *DeploymentState) InitializeOperationBuffer(ctx context.Context, deploy db.VersionID = versionID } -// SetDeploymentID publishes the id of a deployment created after Open, which could not know it. -// Readers take the deployment from here rather than from the config tree. -func (db *DeploymentState) SetDeploymentID(id string) { - db.mu.Lock() - defer db.mu.Unlock() - db.DeploymentID = id -} - // RecordingError reports whether recording state writes to the service has failed, so the apply // stops touching resources once the service is no longer keeping up. Nil when the bundle does not // record deployment history or recording is healthy. diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index fe03d4f3d86..74dfc1b9690 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -79,7 +79,7 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu return } deploymentID = id - db.StateDB.SetDeploymentID(deploymentID) + db.StateDB.DeploymentID = deploymentID } else if mask := metadata.StaleFields(current); mask != "" { if err := dmsClient.UpdateDeployment(ctx, deploymentID, metadata, mask); err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to update deployment: %w", err)) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 1d50eb5629c..895aac91ec7 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -477,10 +477,15 @@ func validatePlan(b *bundle.Bundle, plan *deployplan.Plan) error { return errors.New("this plan was created for a different set of state features than the target now has; run 'bundle plan' again") } + // A plan taken before the first deploy carries no lineage, so both sides are empty then and this + // passes. If a deployment has happened since, the lineage no longer matches. + if plan.Lineage != stateDB.Data.Lineage { + return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) + } + // Recorded deployments keep this counter in VersionID, everything else in the state serial; the - // two agree after the same sequence of deploys. A recorded plan built against an earlier version - // gets the clearer message, and is checked before the lineage comparison because a plan taken - // before the first deploy carries no lineage to compare. + // two agree after the same sequence of deploys. A plan built against an earlier + // recorded version gets the clearer message; anything else falls through to the serial check. expected := stateDB.Data.Serial if stateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { expected = stateDB.VersionID @@ -492,12 +497,6 @@ func validatePlan(b *bundle.Bundle, plan *deployplan.Plan) error { return fmt.Errorf("plan serial %d does not match state serial %d; the state has been modified since the plan was created. Please run 'bundle plan' again", plan.Serial, expected) } - // A plan taken before the first deploy carries no lineage, so both sides are empty then and this - // passes. If a deployment has happened since, the counter above has already caught it. - if plan.Lineage != stateDB.Data.Lineage { - return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) - } - return nil } From f2c5ec68fdedef43e30e43fe0f56717dccdd51d2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 17:03:22 +0200 Subject: [PATCH 128/164] cmd/bundle/utils: assign the fetched deployment to its own variables The fetch declared local copies and immediately aliased them to the outer variables. Assign them directly and use them throughout, which also drops a second read of LastVersionId in favour of the lastVersionID already derived from it. Co-authored-by: Isaac --- cmd/bundle/utils/process.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 895aac91ec7..b862fe7366b 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -248,22 +248,20 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle _, localPath := b.StateFilenameDirect(ctx) if b.ConfiguresDeploymentHistory(ctx) { - deploymentID, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + var err error + dmsDeploymentID, dmsDeployment, err = fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } - dmsDeploymentID = deploymentID - dmsDeployment = deployment - // Stamp the deployment and the version this run records onto every job and pipeline so // the plan carries them. version_id is always known (last recorded + 1); deployment_id // does not exist until a first deploy creates it, so it is left off here and the deploy // phase stamps the created id. lastVersionID := "" - if deployment != nil { - lastVersionID = deployment.LastVersionId + if dmsDeployment != nil { + lastVersionID = dmsDeployment.LastVersionId } nextVersion, verr := dms.NextVersion(lastVersionID) if verr != nil { @@ -271,14 +269,14 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} - if deploymentID != "" { + if dmsDeploymentID != "" { bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { b.Config.Bundle.Deployment.History = &config.DeploymentHistory{ - DeploymentID: deploymentID, - LatestVersionID: deployment.LastVersionId, + DeploymentID: dmsDeploymentID, + LatestVersionID: lastVersionID, } }) - muts = append(muts, metadata.AnnotateDeployment(deploymentID)) + muts = append(muts, metadata.AnnotateDeployment(dmsDeploymentID)) } bundle.ApplySeqContext(ctx, b, muts...) if logdiag.HasError(ctx) { From 94bf4ed7b4439344a9f4c43a9e78b041dad2c37d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 17:12:40 +0200 Subject: [PATCH 129/164] acceptance: restore serial-mismatch script The tmp-file rename came from the nostamp convention, which this test does not use: plan.json is never a golden, it is removed at the end. Back to the in-place jq edit, which also drops the Ignore entry the extra file needed. Co-authored-by: Isaac --- acceptance/bundle/deploy/readplan/serial-mismatch/script | 6 +++--- acceptance/bundle/deploy/readplan/serial-mismatch/test.toml | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/script b/acceptance/bundle/deploy/readplan/serial-mismatch/script index 4f37c55f218..170f44a4c87 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/script +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/script @@ -2,13 +2,13 @@ trace $CLI bundle deploy # Generate a plan (captures current lineage and serial) -trace $CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle plan -o json > plan.json # Modify only the serial to simulate stale plan -jq '.serial = 999' tmp.plan.json > plan.json +jq '.serial = 999' plan.json > plan.json.tmp && mv plan.json.tmp plan.json # Try to deploy with stale plan (should fail with serial mismatch) musterr $CLI bundle deploy --plan plan.json -rm plan.json tmp.plan.json +rm plan.json trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/test.toml b/acceptance/bundle/deploy/readplan/serial-mismatch/test.toml index a35ede05603..dc4dcf06fc7 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/test.toml +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/test.toml @@ -1,4 +1,3 @@ # Test that --plan with mismatched serial produces an error EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = false -Ignore = ["tmp.plan.json"] From 4776d578fe33f4fc457c16540c94c460f9c2a14f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 17:16:59 +0200 Subject: [PATCH 130/164] acceptance/readplan: keep tmp plans out of the file count, run lineage-mismatch under recording The raw plans deploy --plan reads sit in the bundle root, so they were synced and the uploaded-file counts shifted. basic excludes them from sync; cli-version-mismatch needed no temp file at all - plan output pipes straight into jq. lineage-mismatch was skipped under recording because its committed plan.json carries no features, so the features check fired instead of the lineage one. Generating the plan and tampering only with its lineage - the same shape serial-mismatch uses - lets it carry whatever headers the variant produces, so it now runs both ways. Co-authored-by: Isaac --- acceptance/bundle/deploy/readplan/basic/databricks.yml | 5 +++++ acceptance/bundle/deploy/readplan/basic/output.txt | 4 ++-- .../bundle/deploy/readplan/cli-version-mismatch/output.txt | 2 +- .../bundle/deploy/readplan/cli-version-mismatch/script | 4 +--- .../bundle/deploy/readplan/cli-version-mismatch/test.toml | 1 - .../bundle/deploy/readplan/lineage-mismatch/out.test.toml | 2 +- .../bundle/deploy/readplan/lineage-mismatch/output.txt | 7 ++++--- .../bundle/deploy/readplan/lineage-mismatch/plan.json | 7 ------- acceptance/bundle/deploy/readplan/lineage-mismatch/script | 7 ++++++- .../bundle/deploy/readplan/lineage-mismatch/test.toml | 2 -- 10 files changed, 20 insertions(+), 21 deletions(-) delete mode 100644 acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json diff --git a/acceptance/bundle/deploy/readplan/basic/databricks.yml b/acceptance/bundle/deploy/readplan/basic/databricks.yml index b93d6b5b021..9c77e2e4d98 100644 --- a/acceptance/bundle/deploy/readplan/basic/databricks.yml +++ b/acceptance/bundle/deploy/readplan/basic/databricks.yml @@ -1,6 +1,11 @@ bundle: name: test-bundle +# The raw plans deploy --plan reads live in the bundle root; keep them out of the file count. +sync: + exclude: + - tmp.plan_*.json + resources: jobs: job: diff --git a/acceptance/bundle/deploy/readplan/basic/output.txt b/acceptance/bundle/deploy/readplan/basic/output.txt index d2e2627c79b..b47a16f11ed 100644 --- a/acceptance/bundle/deploy/readplan/basic/output.txt +++ b/acceptance/bundle/deploy/readplan/basic/output.txt @@ -6,7 +6,7 @@ >>> [CLI] bundle deploy --plan tmp.plan_create.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.job -Files: 7 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --get //jobs @@ -41,7 +41,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy --plan tmp.plan_skip.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Files: 4 uploaded, 0 deleted +Files: 3 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> print_requests.py --get //jobs diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/output.txt b/acceptance/bundle/deploy/readplan/cli-version-mismatch/output.txt index a20a3149cb1..7d6f07a44cd 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/output.txt +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/output.txt @@ -5,7 +5,7 @@ Warn: Plan was created with CLI version 1.2.3 but current version is [CLI_VERSION] Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.job -Files: 7 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "method": "POST", diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/script b/acceptance/bundle/deploy/readplan/cli-version-mismatch/script index 75122c8abee..e87a023055c 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/script +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/script @@ -1,6 +1,4 @@ -trace $CLI bundle plan -o json > tmp.plan.json - -jq '.cli_version = "1.2.3"' tmp.plan.json > plan.json +trace $CLI bundle plan -o json | jq '.cli_version = "1.2.3"' > plan.json trace $CLI bundle deploy --plan plan.json rm plan.json diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/test.toml b/acceptance/bundle/deploy/readplan/cli-version-mismatch/test.toml index f69f2af077d..c8f942fbf84 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/test.toml +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/test.toml @@ -1,4 +1,3 @@ # Test that --plan with different cli_version produces a warning but succeeds EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = true -Ignore = ["tmp.plan.json"] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt b/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt index 30736fe2178..bbc660d1f0f 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt @@ -1,5 +1,6 @@ -Warn: Plan was created with CLI version 0.0.0-test but current version is [CLI_VERSION] -Error: plan lineage "different-lineage" does not match state lineage ""; the state may have been modified by another process +>>> [CLI] bundle plan -o json + +>>> musterr [CLI] bundle deploy --plan plan.json +Error: plan lineage "different-lineage" does not match state lineage ""; the state may have been modified by another process -Exit code: 1 diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json deleted file mode 100644 index 42cce15cbfc..00000000000 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "plan_version": 2, - "cli_version": "0.0.0-test", - "lineage": "different-lineage", - "serial": 0, - "plan": {} -} diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/script b/acceptance/bundle/deploy/readplan/lineage-mismatch/script index d386f293771..113860e9edc 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/script +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/script @@ -1,2 +1,7 @@ +# Generate a plan, then tamper only with its lineage. Generated rather than committed so the plan +# carries whatever headers the variant produces, which lets this run with recording on too. +trace $CLI bundle plan -o json | jq '.lineage = "different-lineage"' > plan.json + # Try to deploy with mismatched lineage (should fail) -$CLI bundle deploy --plan plan.json +trace musterr $CLI bundle deploy --plan plan.json +rm plan.json diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml index 7591246cc57..de0ba18b25f 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml @@ -1,4 +1,2 @@ # Test that --plan with mismatched lineage produces an error EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# Committed plan.json lacks DMS headers, so skip DMS=true -EnvMatrixExclude.plan_no_dms = ["DMS=true"] From 73ca1d218d916101c66bbb72f67936380f2f42ad Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 17:27:58 +0200 Subject: [PATCH 131/164] acceptance: run escaped_refs under recording, drop a needless nostamp in auto-migrate-clean escaped_refs replayed the stamped-out plan dump, which cannot carry the headers deploy --plan validates, so it had opted out of recording. The raw plan now goes under .databricks - which sync skips, so the uploaded file count is unchanged - and the golden keeps the stamped-out copy. The test runs all four variants now. auto-migrate-clean stays without recording, but for the real reason: migrating an existing deployment is not supported with it. Its plan dump is never replayed, so the nostamp there was doing nothing. Co-authored-by: Isaac --- acceptance/bundle/escaped_refs/out.test.toml | 2 +- acceptance/bundle/escaped_refs/script | 10 ++++++++-- acceptance/bundle/escaped_refs/test.toml | 4 ---- acceptance/bundle/migrate/auto-migrate-clean/script | 2 +- acceptance/bundle/migrate/auto-migrate-clean/test.toml | 4 ++-- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/escaped_refs/out.test.toml b/acceptance/bundle/escaped_refs/out.test.toml index d5b9a6f4a16..bf4f0e7d3c6 100644 --- a/acceptance/bundle/escaped_refs/out.test.toml +++ b/acceptance/bundle/escaped_refs/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/escaped_refs/script b/acceptance/bundle/escaped_refs/script index 0f75085c239..c23d01d3e5d 100644 --- a/acceptance/bundle/escaped_refs/script +++ b/acceptance/bundle/escaped_refs/script @@ -1,6 +1,9 @@ # Deploys the same config the invariant suite uses, and records the plan and the # job payload so both engines are shown to send "${...}" (single $) to the API. envsubst < "$TESTDIR/../invariant/configs/job_escaped_refs.yml.tmpl" > databricks.yml +# The raw plan the replay reads goes under .databricks, which sync skips, so it stays out of the +# uploaded file count. +mkdir -p .databricks cleanup() { $CLI bundle destroy --auto-approve &> LOG.destroy @@ -12,12 +15,15 @@ title "Validate: escapes are still escaped in the config" trace $CLI bundle validate -o json | jq '.resources.jobs.example_ingestion_job.tasks[0].notebook_task.base_parameters' title "Plan (json)" -trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json > .databricks/tmp.plan.json +# The golden gets the stamped-out copy; the replay below reads the raw one, which still carries +# the DMS headers deploy --plan validates. +nostamp < .databricks/tmp.plan.json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Not traced: the READPLAN variant adds "--plan ...", and the point of that variant is # that the payload below is identical either way. title "Deploy: payload sent to the jobs API\n" -$CLI bundle deploy $(readplanarg out.plan.$DATABRICKS_BUNDLE_ENGINE.json) +$CLI bundle deploy $(readplanarg .databricks/tmp.plan.json) # --nostamp: under deployment-history recording the job payload carries the DMS deployment # stamp; drop it so the assertion is the same whether or not recording is on. trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/escaped_refs/test.toml b/acceptance/bundle/escaped_refs/test.toml index 94b36638c0e..e7b9cac3bf1 100644 --- a/acceptance/bundle/escaped_refs/test.toml +++ b/acceptance/bundle/escaped_refs/test.toml @@ -8,7 +8,3 @@ Ignore = [ ".databricks", "databricks.yml", ] - -# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't -# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). -EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto-migrate-clean/script b/acceptance/bundle/migrate/auto-migrate-clean/script index d2779441da1..0f2eb7c1ce1 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/script +++ b/acceptance/bundle/migrate/auto-migrate-clean/script @@ -31,7 +31,7 @@ rm -f out.requests.txt title "Modify the config — plan shows the pending update" trace update_file.py databricks.yml "Test Auto-Migrate Job" "Test Auto-Migrate Job (renamed)" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_update.direct.json +$CLI bundle plan -o json > out.plan_update.direct.json title "Deploy picks up the update (with and without --plan)\n" $CLI bundle deploy $(readplanarg out.plan_update.direct.json) diff --git a/acceptance/bundle/migrate/auto-migrate-clean/test.toml b/acceptance/bundle/migrate/auto-migrate-clean/test.toml index 32db4804fbf..513685bc005 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/test.toml +++ b/acceptance/bundle/migrate/auto-migrate-clean/test.toml @@ -1,5 +1,5 @@ EnvMatrix.READPLAN = ["", "1"] -# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't -# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +# Migrating an existing deployment is not supported with deployment-history recording, so this +# test only runs without it (recording is covered by bundle/dms). EnvMatrix.DMS = [""] From d1aa9bdeca40da83e098eb9914ca142a13b087ff Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 17:33:31 +0200 Subject: [PATCH 132/164] acceptance: run two more jobs tests under recording, document the grants badness delete_task and remote_delete/deploy replayed their stamped-out plan dumps, which cannot carry the headers deploy --plan validates. The raw plans now go under .databricks - sync skips it, so uploaded file counts are unchanged - and the goldens keep the stamped-out copies. big_id and update stay excluded: they dump state, not just plans, and a recorded state dump differs by more than a stamp (features gained, serial dropped, deployment and version ids on the job payloads). grants/schemas/remove_all gets a Badness entry naming why it is excluded, matching bundle/dms/empty-grants. Co-authored-by: Isaac --- .../resources/grants/schemas/remove_all/test.toml | 2 ++ .../bundle/resources/jobs/delete_task/out.test.toml | 2 +- acceptance/bundle/resources/jobs/delete_task/script | 13 +++++++++---- .../bundle/resources/jobs/delete_task/test.toml | 3 --- .../jobs/remote_delete/deploy/out.test.toml | 2 +- .../resources/jobs/remote_delete/deploy/script | 8 ++++++-- .../resources/jobs/remote_delete/deploy/test.toml | 3 --- 7 files changed, 19 insertions(+), 14 deletions(-) diff --git a/acceptance/bundle/resources/grants/schemas/remove_all/test.toml b/acceptance/bundle/resources/grants/schemas/remove_all/test.toml index df54156495f..b7973b8bdee 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_all/test.toml +++ b/acceptance/bundle/resources/grants/schemas/remove_all/test.toml @@ -1,3 +1,5 @@ +Badness = "Emptying a grants node cannot be deployed with deployment-history recording: apply calls DeleteState, which records a succeeded UPDATE carrying no state, and the service rejects that. This test is excluded from the recording variant until the fix records the emptied state instead of a delete." + # Emptying a grants node is a known bug under DMS (deployment-history recording). # It records a succeeded UPDATE with no state, which the service rejects. # See: acceptance/bundle/dms/empty-grants diff --git a/acceptance/bundle/resources/jobs/delete_task/out.test.toml b/acceptance/bundle/resources/jobs/delete_task/out.test.toml index d5b9a6f4a16..bf4f0e7d3c6 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/delete_task/script b/acceptance/bundle/resources/jobs/delete_task/script index 94d69088c8a..d4b38a78cc8 100644 --- a/acceptance/bundle/resources/jobs/delete_task/script +++ b/acceptance/bundle/resources/jobs/delete_task/script @@ -1,6 +1,11 @@ -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle deploy $(readplanarg out.plan_create.direct.json) +# The raw plans the replays read go under .databricks, which sync skips. +mkdir -p .databricks + +$CLI bundle plan -o json > .databricks/tmp.plan_create.json +nostamp < .databricks/tmp.plan_create.json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle deploy $(readplanarg .databricks/tmp.plan_create.json) grep -v TO_DELETE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle deploy $(readplanarg out.plan_update.direct.json) +$CLI bundle plan -o json > .databricks/tmp.plan_update.json +nostamp < .databricks/tmp.plan_update.json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle deploy $(readplanarg .databricks/tmp.plan_update.json) diff --git a/acceptance/bundle/resources/jobs/delete_task/test.toml b/acceptance/bundle/resources/jobs/delete_task/test.toml index 04d6ea1e5b3..d4bbeb7e7ef 100644 --- a/acceptance/bundle/resources/jobs/delete_task/test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/test.toml @@ -1,5 +1,2 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] -# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, -# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. -EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml index d5b9a6f4a16..bf4f0e7d3c6 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/script b/acceptance/bundle/resources/jobs/remote_delete/deploy/script index 658fca8f399..b12efc6affc 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/script +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/script @@ -1,7 +1,11 @@ +# The raw plans the replays read go under .databricks, which sync skips. +mkdir -p .databricks + trace $CLI bundle plan trace $CLI bundle deploy job_id=`$CLI bundle summary -o json | jq -r .resources.jobs.foo.id` trace $CLI jobs delete $job_id trace $CLI bundle plan -$CLI bundle plan -o json | nostamp &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle deploy $(readplanarg out.plan.direct.json) +$CLI bundle plan -o json &> .databricks/tmp.plan.json +nostamp < .databricks/tmp.plan.json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle deploy $(readplanarg .databricks/tmp.plan.json) diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml index 04d6ea1e5b3..d4bbeb7e7ef 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml @@ -1,5 +1,2 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] -# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, -# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. -EnvMatrix.DMS = [""] From c73985dd0c9d27e32cd962149b806c5ca84d6e78 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 17:54:22 +0200 Subject: [PATCH 133/164] Report the recorded version where the state serial used to be read A recorded state file persists no serial, so anything reading it saw zero. Two readers now take the recorded version instead, which means the same thing and matches what a non-recording run reports. print_state.py fills the serial in from the deployment last_version_id and drops the feature that marks recording, so a recorded run prints the state a non-recording one would. The deployment lookup tolerates a missing record, since destroy deletes it while the state file remains. config-remote-sync telemetry takes the version from the state when it is set, keeping state_serial comparable across backends rather than dropping out under recording. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 34 +++++++++++++++++++++++++++++++- bundle/configsync/telemetry.go | 8 +++++++- cmd/bundle/config_remote_sync.go | 2 +- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 42e55b39648..3c8e8db7dd7 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -151,6 +151,19 @@ def get_recorded_state(target): return state +def get_last_version_id(target): + """The version the service has recorded for this deployment, or None when nothing is recorded.""" + state_path = get_remote_state_path(target) + if not state_path: + return None + node = run_json([CLI, "workspace", "get-status", f"{state_path}/{DEPLOYMENT_NODE_NAME}"], allow_failure=True) + if not node or not node.get("object_id"): + return None + # Tolerate a missing deployment: a destroy deletes the record while the state file remains. + deployment = run_json([CLI, "api", "get", f"/api/2.0/bundle/deployments/{node['object_id']}"], allow_failure=True) + return (deployment or {}).get("last_version_id") + + def print_recorded_state(filename, target): """Print the state file with its resources filled in from the deployment metadata service. @@ -159,7 +172,26 @@ def print_recorded_state(filename, target): """ data = json.loads(open(filename).read()) data["state"] = get_recorded_state(target) - print(json.dumps(data, indent=1)) + + # The service owns the version and the file persists no serial, so take it from the deployment. + # Rebuilt in header order, since the file has no serial key to overwrite in place. + last_version_id = get_last_version_id(target) + serial = int(last_version_id) if last_version_id else 0 + + # Recording itself is not what these tests assert, so drop the feature that marks it. + features = {k: v for k, v in (data.get("features") or {}).items() if k != "deployment_history"} + + rebuilt = {} + for key in ("state_version", "cli_version", "lineage"): + if key in data: + rebuilt[key] = data[key] + rebuilt["serial"] = serial + if features: + rebuilt["features"] = features + for key, value in data.items(): + if key not in rebuilt and key != "features": + rebuilt[key] = value + print(json.dumps(rebuilt, indent=1)) def main(): diff --git a/bundle/configsync/telemetry.go b/bundle/configsync/telemetry.go index cb7c4ed70c3..79a1ea3aa84 100644 --- a/bundle/configsync/telemetry.go +++ b/bundle/configsync/telemetry.go @@ -81,11 +81,17 @@ type Stats struct { // // None of these can contain PII: the lineage is an opaque random UUID minted by // the state layer, and the source is one of two fixed literals. -func (s *Stats) CollectStateStats(desc *statemgmt.StateDesc) { +// recordedVersion is the version the deployment metadata service holds, zero when the bundle does +// not record deployment history. A recorded state file persists no serial of its own, so the +// recorded version stands in for it and the field means the same thing for both backends. +func (s *Stats) CollectStateStats(desc *statemgmt.StateDesc, recordedVersion int) { if desc == nil { return } s.StateSerial = int64(desc.Serial) + if recordedVersion > 0 { + s.StateSerial = int64(recordedVersion) + } s.StateLineage = desc.Lineage s.StateSource = "remote" if desc.IsLocal { diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index 1f3e6777298..51f418f9180 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -82,7 +82,7 @@ Examples: }, PostStateFunc: func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { stats.Engine = stateDesc.Engine - stats.CollectStateStats(stateDesc) + stats.CollectStateStats(stateDesc, b.DeploymentBundle.StateDB.VersionID) // Open the deployment state once and reuse it for both planning and // selector resolution (avoids reading the terraform snapshot twice). From c7135c1617b9999b3b0f27854f774df65c019bbe Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 18:01:04 +0200 Subject: [PATCH 134/164] acceptance: run jobs/big_id under recording The replay reads the raw plan from under .databricks, which sync skips, and the state dump goes through nostamp to drop the deployment stamp recording adds to each resource payload. nostamp needs --indent 1 here: a state dump uses a single space, and its default of two is for plans, so without it the whole file reformats. Co-authored-by: Isaac --- acceptance/bundle/resources/jobs/big_id/out.test.toml | 2 +- acceptance/bundle/resources/jobs/big_id/script | 9 ++++++--- acceptance/bundle/resources/jobs/big_id/test.toml | 3 --- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/resources/jobs/big_id/out.test.toml b/acceptance/bundle/resources/jobs/big_id/out.test.toml index bf0f576652e..dd03baaabb6 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.test.toml +++ b/acceptance/bundle/resources/jobs/big_id/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script index e6915a6acd3..8bd6fa5530b 100644 --- a/acceptance/bundle/resources/jobs/big_id/script +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -1,8 +1,11 @@ +mkdir -p .databricks + trace $CLI bundle validate -o json | jq .resources > out.validate.json -trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle deploy $(readplanarg out.plan.direct.json) +trace $CLI bundle plan -o json > .databricks/tmp.plan.json +nostamp < .databricks/tmp.plan.json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle deploy $(readplanarg .databricks/tmp.plan.json) trace print_requests.py //jobs --nostamp -print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json +print_state.py | nostamp --indent 1 > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan | contains.py '0 to add, 0 to change, 0 to delete' trace $CLI bundle destroy --auto-approve trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/jobs/big_id/test.toml b/acceptance/bundle/resources/jobs/big_id/test.toml index 8957ed9ce06..e8500a3ae18 100644 --- a/acceptance/bundle/resources/jobs/big_id/test.toml +++ b/acceptance/bundle/resources/jobs/big_id/test.toml @@ -3,9 +3,6 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ['direct'] EnvMatrix.READPLAN = ["", "1"] -# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, -# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. -EnvMatrix.DMS = [""] [[Repls]] Old = '9223372036854775807' From ab50d9dde7947e7ab61a521261114d14bc883687 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 18:04:47 +0200 Subject: [PATCH 135/164] acceptance/telemetry: name the serial in the recording exclusion reason The suite asserts numbers recording legitimately changes - the serialized state size, and now the state serial, which comes from the recorded version rather than the file. Co-authored-by: Isaac --- acceptance/bundle/telemetry/test.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/telemetry/test.toml b/acceptance/bundle/telemetry/test.toml index 0fe0f33b778..ba5f7a2f8a6 100644 --- a/acceptance/bundle/telemetry/test.toml +++ b/acceptance/bundle/telemetry/test.toml @@ -1,8 +1,10 @@ RecordRequests = true IncludeRequestHeaders = ["User-Agent"] -# Telemetry reports the byte size of the serialized state, which the deployment stamp -# legitimately grows. The number is the assertion here, so there is nothing to normalize. +# Telemetry reports numbers about the state that recording legitimately changes: the byte size of +# the serialized state, which the deployment stamp grows, and the state serial, which comes from the +# deployment's recorded version rather than the file. Those numbers are the assertion here, so there +# is nothing to normalize. EnvMatrix.DMS = [""] Env.DATABRICKS_CACHE_ENABLED = 'false' From a7e430a29ef0ee416fb654b8fb9eafc8978afd09 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 18:09:19 +0200 Subject: [PATCH 136/164] acceptance/state_present: do not run it with recording The test deploys on terraform and migrates to direct. Terraform deployments are not supported with deployment-history recording, so the recording variant has nothing to exercise here. Co-authored-by: Isaac --- acceptance/bundle/state/state_present/out.test.toml | 2 +- acceptance/bundle/state/state_present/test.toml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/state/state_present/out.test.toml b/acceptance/bundle/state/state_present/out.test.toml index e1af1a235ad..ae800809893 100644 --- a/acceptance/bundle/state/state_present/out.test.toml +++ b/acceptance/bundle/state/state_present/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/state_present/test.toml b/acceptance/bundle/state/state_present/test.toml index e25c2749483..041121b226d 100644 --- a/acceptance/bundle/state/state_present/test.toml +++ b/acceptance/bundle/state/state_present/test.toml @@ -1,2 +1,6 @@ +# This test deploys on terraform and migrates to direct. Terraform deployments are not supported +# with deployment-history recording, so it only runs without it. +EnvMatrix.DMS = [""] + RecordRequests = true IncludeRequestHeaders = ["User-Agent"] From 320cd349ca403d9e9efdbd4f2684c0aaa281b2d7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 18:12:24 +0200 Subject: [PATCH 137/164] acceptance: normalize the state serial in lineage-mismatch-after-redeploy The candidate list the CLI prints carries each state file serial, and a recorded deployment keeps that counter in the service rather than the file. The test is about the lineage mismatch, not the counter, so the serial is replaced. Co-authored-by: Isaac --- .../lineage-mismatch-after-redeploy/output.txt | 12 ++++++------ .../lineage-mismatch-after-redeploy/test.toml | 6 ++++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt index cd5a4ff7df6..ae0b07d901b 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt @@ -29,22 +29,22 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Error: lineage mismatch in state files Available state files: -- resources.json: remote direct state serial=1 lineage="[LINEAGE_B]" -- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[LINEAGE_A]" +- resources.json: remote direct state serial=[SERIAL] lineage="[LINEAGE_B]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=[SERIAL] lineage="[LINEAGE_A]" >>> musterr [CLI] bundle deploy Error: lineage mismatch in state files Available state files: -- resources.json: remote direct state serial=1 lineage="[LINEAGE_B]" -- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[LINEAGE_A]" +- resources.json: remote direct state serial=[SERIAL] lineage="[LINEAGE_B]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=[SERIAL] lineage="[LINEAGE_A]" >>> musterr [CLI] bundle destroy --auto-approve Error: lineage mismatch in state files Available state files: -- resources.json: remote direct state serial=1 lineage="[LINEAGE_B]" -- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[LINEAGE_A]" +- resources.json: remote direct state serial=[SERIAL] lineage="[LINEAGE_B]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=[SERIAL] lineage="[LINEAGE_A]" diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml index e2d630064c8..d6a9102dbf2 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml @@ -4,3 +4,9 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # .databricks.machineA is the stashed local state; may linger if the test fails mid-run. Ignore = [".databricks.machineA"] + +# The state serial differs by backend: a recorded deployment keeps its counter in the service, so +# the local file has none. This test is about the lineage mismatch, not the counter. +[[Repls]] +Old = 'serial=[0-9]+' +New = 'serial=[SERIAL]' From 9f034392d4b3a30f185fbd6e143d34c7b01eb9dc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 18:15:10 +0200 Subject: [PATCH 138/164] acceptance: say why the serial repl is needed The serial is not set under DMS - the state file persists none - so the candidate list the CLI prints shows serial=0 where a non-recording run shows the file counter. Co-authored-by: Isaac --- .../bundle/destroy/lineage-mismatch-after-redeploy/test.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml index d6a9102dbf2..a075bb4eefc 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml @@ -5,8 +5,8 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # .databricks.machineA is the stashed local state; may linger if the test fails mid-run. Ignore = [".databricks.machineA"] -# The state serial differs by backend: a recorded deployment keeps its counter in the service, so -# the local file has none. This test is about the lineage mismatch, not the counter. +# Needed because the serial is not set under DMS: the state file persists none, so the candidate +# list the CLI prints shows serial=0 where a non-recording run shows the file's counter. [[Repls]] Old = 'serial=[0-9]+' New = 'serial=[SERIAL]' From e1339e2f1da2e207ef44ee9af63168534d23d910 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 18:24:29 +0200 Subject: [PATCH 139/164] acceptance: resolve the deployment once, keep the lineage-mismatch fixture print_state.py resolved the deployment node twice - once to list resources, once for the version. It is one cached lookup now, shared by both. The speculative allow_failure on the deployment GET is gone: after a destroy the node is already absent, so that call is never reached, which destroy/lineage-mismatch-after-redeploy exercises. Recording only applies to the direct engine, so a terraform run prints the state file as-is rather than taking the recorded path. readplan/lineage-mismatch keeps its committed plan.json. A committed plan carries no features, so under recording the feature check fires before the lineage one it asserts, and nostamp cannot help - it normalizes output, not the plan fed in. It opts out of the recording variant with that reason stated. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 44 ++++++++++++------- .../readplan/lineage-mismatch/out.test.toml | 2 +- .../readplan/lineage-mismatch/output.txt | 7 ++- .../readplan/lineage-mismatch/plan.json | 7 +++ .../deploy/readplan/lineage-mismatch/script | 7 +-- .../readplan/lineage-mismatch/test.toml | 5 +++ 6 files changed, 44 insertions(+), 28 deletions(-) create mode 100644 acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 3c8e8db7dd7..c021cc066df 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -100,22 +100,33 @@ def get_remote_state_path(target): @functools.cache -def get_resources(target): - """Map every recorded resource key ("jobs.foo") to its {"id", "state"}. +def get_deployment_id(target): + """The recorded deployment's id, or None when nothing is recorded. - Empty when the bundle has no deployment recorded yet. Cached because a lookup costs three - round trips and a script asks for one resource at a time. + Cached and shared: both the resource listing and the version lookup need it, and resolving it + costs a workspace round trip. No node means nothing has been recorded, the conclusion + dms.resolveDeploymentID also draws from a 404 - the deployment is gone once the bundle is + destroyed. """ state_path = get_remote_state_path(target) if not state_path: - return {} - - # No node means nothing has been recorded, the conclusion dms.resolveDeploymentID also draws - # from a 404 - the deployment is gone once the bundle is destroyed. + return None node = run_json([CLI, "workspace", "get-status", f"{state_path}/{DEPLOYMENT_NODE_NAME}"], allow_failure=True) if not node or not node.get("object_id"): + return None + return node["object_id"] + + +@functools.cache +def get_resources(target): + """Map every recorded resource key ("jobs.foo") to its {"id", "state"}. + + Empty when the bundle has no deployment recorded yet. Cached because a lookup costs three + round trips and a script asks for one resource at a time. + """ + deployment_id = get_deployment_id(target) + if not deployment_id: return {} - deployment_id = node["object_id"] result = {} # The service pages at 50 resources; the local fake returns everything at once. @@ -153,14 +164,10 @@ def get_recorded_state(target): def get_last_version_id(target): """The version the service has recorded for this deployment, or None when nothing is recorded.""" - state_path = get_remote_state_path(target) - if not state_path: - return None - node = run_json([CLI, "workspace", "get-status", f"{state_path}/{DEPLOYMENT_NODE_NAME}"], allow_failure=True) - if not node or not node.get("object_id"): + deployment_id = get_deployment_id(target) + if not deployment_id: return None - # Tolerate a missing deployment: a destroy deletes the record while the state file remains. - deployment = run_json([CLI, "api", "get", f"/api/2.0/bundle/deployments/{node['object_id']}"], allow_failure=True) + deployment = run_json([CLI, "api", "get", f"/api/2.0/bundle/deployments/{deployment_id}"]) return (deployment or {}).get("last_version_id") @@ -209,7 +216,10 @@ def main(): for filename in get_state_files(args.target, args.backup): if not os.path.exists(filename): continue - if os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true" and not args.no_dms: + # Recording only applies to the direct engine, so a terraform run prints the file as-is. + recording = os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true" + terraform = os.environ.get("DATABRICKS_BUNDLE_ENGINE") == "terraform" + if recording and not terraform and not args.no_dms: print_recorded_state(filename, args.target) else: print_file(filename) diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt b/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt index bbc660d1f0f..30736fe2178 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt @@ -1,6 +1,5 @@ - ->>> [CLI] bundle plan -o json - ->>> musterr [CLI] bundle deploy --plan plan.json +Warn: Plan was created with CLI version 0.0.0-test but current version is [CLI_VERSION] Error: plan lineage "different-lineage" does not match state lineage ""; the state may have been modified by another process + +Exit code: 1 diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json new file mode 100644 index 00000000000..42cce15cbfc --- /dev/null +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json @@ -0,0 +1,7 @@ +{ + "plan_version": 2, + "cli_version": "0.0.0-test", + "lineage": "different-lineage", + "serial": 0, + "plan": {} +} diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/script b/acceptance/bundle/deploy/readplan/lineage-mismatch/script index 113860e9edc..d386f293771 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/script +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/script @@ -1,7 +1,2 @@ -# Generate a plan, then tamper only with its lineage. Generated rather than committed so the plan -# carries whatever headers the variant produces, which lets this run with recording on too. -trace $CLI bundle plan -o json | jq '.lineage = "different-lineage"' > plan.json - # Try to deploy with mismatched lineage (should fail) -trace musterr $CLI bundle deploy --plan plan.json -rm plan.json +$CLI bundle deploy --plan plan.json diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml index de0ba18b25f..527d0966f26 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml @@ -1,2 +1,7 @@ # Test that --plan with mismatched lineage produces an error EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The committed plan.json carries no features, while a recorded target has deployment_history, so +# under DMS the feature check fires before the lineage one this test asserts. nostamp cannot help: +# it normalizes output, not the plan being fed in. +EnvMatrixExclude.plan_no_dms = ["DMS=true"] From 324bd9d146aefa7fe5407b4c56688abca586cc44 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 18:30:57 +0200 Subject: [PATCH 140/164] Scrub the deployment stamp inside print_state.py, run two more tests with recording print_state.py now drops the stamp recording adds to each resource payload, so a caller no longer has to pipe through nostamp - which was wrong for any test whose matrix includes terraform, since a terraform state file uses a different indent and the pipe reformatted it. big_id drops that pipe accordingly. With that, jobs/update and state/feature_flags run with recording. update replays the raw plan from under .databricks and scrubs the job it reads back from the API. quality_monitors/change_assets_dir stays out: it captures the plan with errcode ... &>, so the trace line lands in the file alongside the JSON and nothing can filter it. Also renames dstate.DMSDeployment to OpenDmsArgs. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 6 +++- .../bundle/resources/jobs/big_id/script | 2 +- .../resources/jobs/update/out.test.toml | 2 +- .../bundle/resources/jobs/update/script | 20 +++++++---- .../bundle/resources/jobs/update/test.toml | 3 -- .../bundle/state/feature_flags/out.test.toml | 2 +- .../bundle/state/feature_flags/test.toml | 3 -- bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +++---- bundle/direct/dstate/state.go | 8 ++--- bundle/direct/dstate/state_test.go | 36 +++++++++---------- bundle/statemgmt/direct_migration.go | 2 +- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 4 +-- 16 files changed, 56 insertions(+), 52 deletions(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index c021cc066df..7eb9ea89c79 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -14,6 +14,8 @@ import posixpath import subprocess +from nostamp import scrub + def print_file(filename): data = open(filename).read() @@ -178,7 +180,9 @@ def print_recorded_state(filename, target): printing it raw would show an empty state and differ from the same test's non-recording run. """ data = json.loads(open(filename).read()) - data["state"] = get_recorded_state(target) + # Recording stamps each resource payload with the deployment and version; drop it here so the + # printed state matches a non-recording run without every caller piping through nostamp. + data["state"] = scrub(get_recorded_state(target)) # The service owns the version and the file persists no serial, so take it from the deployment. # Rebuilt in header order, since the file has no serial key to overwrite in place. diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script index 8bd6fa5530b..6ad92c41f56 100644 --- a/acceptance/bundle/resources/jobs/big_id/script +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -5,7 +5,7 @@ trace $CLI bundle plan -o json > .databricks/tmp.plan.json nostamp < .databricks/tmp.plan.json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg .databricks/tmp.plan.json) trace print_requests.py //jobs --nostamp -print_state.py | nostamp --indent 1 > out.state.$DATABRICKS_BUNDLE_ENGINE.json +print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan | contains.py '0 to add, 0 to change, 0 to delete' trace $CLI bundle destroy --auto-approve trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/jobs/update/out.test.toml b/acceptance/bundle/resources/jobs/update/out.test.toml index d5b9a6f4a16..bf4f0e7d3c6 100644 --- a/acceptance/bundle/resources/jobs/update/out.test.toml +++ b/acceptance/bundle/resources/jobs/update/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/update/script b/acceptance/bundle/resources/jobs/update/script index dbc80a88b72..2ad2ae4d14f 100644 --- a/acceptance/bundle/resources/jobs/update/script +++ b/acceptance/bundle/resources/jobs/update/script @@ -1,22 +1,28 @@ +# The raw plans the replays read go under .databricks, which sync skips. +mkdir -p .databricks + echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle deploy $(readplanarg out.plan_create.direct.json) +$CLI bundle plan -o json > .databricks/tmp.plan_create.json +nostamp < .databricks/tmp.plan_create.json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle deploy $(readplanarg .databricks/tmp.plan_create.json) trace print_requests.py //jobs --nostamp > out.create.requests.json print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json > .databricks/tmp.plan_skip.json +nostamp < .databricks/tmp.plan_skip.json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle deploy $(readplanarg out.plan_skip.direct.json) +$CLI bundle deploy $(readplanarg .databricks/tmp.plan_skip.json) trace print_requests.py //jobs --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json > .databricks/tmp.plan_update.json +nostamp < .databricks/tmp.plan_update.json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle deploy $(readplanarg out.plan_update.direct.json) +$CLI bundle deploy $(readplanarg .databricks/tmp.plan_update.json) trace print_requests.py //jobs --nostamp | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json trace $CLI bundle plan @@ -25,7 +31,7 @@ title "Fetch job ID and verify remote state" ppid=`read_id.py foo` -trace $CLI jobs get $ppid | jq 'del(.settings.run_as)' +trace $CLI jobs get $ppid | jq 'del(.settings.run_as)' | nostamp rm out.requests.txt title "Destroy the job and verify that it's removed from the state and from remote" diff --git a/acceptance/bundle/resources/jobs/update/test.toml b/acceptance/bundle/resources/jobs/update/test.toml index a8a43da097a..ff8a66c196e 100644 --- a/acceptance/bundle/resources/jobs/update/test.toml +++ b/acceptance/bundle/resources/jobs/update/test.toml @@ -1,4 +1 @@ EnvMatrix.READPLAN = ["", "1"] -# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, -# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. -EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/state/feature_flags/test.toml b/acceptance/bundle/state/feature_flags/test.toml index dfe3377a2b7..5640bffa4dc 100644 --- a/acceptance/bundle/state/feature_flags/test.toml +++ b/acceptance/bundle/state/feature_flags/test.toml @@ -1,6 +1,3 @@ Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# State feature-flag handling, not DMS recording: opt out of the DMS variant so the deploy in the -# empty-features case does not record (which would diverge from the non-recording variant). -EnvMatrix.DMS = [""] diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 3bf8e08fcb7..b715a1f6fa9 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index aab709ab4b8..dba2164b8c2 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -156,7 +156,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 3a2ce2f54e6..115673db731 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -65,7 +65,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac // The state is opened without a DMS client, so the writes below record nothing; // phases.Bind and phases.Unbind refuse to run at all when recording is enabled. var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -89,7 +89,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -112,7 +112,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -148,7 +148,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -168,7 +168,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -218,7 +218,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}) if err != nil { if errors.Is(err, dstate.ErrUnsettingRecording) { return errors.New("unbind is not supported for a bundle target that records deployment history") diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index e9329c65ac9..77673f3d292 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -159,9 +159,9 @@ type DeploymentState struct { VersionID int } -// DMSDeployment identifies the recorded deployment Open reads from. The zero value means the +// OpenDmsArgs identifies the recorded deployment Open reads from. The zero value means the // bundle does not record deployment history, or no deployment exists for it yet. -type DMSDeployment struct { +type OpenDmsArgs struct { // ID is the deployment's server-minted id. ID string @@ -484,7 +484,7 @@ type ( // recorded deploy); lineage and serial still come from the file, since that is what the write path // increments. Open only reads through the client - InitializeOperationBuffer installs the write path once a // version exists. -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeployment DMSDeployment) error { +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeployment OpenDmsArgs) error { db.mu.Lock() defer db.mu.Unlock() @@ -515,7 +515,7 @@ func (db *DeploymentState) reset() { db.openedForWrite = false } -func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeployment DMSDeployment) error { +func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeployment OpenDmsArgs) error { db.Path = path // The state file is the source of truth for whether this deployment records history: read it diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index bea61811634..24af630b069 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -21,14 +21,14 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), OpenDmsArgs{})) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), OpenDmsArgs{})) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) @@ -38,7 +38,7 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), OpenDmsArgs{})) mustFinalize(t, &db) _, err := os.Stat(path) @@ -94,10 +94,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), OpenDmsArgs{})) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{}) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), OpenDmsArgs{}) }) mustFinalize(t, &db) } @@ -115,12 +115,12 @@ func TestCLIVersionRecordsLastWriter(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), OpenDmsArgs{})) require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) mustFinalize(t, &db) var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), OpenDmsArgs{})) assert.Equal(t, build.GetInfo().Version, reopened.Data.CLIVersion) assert.Equal(t, 2, reopened.Data.Serial) mustFinalize(t, &reopened) @@ -142,7 +142,7 @@ func TestHeaderOnlyWALDoesNotUpdateCLIVersion(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), OpenDmsArgs{})) assert.Equal(t, "0.1.2", recovered.Data.CLIVersion, "a header-only WAL wrote no state, so the version must not move") assert.Equal(t, 1, recovered.Data.Serial) mustFinalize(t, &recovered) @@ -154,12 +154,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), OpenDmsArgs{})) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), OpenDmsArgs{})) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -175,7 +175,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), OpenDmsArgs{})) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -218,17 +218,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), OpenDmsArgs{})) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), OpenDmsArgs{})) require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", false)) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), OpenDmsArgs{})) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -240,7 +240,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), OpenDmsArgs{})) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -257,7 +257,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), OpenDmsArgs{})) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } @@ -271,13 +271,13 @@ func TestOpenFailureLeavesStateClosed(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte("{not json"), 0o600)) var db DeploymentState - require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) + require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), OpenDmsArgs{})) assert.Empty(t, db.Path) // Once the state file is readable, the same receiver opens without panicking. seed := `{"state_version":2,"cli_version":"0.1.2","lineage":"test-lineage","serial":1,"state":{}}` require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), DMSDeployment{})) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), OpenDmsArgs{})) assert.Equal(t, "test-lineage", db.Data.Lineage) mustFinalize(t, &db) } diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index fe8f55c4e35..cab2e5cc115 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -175,7 +175,7 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s // This plan is not created with the deployment history feature enabled, // so we can safely pass false for withDeploymentHistory. - if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { + if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { return fmt.Errorf("opening migrated state for plan check: %w", err) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index d2667a19dd1..0c62196739f 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 2df4efc22b1..e171df7388a 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index b862fe7366b..61500938735 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -286,12 +286,12 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.DMSDeployment{ID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{ID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } } else { - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.DMSDeployment{}); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } From fb1eb3c54110896de17fdeb6781b845af99550b7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 8 Sep 2026 18:33:20 +0200 Subject: [PATCH 141/164] acceptance/change_assets_dir: run it with recording The plan was captured with trace and &> together, so the trace header landed in the file alongside the JSON and no filter could touch it. Capture it untraced into a raw file and stamp that out into the golden, which is what lets the recording variant run. Co-authored-by: Isaac --- .../change_assets_dir/out.plan.direct.json | 2 -- .../change_assets_dir/out.plan.terraform.json | 2 -- .../quality_monitors/change_assets_dir/out.test.toml | 2 +- .../quality_monitors/change_assets_dir/output.txt | 2 +- .../resources/quality_monitors/change_assets_dir/script | 7 ++++++- .../resources/quality_monitors/change_assets_dir/test.toml | 3 --- 6 files changed, 8 insertions(+), 10 deletions(-) delete mode 100644 acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json index b7ec47c4b22..0471f4d7f85 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json @@ -1,5 +1,3 @@ - ->>> errcode [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.terraform.json b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.terraform.json index abff55e1594..ba464e9a3c2 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.terraform.json +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.terraform.json @@ -1,5 +1,3 @@ - ->>> errcode [CLI] bundle plan -o json { "cli_version": "[CLI_VERSION]", "plan": { diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml index 9b8ddfcd2bd..cef45fe553a 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml @@ -1,3 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index 796ce31264a..15c9c619892 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -17,7 +17,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... Created quality_monitors.monitor1 -Files: 6 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 99e80f5f70a..46ca61613f3 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -1,3 +1,5 @@ +mkdir -p .databricks + SCHEMA_NAME="main.qm_test_${UNIQUE_NAME}" TABLE_NAME="${SCHEMA_NAME}.test_table" @@ -22,7 +24,10 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets_$UNIQUE_NAME" "assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets2_$UNIQUE_NAME" trace errcode $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# Untraced: with &> the trace header lands in the file, leaving it unfilterable. Capture the raw +# plan aside and stamp it out into the golden instead. +errcode $CLI bundle plan -o json &> .databricks/tmp.plan.json +nostamp < .databricks/tmp.plan.json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml deleted file mode 100644 index 2acd446f3cd..00000000000 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py -# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). -EnvMatrix.DMS = [""] From 3aabe82cd6c39aa40849b2897209ca02ece3168f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 00:26:47 +0200 Subject: [PATCH 142/164] bundle/direct/dstate: keep recorded state in memory so metadata.json gets resource ids SaveState stopped populating Data.State under recording, since the service holds the state and the file is a tombstone. But Finalize exports Data.State, and that export is what metadata.Compute reads to put resource ids into metadata.json - so a recorded deploy uploaded metadata with no job or pipeline ids. Keep the entry in memory and let dataForFile empty State before the file is written, as it already does. The tombstone is unchanged; only the in-process export is restored. Caught by bundle/resources/jobs/check-metadata, which greps metadata.json for the ids it just deployed. Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 4003027cd53..deb2b71a902 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -340,8 +340,11 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta DependsOn: dependsOn, } - // A recorded deployment persists through the service, everything else through the WAL. + // A recorded deployment persists through the service, everything else through the WAL. The + // entry is still kept in memory: Finalize exports it for metadata.json and the deploy summary, + // and dataForFile empties State again before the tombstone is written. if db.StorageBackend() == StorageBackendDeploymentMetadataService { + db.Data.State[key] = entry db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) @@ -374,6 +377,7 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgre // Read before the delete: DMS needs the id to say which resource went away. deletedID := db.stateIDs[key] if db.StorageBackend() == StorageBackendDeploymentMetadataService { + delete(db.Data.State, key) delete(db.stateIDs, key) if buf := db.operationBuffer; buf != nil { buf.RecordOperation(ctx, key, inProgress, deletedID, nil) From 2e8baf1a4b374b88355c42df9e7b4e5246d9af57 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 10:28:39 +0200 Subject: [PATCH 143/164] acceptance: commit a DMS plan fixture, run two more tests with recording readplan/lineage-mismatch fed one committed plan to both variants, and a committed plan has to match the shape of the target it is fed to: a recorded target carries the deployment_history feature, and validatePlan rejects a features mismatch before it looks at lineage. There are now two fixtures - plan.direct.json and plan.dms.json - and the script picks by variant, so the lineage error is asserted either way. volume_path_job_ref captured its plan with trace and &> together, so the trace header landed in the file alongside the JSON and nothing could filter it. Captured untraced into a raw file and stamped out into the golden, same as change_assets_dir. Also drops deployment_id and next_version_id from a plan-header assertion and from the nostamp docstring: neither is part of a plan any more. Co-authored-by: Isaac --- acceptance/bin/nostamp.py | 2 +- .../deploy/readplan/lineage-mismatch/out.test.toml | 2 +- .../deploy/readplan/lineage-mismatch/output.txt | 4 ++-- .../{plan.json => plan.direct.json} | 0 .../deploy/readplan/lineage-mismatch/plan.dms.json | 10 ++++++++++ .../bundle/deploy/readplan/lineage-mismatch/script | 14 +++++++++++++- .../deploy/readplan/lineage-mismatch/test.toml | 5 +---- acceptance/bundle/dms/plan-without-dms/output.txt | 6 ++---- acceptance/bundle/dms/plan-without-dms/script | 2 +- acceptance/bundle/invariant/no_drift/test.toml | 2 +- .../volume_path_job_ref/out.plan.direct.json | 2 -- .../volume_path_job_ref/out.test.toml | 2 +- .../resource_deps/volume_path_job_ref/script | 6 +++++- .../resource_deps/volume_path_job_ref/test.toml | 4 ---- 14 files changed, 38 insertions(+), 23 deletions(-) rename acceptance/bundle/deploy/readplan/lineage-mismatch/{plan.json => plan.direct.json} (100%) create mode 100644 acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json diff --git a/acceptance/bin/nostamp.py b/acceptance/bin/nostamp.py index 6dcd7f8cbdd..8a0c222630c 100755 --- a/acceptance/bin/nostamp.py +++ b/acceptance/bin/nostamp.py @@ -13,7 +13,7 @@ == "" is kept (a Terraform state dump carries that for an unstamped job). 2. changes entries keyed "deployment.deployment_id" / "deployment.version_id". 3. a changes object emptied by (2), or already empty. - 4. the recorded plan header fields deployment_id, next_version_id, last_version_id, features. + 4. the recorded plan header field features. """ import argparse diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt b/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt index 30736fe2178..0d00e2f01d1 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/output.txt @@ -1,5 +1,5 @@ + +>>> musterr [CLI] bundle deploy --plan plan.json Warn: Plan was created with CLI version 0.0.0-test but current version is [CLI_VERSION] Error: plan lineage "different-lineage" does not match state lineage ""; the state may have been modified by another process - -Exit code: 1 diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.direct.json similarity index 100% rename from acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json rename to acceptance/bundle/deploy/readplan/lineage-mismatch/plan.direct.json diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json new file mode 100644 index 00000000000..58e072b84f1 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json @@ -0,0 +1,10 @@ +{ + "plan_version": 2, + "cli_version": "0.0.0-test", + "lineage": "different-lineage", + "serial": 0, + "plan": {}, + "features": { + "deployment_history": {} + } +} diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/script b/acceptance/bundle/deploy/readplan/lineage-mismatch/script index d386f293771..e6601a86c5b 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/script +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/script @@ -1,2 +1,14 @@ +# The plan is committed rather than generated, so it must match the shape of the target it is fed +# to: a recorded target's state carries the deployment_history feature, and validatePlan rejects a +# plan whose features differ before it looks at lineage. +plan=plan.direct.json +if [ "$DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY" = "true" ]; then + plan=plan.dms.json +fi + +# Copy to one name so the traced command line is the same in both variants. +cp $plan plan.json + # Try to deploy with mismatched lineage (should fail) -$CLI bundle deploy --plan plan.json +trace musterr $CLI bundle deploy --plan plan.json +rm plan.json diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml index 527d0966f26..c506e6825ce 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml @@ -1,7 +1,4 @@ # Test that --plan with mismatched lineage produces an error EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# The committed plan.json carries no features, while a recorded target has deployment_history, so -# under DMS the feature check fires before the lineage one this test asserts. nostamp cannot help: -# it normalizes output, not the plan being fed in. -EnvMatrixExclude.plan_no_dms = ["DMS=true"] +Ignore = ["plan.json"] diff --git a/acceptance/bundle/dms/plan-without-dms/output.txt b/acceptance/bundle/dms/plan-without-dms/output.txt index 6d19f34d85a..ce1135f624d 100644 --- a/acceptance/bundle/dms/plan-without-dms/output.txt +++ b/acceptance/bundle/dms/plan-without-dms/output.txt @@ -1,10 +1,8 @@ === Generate and save plan WITHOUT DMS recording ->>> jq {features, deployment_id, next_version_id} +>>> jq {features} { - "features": null, - "deployment_id": null, - "next_version_id": null + "features": null } === Enable DMS recording and try to deploy the non-DMS plan diff --git a/acceptance/bundle/dms/plan-without-dms/script b/acceptance/bundle/dms/plan-without-dms/script index cffd78643ae..f1b1dfbbf82 100644 --- a/acceptance/bundle/dms/plan-without-dms/script +++ b/acceptance/bundle/dms/plan-without-dms/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Generate and save plan WITHOUT DMS recording" $CLI bundle plan -o json > tmp.plan.json -trace jq '{features, deployment_id, next_version_id}' < tmp.plan.json +trace jq '{features}' < tmp.plan.json title "Enable DMS recording and try to deploy the non-DMS plan" update_file.py databricks.yml 'bundle:' 'experimental: diff --git a/acceptance/bundle/invariant/no_drift/test.toml b/acceptance/bundle/invariant/no_drift/test.toml index d4c33907d3f..1fece2a12ab 100644 --- a/acceptance/bundle/invariant/no_drift/test.toml +++ b/acceptance/bundle/invariant/no_drift/test.toml @@ -10,6 +10,6 @@ EnvMatrixExclude.dms_state_too_large = ["DMS=true", "INPUT_CONFIG=job_pydabs_100 EnvMatrixExclude.no_secret_with_readplan = ["READPLAN=1", "INPUT_CONFIG=secret.yml.tmpl"] # Emptying a grants node records the operation as a delete with no state, which the deployment -# metadata service rejects on a succeeded UPDATE that leaves the schema (a live resource). This +# metadata service rejects on a succeeded UPDATE action that leaves the grants (a live resource). This # is a known bug tracked by bundle/dms/empty-grants (Cloud=false); skip it under recording here. EnvMatrixExclude.dms_empty_grants_bug = ["DMS=true", "INPUT_CONFIG=schema_empty_grants.yml.tmpl"] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json index 660f9e2f16e..ed268732a02 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json @@ -1,5 +1,3 @@ - ->>> errcode [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml index 27ec2a7fcd6..59b56a2037c 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script index 6c7250cc564..0f2c279fecf 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/script +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -1,10 +1,14 @@ envsubst < databricks.yml.tmpl > databricks.yml +mkdir -p .databricks trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs, .resources.volumes' # The job's data_path default references the volume's computed volume_path. # Record the JSON plan per engine to confirm the reference is resolved into the # planned new_state rather than left as ${resources...}. -trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# Untraced: with &> the trace header lands in the file, leaving it unfilterable. Capture the raw +# plan aside and stamp it out into the golden instead. +errcode $CLI bundle plan -o json &> .databricks/tmp.plan.json +nostamp < .databricks/tmp.plan.json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Deploy and inspect the create-job request: the parameter default sent to # the Jobs API must be the interpolated volume_path. diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml index 10ce9d961a6..b53e5e7fbe8 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml @@ -6,7 +6,3 @@ RecordRequests = true # so the case is unsupported there (see CHANGELOG.md). Direct is the # default engine, so the motivating use case works out of the box. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py -# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). -EnvMatrix.DMS = [""] From 8bcdc810c68877b41e47f6e6402e27032676054f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 10:33:59 +0200 Subject: [PATCH 144/164] acceptance/destroy_without_mgmtperms: run it with recording The deploy runs as the test service principal, but the summary that reads the job id back ran as the user. Under recording the state lives in the deploying identity deployment record, which another identity does not resolve, so the id came back null. Read the summary as the same identity that deployed. The test is about the permission error on destroy, so which identity reads the id back is incidental - the goldens are unchanged, and both variants now run. Co-authored-by: Isaac --- .../permissions/jobs/destroy_without_mgmtperms/test.toml | 3 --- .../destroy_without_mgmtperms/with_permissions/out.test.toml | 2 +- .../jobs/destroy_without_mgmtperms/with_permissions/script | 4 +++- .../without_permissions/out.test.toml | 2 +- .../jobs/destroy_without_mgmtperms/without_permissions/script | 4 +++- 5 files changed, 8 insertions(+), 7 deletions(-) delete mode 100644 acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml deleted file mode 100644 index 4ca61be3f36..00000000000 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Deploy runs as the service principal, later commands as the user - who resolves no deployment of -# its own, so bundle summary reads the job id back null. Recording needs state staged for the reader. -EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml index 95d84941c44..3220a0c8e19 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml @@ -3,4 +3,4 @@ RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script index a7ec688100b..f3bd0d9c85a 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script @@ -22,7 +22,9 @@ cleanup EXIT trace as-test-sp $CLI bundle deploy -job_id=$($CLI bundle summary --output json | jq -r '.resources.jobs.foo.id') +# Read the summary as the deploying identity: under recording the state lives in that +# identity's deployment record, which another identity does not resolve. +job_id=$(as-test-sp $CLI bundle summary --output json | jq -r '.resources.jobs.foo.id') trace $CLI permissions set jobs $job_id --json @take_ownership.json > /dev/null diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml index 95d84941c44..3220a0c8e19 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml @@ -3,4 +3,4 @@ RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script index a7ec688100b..f3bd0d9c85a 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script @@ -22,7 +22,9 @@ cleanup EXIT trace as-test-sp $CLI bundle deploy -job_id=$($CLI bundle summary --output json | jq -r '.resources.jobs.foo.id') +# Read the summary as the deploying identity: under recording the state lives in that +# identity's deployment record, which another identity does not resolve. +job_id=$(as-test-sp $CLI bundle summary --output json | jq -r '.resources.jobs.foo.id') trace $CLI permissions set jobs $job_id --json @take_ownership.json > /dev/null From 96ffaa239f7d7c276e724b5ea7189baebfcce7ac Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 10:39:01 +0200 Subject: [PATCH 145/164] acceptance/change_table_name: run it with recording Same fix as change_assets_dir: the plan was captured with trace and &> together, so the trace header landed in the file alongside the JSON and no filter could touch it. Capture it untraced into a raw file and stamp that out into the golden. Co-authored-by: Isaac --- .../change_table_name/out.plan.direct.json | 2 -- .../quality_monitors/change_table_name/out.test.toml | 2 +- .../resources/quality_monitors/change_table_name/script | 7 ++++++- .../resources/quality_monitors/change_table_name/test.toml | 4 ---- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json index 97e0a65c984..850061e1c60 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json @@ -1,5 +1,3 @@ - ->>> errcode [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml index d73c45e3119..ae5c7bd798f 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml @@ -1,3 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index 6cf47e95c59..b607150e70e 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -1,3 +1,5 @@ +mkdir -p .databricks + SCHEMA_NAME="main.qm_test_${UNIQUE_NAME}" TABLE_NAME="${SCHEMA_NAME}.test_table" @@ -22,7 +24,10 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "table_name: $TABLE_NAME" "table_name: ${TABLE_NAME}_2" trace errcode $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# Untraced: with &> the trace header lands in the file, leaving it unfilterable. Capture the raw +# plan aside and stamp it out into the golden instead. +errcode $CLI bundle plan -o json &> .databricks/tmp.plan.json +nostamp < .databricks/tmp.plan.json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml index 808576f4791..7e03ffedd9a 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml @@ -1,6 +1,2 @@ # this does not work on terraform EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py -# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). -EnvMatrix.DMS = [""] From 8f047e8823cd8b8eacceb4968e54eb167802e01d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 10:44:14 +0200 Subject: [PATCH 146/164] acceptance/lineage-mismatch: drop the redundant Ignore The script removes plan.json itself, so there is nothing left for the harness to flag. Co-authored-by: Isaac --- acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml index c506e6825ce..de0ba18b25f 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/test.toml @@ -1,4 +1,2 @@ # Test that --plan with mismatched lineage produces an error EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -Ignore = ["plan.json"] From 8af91e80cdc8fb6bedd3189395df0bd17876b168 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 11:16:15 +0200 Subject: [PATCH 147/164] bundle/direct/dstate: add GetSerial, always copy features into the plan CalculatePlan copied features only for a recorded deployment, but a plan should carry whatever features its target has - the check on the other side compares both sides regardless. Copying unconditionally needs one way to read the counter for any backend, which GetSerial provides: the recorded version when the bundle records history, since its state file persists no serial of its own, and the state serial otherwise. validatePlan reads the same accessor, so the plan and the check that validates it can no longer disagree about where the number comes from. StateFeatures returns a copy. It handed out the live map, which the caller stamps into a plan that outlives the lock, so the lock was protecting nothing. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 12 ++++-------- bundle/direct/dstate/state.go | 17 ++++++++++++++++- cmd/bundle/utils/process.go | 14 +++++--------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index b75fe07ee46..d81e695fad9 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -145,14 +145,10 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, fmt.Errorf("reading config: %w", err) } - // The plan records the state features it was built against so deploy --plan can reject a plan - // built for a target of a different shape. - if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { - plan.Features = b.StateDB.StateFeatures() - // The state file persists no serial under recording; the version the service recorded is - // what a saved plan is validated against, so carry that instead. - plan.Serial = b.StateDB.VersionID - } + // The plan records the state it was built against so deploy --plan can reject a plan built for a + // target of a different shape or a state that has moved on since. + plan.Features = b.StateDB.StateFeatures() + plan.Serial = b.StateDB.GetSerial() b.Plan = plan diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index deb2b71a902..7187801a196 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io/fs" + "maps" "os" "path/filepath" "slices" @@ -432,7 +433,21 @@ func (db *DeploymentState) StateCLIVersion() string { func (db *DeploymentState) StateFeatures() map[string]struct{} { db.mu.Lock() defer db.mu.Unlock() - return db.Data.Features + // Copied: the caller stamps this into a plan that outlives the lock, so handing out the live map + // would let it be mutated behind the state's back. + return maps.Clone(db.Data.Features) +} + +// GetSerial reports the counter a saved plan is validated against: the recorded version for a +// deployment that records history, since its state file persists no serial of its own, and the +// state serial otherwise. The two mean the same thing and advance together. +func (db *DeploymentState) GetSerial() int { + if db.StorageBackend() == StorageBackendDeploymentMetadataService { + return db.VersionID + } + db.mu.Lock() + defer db.mu.Unlock() + return db.Data.Serial } // StorageBackend reports where this deployment's state lives - the source of truth for whether the diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 61500938735..d793cc8a400 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -481,15 +481,11 @@ func validatePlan(b *bundle.Bundle, plan *deployplan.Plan) error { return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) } - // Recorded deployments keep this counter in VersionID, everything else in the state serial; the - // two agree after the same sequence of deploys. A plan built against an earlier - // recorded version gets the clearer message; anything else falls through to the serial check. - expected := stateDB.Data.Serial - if stateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { - expected = stateDB.VersionID - if plan.Serial < expected { - return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, expected) - } + // A plan built against an earlier recorded version gets the clearer message; anything else falls + // through to the serial comparison. + expected := stateDB.GetSerial() + if stateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService && plan.Serial < expected { + return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, expected) } if plan.Serial != expected { return fmt.Errorf("plan serial %d does not match state serial %d; the state has been modified since the plan was created. Please run 'bundle plan' again", plan.Serial, expected) From 7a4424a0d0d0bb8fc22abf89f82bbe53ca875836 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 11:30:36 +0200 Subject: [PATCH 148/164] bundle/direct/dstate: GetSerial, refuse a leftover WAL, tidy DeleteState CalculatePlan copies features unconditionally now - a plan should carry whatever features its target has, and the check on the other side compares both sides regardless. That needs one way to read the counter for any backend, which GetSerial provides: the recorded version when the bundle records history, since its state file persists no serial of its own, and the state serial otherwise. validatePlan reads the same accessor, so it no longer branches on the backend at all and one equality comparison covers both. StateFeatures returns a copy so callers cannot modify the state feature set. A recording open now refuses a leftover WAL rather than deleting it. A recorded deployment writes none, so one can only have come from a deploy that did not record - and that file is the only record of its writes. DeleteState no longer clears Data.State, matching the direct path, and the shared stateIDs delete is hoisted out of both branches. Addresses review feedback. Co-authored-by: Isaac --- .../dms/existing-state/out.requests.txt | 108 ++++++++++++++++++ .../bundle/dms/existing-state/output.txt | 39 +------ acceptance/bundle/dms/existing-state/script | 8 +- acceptance/bundle/dms/stale-plan/output.txt | 2 +- bundle/direct/dstate/state.go | 28 ++--- cmd/bundle/utils/process.go | 5 - 6 files changed, 129 insertions(+), 61 deletions(-) create mode 100644 acceptance/bundle/dms/existing-state/out.requests.txt diff --git a/acceptance/bundle/dms/existing-state/out.requests.txt b/acceptance/bundle/dms/existing-state/out.requests.txt new file mode 100644 index 00000000000..dc7e060d33a --- /dev/null +++ b/acceptance/bundle/dms/existing-state/out.requests.txt @@ -0,0 +1,108 @@ +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.deployment.json" + } +} +{ + "method": "POST", + "path": "/telemetry-ext", + "body": { + "uploadTime": [UNIX_TIME_MILLIS][0], + "items": [], + "protoLogs": [ + "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[UUID]\",\"version\":\"[CLI_VERSION]\",\"command\":\"bundle_deploy\",\"operating_system\":\"darwin\",\"execution_time_ms\":8,\"exit_code\":1},\"bundle_deploy_event\":{\"bundle_uuid\":\"[UUID]\",\"deployment_id\":\"[UUID]\",\"error_message\":\"this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources\\n\\nTo record this bundle's history, start it over as a new deployment:\\n 1. remove experimental.deployment_history from your bundle configuration\\n 2. run \\\"databricks bundle destroy\\\" to delete the existing resources\\n 3. add experimental.deployment_history back and deploy again\",\"resource_count\":1,\"resource_job_count\":1,\"resource_pipeline_count\":0,\"resource_model_count\":0,\"resource_experiment_count\":0,\"resource_model_serving_endpoint_count\":0,\"resource_registered_model_count\":0,\"resource_quality_monitor_count\":0,\"resource_schema_count\":0,\"resource_volume_count\":0,\"resource_cluster_count\":0,\"resource_dashboard_count\":0,\"resource_app_count\":0,\"resources_metadata\":{\"resources\":[{\"resource_type\":\"jobs\",\"count\":1}]},\"experimental\":{\"configuration_file_count\":1,\"variable_count\":0,\"complex_variable_count\":0,\"lookup_variable_count\":0,\"target_count\":1,\"bool_values\":[{\"key\":\"local.cache.attempt\",\"value\":true},{\"key\":\"local.cache.hit\",\"value\":true},{\"key\":\"experimental.use_legacy_run_as\",\"value\":false},{\"key\":\"run_as_set\",\"value\":false},{\"key\":\"presets_name_prefix_is_set\",\"value\":false}],\"bundle_mode\":\"TYPE_UNSPECIFIED\",\"workspace_artifact_path_type\":\"WORKSPACE_FILE_SYSTEM\",\"bundle_mutator_execution_time_ms\":[{\"key\":\"phases.Initialize\",\"value\":4},{\"key\":\"resourcemutator.(processStaticResources)\",\"value\":1}],\"upload_file_count\":0}}}}}" + ] + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.deployment.json" + } +} +{ + "method": "POST", + "path": "/telemetry-ext", + "body": { + "uploadTime": [UNIX_TIME_MILLIS][1], + "items": [], + "protoLogs": [ + "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[UUID]\",\"version\":\"[CLI_VERSION]\",\"command\":\"bundle_deploy\",\"operating_system\":\"darwin\",\"execution_time_ms\":7,\"exit_code\":1},\"bundle_deploy_event\":{\"bundle_uuid\":\"[UUID]\",\"deployment_id\":\"[UUID]\",\"error_message\":\"unexpected WAL file found at [REDACTED_PATH]: this deployment records deployment history, which does not write one\",\"resource_count\":1,\"resource_job_count\":1,\"resource_pipeline_count\":0,\"resource_model_count\":0,\"resource_experiment_count\":0,\"resource_model_serving_endpoint_count\":0,\"resource_registered_model_count\":0,\"resource_quality_monitor_count\":0,\"resource_schema_count\":0,\"resource_volume_count\":0,\"resource_cluster_count\":0,\"resource_dashboard_count\":0,\"resource_app_count\":0,\"resources_metadata\":{\"resources\":[{\"resource_type\":\"jobs\",\"count\":1}]},\"experimental\":{\"configuration_file_count\":1,\"variable_count\":0,\"complex_variable_count\":0,\"lookup_variable_count\":0,\"target_count\":1,\"bool_values\":[{\"key\":\"local.cache.attempt\",\"value\":true},{\"key\":\"local.cache.hit\",\"value\":true},{\"key\":\"experimental.use_legacy_run_as\",\"value\":false},{\"key\":\"run_as_set\",\"value\":false},{\"key\":\"presets_name_prefix_is_set\",\"value\":false}],\"bundle_mode\":\"TYPE_UNSPECIFIED\",\"workspace_artifact_path_type\":\"WORKSPACE_FILE_SYSTEM\",\"bundle_mutator_execution_time_ms\":[{\"key\":\"phases.Initialize\",\"value\":4},{\"key\":\"resourcemutator.(processStaticResources)\",\"value\":1}],\"upload_file_count\":0}}}}}" + ] + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json" + } +} diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 1b4a4f1eba7..d00dc54e39a 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -47,43 +47,14 @@ To record this bundle's history, start it over as a new deployment: 3. add experimental.deployment_history back and deploy again -=== A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands +=== A leftover local WAL is refused: a recorded deployment writes none, so one can only have come from a deploy that did not record >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources - -To record this bundle's history, start it over as a new deployment: - 1. remove experimental.deployment_history from your bundle configuration - 2. run "databricks bundle destroy" to delete the existing resources - 3. add experimental.deployment_history back and deploy again +Error: unexpected WAL file found at [TEST_TMP_DIR]/.databricks/bundle/default/resources.json.wal: this deployment records deployment history, which does not write one -=== Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed +=== Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone >>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.one - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default - -Destroy: 1 deleted - ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... -Created jobs.one -Files: 6 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> print_requests.py --dms //api/2.0/bundle --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.one", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +Error: reading state from [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: WAL recovery failed: WAL lineage ("stale") does not match state lineage ("[UUID]") ->>> find.py resources[.]json[.]wal --expect 0 ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 2, - "features": { - "deployment_history": {} - } -} +Exit code: 1 diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index df6659f1ff0..8191767b171 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -14,16 +14,14 @@ title "The check is against the remote state, not the local cache: wiping .datab rm -rf .databricks musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy -title "A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands" -# The WAL header carries the deployment_history feature: if it were replayed, the -# feature would merge into the state. The refusal below proves it is discarded, not read. +title "A leftover local WAL is refused: a recorded deployment writes none, so one can only have come from a deploy that did not record" mkdir -p .databricks/bundle/default printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy -title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed" +title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone" trace $CLI bundle destroy --auto-approve -printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal +rm -f .databricks/bundle/default/resources.json.wal trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --oneline trace find.py 'resources[.]json[.]wal' --expect 0 diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 076fa5c46b5..2ce611c6247 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -78,5 +78,5 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Replaying the older plan is rejected: same deployment, outdated version >>> musterr [CLI] bundle deploy --plan tmp.plan2.json -Error: this plan was built against version 1 but the deployment has recorded version 2; run 'bundle plan' again +Error: plan serial 1 does not match state serial 2; the state has been modified since the plan was created. Please run 'bundle plan' again diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 7187801a196..8f5fd8ad05b 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -375,20 +375,18 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgre if db.Data.State == nil { return nil } - // Read before the delete: DMS needs the id to say which resource went away. + // Read before the delete below: DMS needs the id to say which resource went away. deletedID := db.stateIDs[key] + + // A recorded deployment persists through the service, everything else through the WAL. if db.StorageBackend() == StorageBackendDeploymentMetadataService { - delete(db.Data.State, key) - delete(db.stateIDs, key) if buf := db.operationBuffer; buf != nil { buf.RecordOperation(ctx, key, inProgress, deletedID, nil) } - return nil - } - - if err := appendJSONLine(db.walFile, WALEntry{Key: key}); err != nil { + } else if err := appendJSONLine(db.walFile, WALEntry{Key: key}); err != nil { return err } + delete(db.stateIDs, key) return nil } @@ -433,8 +431,7 @@ func (db *DeploymentState) StateCLIVersion() string { func (db *DeploymentState) StateFeatures() map[string]struct{} { db.mu.Lock() defer db.mu.Unlock() - // Copied: the caller stamps this into a plan that outlives the lock, so handing out the live map - // would let it be mutated behind the state's back. + // Copied so callers cannot modify the state's set of features. return maps.Clone(db.Data.Features) } @@ -442,11 +439,11 @@ func (db *DeploymentState) StateFeatures() map[string]struct{} { // deployment that records history, since its state file persists no serial of its own, and the // state serial otherwise. The two mean the same thing and advance together. func (db *DeploymentState) GetSerial() int { + db.mu.Lock() + defer db.mu.Unlock() if db.StorageBackend() == StorageBackendDeploymentMetadataService { return db.VersionID } - db.mu.Lock() - defer db.mu.Unlock() return db.Data.Serial } @@ -579,11 +576,10 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe default: // WAL exists switch { case recording: - // A recording open discards a leftover WAL - the service owns the resources, so a local - // log from a crash or a declined deploy is irrelevant, even when recording is refused below. - if err := os.Remove(walPath); err != nil { - return fmt.Errorf("removing WAL file %s: %w", walPath, err) - } + // A recorded deployment writes no WAL, so finding one means this state was written by a + // deployment that did not record history. Refuse rather than discard it: the file is the + // only record of that deploy's writes, and recording is refused below anyway. + return fmt.Errorf("unexpected WAL file found at %s: this deployment records deployment history, which does not write one", walPath) case bool(withRecovery): if err := db.replayWAL(ctx); err != nil { return fmt.Errorf("reading state from %s: %w", path, err) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index d793cc8a400..73544ce9623 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -481,12 +481,7 @@ func validatePlan(b *bundle.Bundle, plan *deployplan.Plan) error { return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) } - // A plan built against an earlier recorded version gets the clearer message; anything else falls - // through to the serial comparison. expected := stateDB.GetSerial() - if stateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService && plan.Serial < expected { - return fmt.Errorf("this plan was built against version %d but the deployment has recorded version %d; run 'bundle plan' again", plan.Serial, expected) - } if plan.Serial != expected { return fmt.Errorf("plan serial %d does not match state serial %d; the state has been modified since the plan was created. Please run 'bundle plan' again", plan.Serial, expected) } From fe3f963906c70601d3a54cdaeceba5f899259a6e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 11:36:31 +0200 Subject: [PATCH 149/164] acceptance/dms/existing-state: clear the stale WAL before destroy A recording open refuses a leftover WAL now instead of deleting it, so the file survives the refusal. The destroy that follows runs without recording, so it takes the recovery path and died replaying that stale WAL - which aborted the script and left its recorded requests behind, committed by accident as a golden. Remove the WAL first and untrack that file. Co-authored-by: Isaac --- .../dms/existing-state/out.requests.txt | 108 ------------------ .../bundle/dms/existing-state/output.txt | 28 ++++- acceptance/bundle/dms/existing-state/script | 8 +- 3 files changed, 33 insertions(+), 111 deletions(-) delete mode 100644 acceptance/bundle/dms/existing-state/out.requests.txt diff --git a/acceptance/bundle/dms/existing-state/out.requests.txt b/acceptance/bundle/dms/existing-state/out.requests.txt deleted file mode 100644 index dc7e060d33a..00000000000 --- a/acceptance/bundle/dms/existing-state/out.requests.txt +++ /dev/null @@ -1,108 +0,0 @@ -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.deployment.json" - } -} -{ - "method": "POST", - "path": "/telemetry-ext", - "body": { - "uploadTime": [UNIX_TIME_MILLIS][0], - "items": [], - "protoLogs": [ - "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[UUID]\",\"version\":\"[CLI_VERSION]\",\"command\":\"bundle_deploy\",\"operating_system\":\"darwin\",\"execution_time_ms\":8,\"exit_code\":1},\"bundle_deploy_event\":{\"bundle_uuid\":\"[UUID]\",\"deployment_id\":\"[UUID]\",\"error_message\":\"this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources\\n\\nTo record this bundle's history, start it over as a new deployment:\\n 1. remove experimental.deployment_history from your bundle configuration\\n 2. run \\\"databricks bundle destroy\\\" to delete the existing resources\\n 3. add experimental.deployment_history back and deploy again\",\"resource_count\":1,\"resource_job_count\":1,\"resource_pipeline_count\":0,\"resource_model_count\":0,\"resource_experiment_count\":0,\"resource_model_serving_endpoint_count\":0,\"resource_registered_model_count\":0,\"resource_quality_monitor_count\":0,\"resource_schema_count\":0,\"resource_volume_count\":0,\"resource_cluster_count\":0,\"resource_dashboard_count\":0,\"resource_app_count\":0,\"resources_metadata\":{\"resources\":[{\"resource_type\":\"jobs\",\"count\":1}]},\"experimental\":{\"configuration_file_count\":1,\"variable_count\":0,\"complex_variable_count\":0,\"lookup_variable_count\":0,\"target_count\":1,\"bool_values\":[{\"key\":\"local.cache.attempt\",\"value\":true},{\"key\":\"local.cache.hit\",\"value\":true},{\"key\":\"experimental.use_legacy_run_as\",\"value\":false},{\"key\":\"run_as_set\",\"value\":false},{\"key\":\"presets_name_prefix_is_set\",\"value\":false}],\"bundle_mode\":\"TYPE_UNSPECIFIED\",\"workspace_artifact_path_type\":\"WORKSPACE_FILE_SYSTEM\",\"bundle_mutator_execution_time_ms\":[{\"key\":\"phases.Initialize\",\"value\":4},{\"key\":\"resourcemutator.(processStaticResources)\",\"value\":1}],\"upload_file_count\":0}}}}}" - ] - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.deployment.json" - } -} -{ - "method": "POST", - "path": "/telemetry-ext", - "body": { - "uploadTime": [UNIX_TIME_MILLIS][1], - "items": [], - "protoLogs": [ - "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[UUID]\",\"version\":\"[CLI_VERSION]\",\"command\":\"bundle_deploy\",\"operating_system\":\"darwin\",\"execution_time_ms\":7,\"exit_code\":1},\"bundle_deploy_event\":{\"bundle_uuid\":\"[UUID]\",\"deployment_id\":\"[UUID]\",\"error_message\":\"unexpected WAL file found at [REDACTED_PATH]: this deployment records deployment history, which does not write one\",\"resource_count\":1,\"resource_job_count\":1,\"resource_pipeline_count\":0,\"resource_model_count\":0,\"resource_experiment_count\":0,\"resource_model_serving_endpoint_count\":0,\"resource_registered_model_count\":0,\"resource_quality_monitor_count\":0,\"resource_schema_count\":0,\"resource_volume_count\":0,\"resource_cluster_count\":0,\"resource_dashboard_count\":0,\"resource_app_count\":0,\"resources_metadata\":{\"resources\":[{\"resource_type\":\"jobs\",\"count\":1}]},\"experimental\":{\"configuration_file_count\":1,\"variable_count\":0,\"complex_variable_count\":0,\"lookup_variable_count\":0,\"target_count\":1,\"bool_values\":[{\"key\":\"local.cache.attempt\",\"value\":true},{\"key\":\"local.cache.hit\",\"value\":true},{\"key\":\"experimental.use_legacy_run_as\",\"value\":false},{\"key\":\"run_as_set\",\"value\":false},{\"key\":\"presets_name_prefix_is_set\",\"value\":false}],\"bundle_mode\":\"TYPE_UNSPECIFIED\",\"workspace_artifact_path_type\":\"WORKSPACE_FILE_SYSTEM\",\"bundle_mutator_execution_time_ms\":[{\"key\":\"phases.Initialize\",\"value\":4},{\"key\":\"resourcemutator.(processStaticResources)\",\"value\":1}],\"upload_file_count\":0}}}}}" - ] - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json" - } -} diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index d00dc54e39a..5fe12f71a85 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -54,7 +54,31 @@ Error: unexpected WAL file found at [TEST_TMP_DIR]/.databricks/bundle/default/re === Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone >>> [CLI] bundle destroy --auto-approve -Error: reading state from [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: WAL recovery failed: WAL lineage ("stale") does not match state lineage ("[UUID]") +The following resources will be deleted: + delete resources.jobs.one +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default -Exit code: 1 +Destroy: 1 deleted + +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Created jobs.one +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.one", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +>>> find.py resources[.]json[.]wal --expect 0 + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 2, + "features": { + "deployment_history": {} + } +} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 8191767b171..64cff3194e3 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -20,9 +20,15 @@ printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone" -trace $CLI bundle destroy --auto-approve +# Clear the stale WAL first: it was refused, not deleted, and destroy runs without recording, so it +# would otherwise try to replay it and fail on the lineage mismatch. rm -f .databricks/bundle/default/resources.json.wal +trace $CLI bundle destroy --auto-approve trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --oneline trace find.py 'resources[.]json[.]wal' --expect 0 trace jq '{state_version, features}' .databricks/bundle/default/resources.json + +# The recorded requests are asserted above via print_requests.py; drop whatever the run added after +# that (telemetry, and the state probes of the non-recording variant) so nothing is left to compare. +rm -f out.requests.txt From 0aa6c53d43a14be749137da8f372fd508546bea3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 11:48:13 +0200 Subject: [PATCH 150/164] libs/dms: compare workspace info with structdiff.IsEqual, note the sequence id TODO sameWorkspaceInfo copied both structs to blank their ForceSendFields before a DeepEqual, which only works while WorkspaceInfo has no nested structs - true today, not guaranteed. structdiff.IsEqual handles ForceSendFields recursively and needs no copy; its doc comment records the same reason DeepEqual is wrong for SDK structs. Also records a TODO on the operation buffer's sequence ids: they guard against an earlier update overwriting a later one, which cannot happen when updates for a resource are sent one at a time from a single goroutine. UpdateOperation requires them today, so the client cannot simply stop sending them. Addresses review feedback. Co-authored-by: Isaac --- libs/dms/metadata.go | 13 +++++-------- libs/dms/operation_buffer.go | 5 +++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libs/dms/metadata.go b/libs/dms/metadata.go index bb67bd2a65c..ff340c1a187 100644 --- a/libs/dms/metadata.go +++ b/libs/dms/metadata.go @@ -2,10 +2,10 @@ package dms import ( "fmt" - "reflect" "strconv" "strings" + "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -34,17 +34,14 @@ type Metadata struct { // among them: the service derives the deployment's from the version that carried it. var deploymentFields = []string{"display_name", "target_name", "deployment_mode", "workspace_info"} -// sameWorkspaceInfo compares the paths alone. The SDK records which fields a response carried in -// ForceSendFields, so a record read back never deep-equals one built here, and comparing the -// structs whole would report every run as a change. +// sameWorkspaceInfo compares the paths alone. structdiff.IsEqual rather than reflect.DeepEqual: the +// SDK records which fields a response carried in ForceSendFields, so a record read back never +// deep-equals one built here, and every run would report a change. func sameWorkspaceInfo(want, current *bundledeployments.WorkspaceInfo) bool { if want == nil || current == nil { return want == nil && current == nil } - - a, b := *want, *current - a.ForceSendFields, b.ForceSendFields = nil, nil - return reflect.DeepEqual(a, b) + return structdiff.IsEqual(*want, *current) } // deployment renders the metadata the deployment owns. diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go index 11333404336..c4fbb8e379e 100644 --- a/libs/dms/operation_buffer.go +++ b/libs/dms/operation_buffer.go @@ -32,6 +32,11 @@ type OperationBuffer struct { // sequenceIDs holds the token the last update for a resource returned. A resource absent // from it has only what staging left, so its first update sends that. Unguarded: run is the // only goroutine that writes, one update at a time. + // + // TODO: revisit and possibly deprecate. Sequence ids guard against an earlier update + // overwriting a later one, which cannot happen here - updates for a resource are sent one at a + // time from a single goroutine. UpdateOperation requires them today, so the client cannot + // simply stop sending them. sequenceIDs map[string]string // mu guards the fields below. From 3aac3abb0863f6d66b2bcefc16c99f0b96022113 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 11:51:12 +0200 Subject: [PATCH 151/164] libs/dms: shorten the sameWorkspaceInfo doc comment Co-authored-by: Isaac --- libs/dms/metadata.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libs/dms/metadata.go b/libs/dms/metadata.go index ff340c1a187..e9a4a10a8e0 100644 --- a/libs/dms/metadata.go +++ b/libs/dms/metadata.go @@ -34,9 +34,7 @@ type Metadata struct { // among them: the service derives the deployment's from the version that carried it. var deploymentFields = []string{"display_name", "target_name", "deployment_mode", "workspace_info"} -// sameWorkspaceInfo compares the paths alone. structdiff.IsEqual rather than reflect.DeepEqual: the -// SDK records which fields a response carried in ForceSendFields, so a record read back never -// deep-equals one built here, and every run would report a change. +// sameWorkspaceInfo reports whether two workspace infos carry the same paths. func sameWorkspaceInfo(want, current *bundledeployments.WorkspaceInfo) bool { if want == nil || current == nil { return want == nil && current == nil From 2467a05a7f9c0b52ba7320abe820016d7e29135b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 12:03:47 +0200 Subject: [PATCH 152/164] Carry the deployment version as an int, add getOperationBuffer The service reports last_version_id as a string. Parse it once where the deployment is fetched, so OpenDmsArgs.LastVersionID, DeploymentHistory.LatestVersionID and everything below carry a number instead of passing the string around and parsing it again in Open. bundle summary now reports latest_version_id as a number. OpenDmsArgs.ID becomes DeploymentID: easier to grep than an overloaded ID. applyDMSState asserts the file it is filling in is the empty shell a recorded deployment writes, and says "fills in" rather than "replaces", since there is nothing to replace. Its test cases follow: a file carrying resources is now the refusal case rather than a merge. getOperationBuffer replaces three hand-rolled lock-read-unlock sequences. RecordFailure dereferenced the buffer relying on a comment that a version is always open; it checks now. Also trims the DeploymentID and VersionID comments; VersionID leads with what it is - the DMS counterpart of Data.Serial, owned by the service, so it never reaches resources.json. Addresses review feedback. Co-authored-by: Isaac --- acceptance/bundle/dms/no-drift/output.txt | 4 +- bundle/config/deployment.go | 4 +- bundle/direct/dstate/dms.go | 12 +++-- bundle/direct/dstate/dms_test.go | 29 ++++++----- bundle/direct/dstate/state.go | 63 ++++++++++------------- cmd/bundle/utils/process.go | 20 +++---- 6 files changed, 66 insertions(+), 66 deletions(-) diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index a8e14bab386..a5e26c6fa8f 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -24,7 +24,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle summary -o json { "deployment_id": "[NUMID]", - "latest_version_id": "1" + "latest_version_id": 1 } === The plan is a no-op now and records the deployment_history feature @@ -157,7 +157,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> [CLI] bundle summary -o json { "deployment_id": "[NUMID]", - "latest_version_id": "2" + "latest_version_id": 2 } === The plan shows the updated pipeline at serial 2 with the deployment_history feature diff --git a/bundle/config/deployment.go b/bundle/config/deployment.go index b59d1b1da02..48218607bbb 100644 --- a/bundle/config/deployment.go +++ b/bundle/config/deployment.go @@ -21,6 +21,6 @@ type DeploymentHistory struct { DeploymentID string `json:"deployment_id,omitempty"` // LatestVersionID is the most recent version recorded for the deployment. It is - // unset when the deployment exists but has no version yet. - LatestVersionID string `json:"latest_version_id,omitempty"` + // zero when the deployment exists but has no version yet. + LatestVersionID int `json:"latest_version_id,omitempty"` } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index c0d4ffb2a7f..3bab77e4ed3 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -16,11 +16,15 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// applyDMSState replaces the file-derived resource state with what DMS recorded. DMS owns the -// resource set outright, so this runs on every recorded open and the file's copy is never read -// back as the truth: an empty set means the service tracks nothing, whether because the deploy -// created nothing or because the deployment is gone. The caller holds db.mu. +// applyDMSState fills in the resource state from what DMS recorded. A recorded deployment's file is +// an empty shell - the service owns the resource set outright - so there is nothing to merge with: +// an empty set means the service tracks nothing, whether because the deploy created nothing or +// because the deployment is gone. The caller holds db.mu. func (db *DeploymentState) applyDMSState(recorded []dms.Resource) error { + if len(db.Data.State) > 0 { + return fmt.Errorf("internal error: state file for a recorded deployment carries %d resources, expected none", len(db.Data.State)) + } + // Built first and assigned together, so a malformed envelope leaves the state as it was // rather than half replaced. resources := make(map[string]ResourceEntry, len(recorded)) diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 8edcf563ee8..ab4e1bc017f 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -44,30 +44,31 @@ func TestApplyDMSState(t *testing.T) { "resources.pipelines.bar": {ID: "456"}, }, }, - { - name: "what DMS holds replaces what the file loaded", - existing: fileState, - recorded: []dms.Resource{{Key: "resources.jobs.foo", ID: "dms-id", State: `{"state":{"name":"from-dms"}}`}}, - want: map[string]ResourceEntry{"resources.jobs.foo": {ID: "dms-id", State: json.RawMessage(`{"name":"from-dms"}`)}}, - }, { // Nil is what a list holding nothing returns, and it means a successful deploy of // nothing rather than missing data. - name: "nothing recorded empties the state", - existing: fileState, - want: map[string]ResourceEntry{}, + name: "nothing recorded leaves the state empty", + want: map[string]ResourceEntry{}, }, { - // The good resource comes first, so the error lands mid-way: what the file loaded - // has to survive whole rather than end up half replaced. - name: "a malformed envelope leaves the state as it was", - existing: fileState, + // The good resource comes first, so the error lands mid-way: the state has to be left + // alone rather than end up half filled. + name: "a malformed envelope leaves the state alone", recorded: []dms.Resource{ {Key: "resources.jobs.ok", ID: "999", State: `{"state":{"name":"ok"}}`}, {Key: "resources.jobs.foo", ID: "123", State: "not json"}, }, wantErr: "interpreting state recorded for resources.jobs.foo", - want: fileState, + want: nil, + }, + { + // A recorded deployment's file is an empty shell, so resources in it mean the state was + // not the one recording wrote. + name: "resources loaded from the file are refused", + existing: fileState, + recorded: []dms.Resource{{Key: "resources.jobs.foo", ID: "dms-id", State: `{"state":{"name":"from-dms"}}`}}, + wantErr: "carries 1 resources, expected none", + want: fileState, }, } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 8f5fd8ad05b..e44569d8bcb 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -12,7 +12,6 @@ import ( "os" "path/filepath" "slices" - "strconv" "strings" "sync" @@ -145,30 +144,24 @@ type DeploymentState struct { // completion after an explicit one does nothing. versionCompleted bool - // DeploymentID is the recorded deployment's id, from the service at Open - // (InitializeOperationBuffer updates it to the id a first deploy creates, which Open cannot - // know). CalculatePlan reads it to stamp the plan; CompleteVersion reads it after Finalize's - // reset - so, like operationBuffer and dmsClient, it must survive reset. + // DeploymentID is the recorded deployment's id. Set at Open, or by the first deploy that creates + // the deployment, which Open cannot know about. DeploymentID string - // VersionID is the source of truth for the deployment version: every version operation reads it - // - which version to create next, which one it follows, what the plan stamps, and what a saved - // plan is validated against. It holds the last version the service recorded, and once this run - // creates one, that new version. - // - // It sits outside Data because Finalize resets Data and CompleteVersion runs after that. + // VersionID is the DMS counterpart of Data.Serial - same meaning, but owned by the service + // rather than the file, so it lives outside Data and is never written to resources.json. VersionID int } // OpenDmsArgs identifies the recorded deployment Open reads from. The zero value means the // bundle does not record deployment history, or no deployment exists for it yet. type OpenDmsArgs struct { - // ID is the deployment's server-minted id. - ID string + // DeploymentID is the deployment's server-minted id. + DeploymentID string - // LastVersionID is the most recent version the service has recorded. The state serial is set - // from it at Open: the service owns the version number, and the serial only tracks it. - LastVersionID string + // LastVersionID is the most recent version the service has recorded, zero when it has none. + // Open takes the state's version from it: the service owns the number. + LastVersionID int } type Header struct { @@ -227,6 +220,13 @@ func (db *DeploymentState) InitializeOperationBuffer(ctx context.Context, deploy db.VersionID = versionID } +// getOperationBuffer returns the buffer for the open version, nil when no version is open. +func (db *DeploymentState) getOperationBuffer() *dms.OperationBuffer { + db.mu.Lock() + defer db.mu.Unlock() + return db.operationBuffer +} + // RecordingError reports whether recording state writes to the service has failed, so the apply // stops touching resources once the service is no longer keeping up. Nil when the bundle does not // record deployment history or recording is healthy. @@ -234,12 +234,11 @@ func (db *DeploymentState) RecordingError() error { if db.StorageBackend() != StorageBackendDeploymentMetadataService { return nil } - db.mu.Lock() - defer db.mu.Unlock() - if db.operationBuffer == nil { + buf := db.getOperationBuffer() + if buf == nil { return nil } - return db.operationBuffer.Err() + return buf.Err() } // CompleteVersion marks the recorded version done, reporting whether it completed here. A no-op @@ -302,11 +301,11 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e } } - // A failure is recorded only while a version is open (after InitializeOperationBuffer), so the buffer is set. - db.mu.Lock() - buf := db.operationBuffer - db.mu.Unlock() - buf.RecordFailure(resourceKey, resourceID, recorded, cause) + // A failure is normally recorded only while a version is open, so the buffer is set; nil means + // no version was created, and there is nothing to record against. + if buf := db.getOperationBuffer(); buf != nil { + buf.RecordFailure(resourceKey, resourceID, recorded, cause) + } } func NewDatabase(lineage string, serial int) Database { @@ -623,10 +622,10 @@ To record this bundle's history, start it over as a new deployment: return err } db.dmsClient = client - db.DeploymentID = dmsDeployment.ID + db.DeploymentID = dmsDeployment.DeploymentID - if dmsDeployment.ID != "" { - resources, err := db.dmsClient.ListResources(ctx, dmsDeployment.ID) + if dmsDeployment.DeploymentID != "" { + resources, err := db.dmsClient.ListResources(ctx, dmsDeployment.DeploymentID) if err != nil { return err } @@ -637,13 +636,7 @@ To record this bundle's history, start it over as a new deployment: // The service owns the version number, so it is read from the deployment rather than from // the state file, which persists none. No deployment yet means no versions, hence zero. - if dmsDeployment.LastVersionID != "" { - last, err := strconv.Atoi(dmsDeployment.LastVersionID) - if err != nil { - return fmt.Errorf("failed to parse last_version_id %q: %w", dmsDeployment.LastVersionID, err) - } - db.VersionID = last - } + db.VersionID = dmsDeployment.LastVersionID } if withWrite { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 73544ce9623..5753c8efeef 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -259,15 +259,17 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // the plan carries them. version_id is always known (last recorded + 1); deployment_id // does not exist until a first deploy creates it, so it is left off here and the deploy // phase stamps the created id. - lastVersionID := "" - if dmsDeployment != nil { - lastVersionID = dmsDeployment.LastVersionId - } - nextVersion, verr := dms.NextVersion(lastVersionID) - if verr != nil { - logdiag.LogError(ctx, verr) - return b, stateDesc, root.ErrAlreadyPrinted + // The service reports the version as a string; parse it here so everything below + // carries a number. + lastVersionID := 0 + if dmsDeployment != nil && dmsDeployment.LastVersionId != "" { + lastVersionID, err = strconv.Atoi(dmsDeployment.LastVersionId) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to parse last_version_id %q: %w", dmsDeployment.LastVersionId, err)) + return b, stateDesc, root.ErrAlreadyPrinted + } } + nextVersion := lastVersionID + 1 muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} if dmsDeploymentID != "" { bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { @@ -286,7 +288,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{ID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } From 7c6a157ecdba32ba6cc0e54bc997a3aa48350a34 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 12:27:25 +0200 Subject: [PATCH 153/164] bundle/direct/dstate: drop StorageBackend, read the features instead The state's features already say whether a deployment records history, so the StorageBackend type, its two constants and the field mirroring them were a second copy of that fact. IsDeploymentMetadataService is a map lookup on Data.Features, with an unlocked twin for the callers that already hold db.mu. Two places cannot use it, and both are after Finalize resets Data, so the features are no longer there to read: CompleteVersion and the deferred calls to it in the deploy and destroy phases. CompleteVersion gates on the operation buffer instead, which exists only for a recorded deployment and survives the reset, and its callers no longer gate at all - it already no-ops when no version was created. The old field survived the reset, which is why this was not visible until the switch. Co-authored-by: Isaac --- bundle/direct/apply.go | 2 +- bundle/direct/dstate/state.go | 61 ++++++++++++++--------------------- bundle/phases/deploy.go | 14 ++++---- bundle/phases/destroy.go | 9 +++--- 4 files changed, 36 insertions(+), 50 deletions(-) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index f04b56e8b72..21822cd0fb3 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -259,7 +259,7 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, log.Warnf(ctx, "Ignoring permission error when deleting %s id=%s: %s", d.ResourceKey, oldID, err) } else if d.deleteConfirmedGone(ctx, oldID) { log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) - } else if db.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + } else if db.IsDeploymentMetadataService() { // When using the deployment metadata service, record the error. err = fmt.Errorf("deleting id=%s: %w", oldID, err) db.RecordFailure(d.ResourceKey, oldID, err) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index e44569d8bcb..60c91bbf5bf 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -72,16 +72,6 @@ const FeatureDeploymentHistory = "deployment_history" // state that depends on a feature it does not recognize. const featuresDocURL = "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" -// StorageBackend identifies where a deployment's resource state lives: the workspace filesystem -// (the state file) or the deployment metadata service. It is the source of truth for gating DMS -// behavior after Open; the plan carries its string form to check against the target's config. -type StorageBackend string - -const ( - StorageBackendWorkspaceFilesystem StorageBackend = "WORKSPACE_FILESYSTEM" - StorageBackendDeploymentMetadataService StorageBackend = "DEPLOYMENT_METADATA_SERVICE" -) - // recognizedFeatures is the set of state feature flags this CLI understands. A state depending on // any feature not listed here is refused (see checkStateFeatures), so a newer CLI's feature is not // silently clobbered by this one. @@ -136,10 +126,6 @@ type DeploymentState struct { // client when the deployment records history; nil otherwise. dmsClient *dms.Client - // storageBackend is where this deployment's state lives, set by Open from the feature marker. - // It is the source of truth for gating DMS behavior after Open. - storageBackend StorageBackend - // versionCompleted makes CompleteVersion a no-op after the first call, so a deferred safety-net // completion after an explicit one does nothing. versionCompleted bool @@ -231,9 +217,6 @@ func (db *DeploymentState) getOperationBuffer() *dms.OperationBuffer { // stops touching resources once the service is no longer keeping up. Nil when the bundle does not // record deployment history or recording is healthy. func (db *DeploymentState) RecordingError() error { - if db.StorageBackend() != StorageBackendDeploymentMetadataService { - return nil - } buf := db.getOperationBuffer() if buf == nil { return nil @@ -247,10 +230,8 @@ func (db *DeploymentState) RecordingError() error { // one does nothing. Finalize has already drained the buffered operations. It reads only fields that // survive Finalize's reset and asserts nothing, so it is safe to call after the state is closed. func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (bool, error) { - if db.StorageBackend() != StorageBackendDeploymentMetadataService { - return false, nil - } - + // Gated on the buffer below rather than on the state's features: this runs after Finalize, which + // resets Data, so the features are gone by now. A buffer exists only for a recorded deployment. db.mu.Lock() buf := db.operationBuffer if buf == nil || db.versionCompleted { @@ -284,7 +265,7 @@ func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (b // RecordFailure records that a resource did not apply, so the history says why rather than // leaving the resource out. resourceID is the id it had before the failure. func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause error) { - if db.StorageBackend() != StorageBackendDeploymentMetadataService { + if !db.IsDeploymentMetadataService() { return } @@ -343,7 +324,7 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta // A recorded deployment persists through the service, everything else through the WAL. The // entry is still kept in memory: Finalize exports it for metadata.json and the deploy summary, // and dataForFile empties State again before the tombstone is written. - if db.StorageBackend() == StorageBackendDeploymentMetadataService { + if db.isDeploymentMetadataService() { db.Data.State[key] = entry db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { @@ -378,7 +359,7 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgre deletedID := db.stateIDs[key] // A recorded deployment persists through the service, everything else through the WAL. - if db.StorageBackend() == StorageBackendDeploymentMetadataService { + if db.isDeploymentMetadataService() { if buf := db.operationBuffer; buf != nil { buf.RecordOperation(ctx, key, inProgress, deletedID, nil) } @@ -440,16 +421,25 @@ func (db *DeploymentState) StateFeatures() map[string]struct{} { func (db *DeploymentState) GetSerial() int { db.mu.Lock() defer db.mu.Unlock() - if db.StorageBackend() == StorageBackendDeploymentMetadataService { + if db.isDeploymentMetadataService() { return db.VersionID } return db.Data.Serial } -// StorageBackend reports where this deployment's state lives - the source of truth for whether the -// bundle records deployment history. Set by Open; valid only after the state is opened. -func (db *DeploymentState) StorageBackend() StorageBackend { - return db.storageBackend +// isDeploymentMetadataService is IsDeploymentMetadataService for callers already holding db.mu. +func (db *DeploymentState) isDeploymentMetadataService() bool { + _, ok := db.Data.Features[FeatureDeploymentHistory] + return ok +} + +// IsDeploymentMetadataService reports whether this deployment's resource state lives in the +// deployment metadata service rather than the state file. The state's features are the source of +// truth; valid only after Open. +func (db *DeploymentState) IsDeploymentMetadataService() bool { + db.mu.Lock() + defer db.mu.Unlock() + return db.isDeploymentMetadataService() } // DmsClient returns the deployment metadata service client Open built from the workspace client, @@ -611,10 +601,7 @@ To record this bundle's history, start it over as a new deployment: return ErrUnsettingRecording } - db.storageBackend = StorageBackendWorkspaceFilesystem if recorded { - db.storageBackend = StorageBackendDeploymentMetadataService - // The service is the source of truth for a recorded deployment; the file is a tombstone // carrying only the marker, and applyDMSState loads the resources the service holds. client, err := dms.NewClient(cmdctx.WorkspaceClient(ctx)) @@ -646,7 +633,7 @@ To record this bundle's history, start it over as a new deployment: // and Open recovers from there rather than replaying a log. Stamp the header fields the // replay would have carried over, then skip the file. The serial is left alone: under // recording it comes from the deployment's last_version_id. - if db.storageBackend == StorageBackendDeploymentMetadataService { + if db.isDeploymentMetadataService() { db.Data.Lineage = db.GetOrInitLineage() db.Data.StateVersion = currentStateVersion db.Data.CLIVersion = build.GetInfo().Version @@ -833,7 +820,7 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported } db.walFile = nil err = db.replayWAL(ctx) - } else if db.openedForWrite && db.storageBackend == StorageBackendDeploymentMetadataService { + } else if db.openedForWrite && db.isDeploymentMetadataService() { // replayWAL is what normally persists the file. Without one, write the tombstone here so // the header (lineage, state version, CLI version) still lands on disk - even when no // operations were recorded, since the deployment and any version this run created exist @@ -842,7 +829,7 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported } // Wait until all operations are recorded in the service. - if db.StorageBackend() == StorageBackendDeploymentMetadataService && db.operationBuffer != nil { + if db.isDeploymentMetadataService() && db.operationBuffer != nil { if drainErr := db.operationBuffer.Drain(); drainErr != nil { err = errors.Join(err, drainErr) } @@ -870,7 +857,7 @@ func (db *DeploymentState) UpgradeToWrite() error { db.openedForWrite = true // As in Open: a recorded deployment writes no WAL, so stamp the header and skip the file. - if db.storageBackend == StorageBackendDeploymentMetadataService { + if db.isDeploymentMetadataService() { db.Data.Lineage = db.GetOrInitLineage() db.Data.StateVersion = currentStateVersion db.Data.CLIVersion = build.GetInfo().Version @@ -1022,7 +1009,7 @@ func (db *DeploymentState) unlockedSave() error { // Data to persist in the remote resources.json file. func (db *DeploymentState) dataForFile() Database { - if db.StorageBackend() == StorageBackendDeploymentMetadataService { + if db.isDeploymentMetadataService() { header := db.Data.Header if header.Features == nil { header.Features = make(map[string]struct{}) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 9550397b164..f972ed42b71 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -18,7 +18,6 @@ import ( "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" @@ -204,11 +203,10 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // The version is created only after approval; CompleteVersion is deferred before // lock.Release and no-ops until then. defer func() { - // Close the version when backend type is deployment metadata service. - if b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { - if _, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { - logdiag.LogError(ctx, err) - } + // Unguarded: CompleteVersion no-ops when no version was created. It runs after Finalize has + // reset the state, so the state's features are no longer there to gate on. + if _, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() @@ -320,8 +318,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. - // IsDirect first: StorageBackend asserts the state is open, and only the direct engine opens it. - if stateEngine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + // IsDirect first: the state must be open to read its features, and only the direct engine opens it. + if stateEngine.IsDirect() && b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { firstDeploy := b.DeploymentBundle.StateDB.DeploymentID == "" createOrUpdateDeployment(ctx, b, dmsDeployment) if logdiag.HasError(ctx) { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 16bfeb24ea2..44cc386cb70 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -14,7 +14,6 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" @@ -160,7 +159,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } - if engine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + if engine.IsDirect() && b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { // Complete version before deleting remote files; the deployment node is under statePath. completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) if err != nil { @@ -219,7 +218,9 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // destroy records nothing. Deferred before lock.Release to hold the lock; a no-op once // destroyCore has completed the version. defer func() { - if engine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + // Unguarded: CompleteVersion no-ops when no version was created. It runs after Finalize has + // reset the state, so the state's features are no longer there to gate on. + if engine.IsDirect() { completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)) if err != nil { logdiag.LogError(ctx, err) @@ -292,7 +293,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // destroy's recording: it opens the operation buffer that destroyCore drains and completes, // so the drain and the deployment delete follow only when a version was started here. // Destroy never creates or updates the deployment - it is about to be deleted. - if engine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + if engine.IsDirect() && b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { staged, err := stagedOperations(plan) if err != nil { logdiag.LogError(ctx, err) From 5d3549862154bd6b37b3fa1f38061d9e74eda4f3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 12:36:18 +0200 Subject: [PATCH 154/164] update comment --- bundle/direct/dstate/state.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 60c91bbf5bf..027c6b1b725 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -147,6 +147,8 @@ type OpenDmsArgs struct { // LastVersionID is the most recent version the service has recorded, zero when it has none. // Open takes the state's version from it: the service owns the number. + // + // Equivalent to serial for a file based deployment that uses resources.json to track state, LastVersionID int } From 83f339ebc47451ca9f0317bd6ed047f29f7e1900 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 12:39:41 +0200 Subject: [PATCH 155/164] better commentn --- bundle/direct/dstate/dms.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 3bab77e4ed3..40034f373f5 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -16,17 +16,10 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// applyDMSState fills in the resource state from what DMS recorded. A recorded deployment's file is -// an empty shell - the service owns the resource set outright - so there is nothing to merge with: -// an empty set means the service tracks nothing, whether because the deploy created nothing or -// because the deployment is gone. The caller holds db.mu. +// applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS +// resources.json does not track state. The service is queried for state for resources and then they +// are filled in-place in DeploymentState.Data.State: func (db *DeploymentState) applyDMSState(recorded []dms.Resource) error { - if len(db.Data.State) > 0 { - return fmt.Errorf("internal error: state file for a recorded deployment carries %d resources, expected none", len(db.Data.State)) - } - - // Built first and assigned together, so a malformed envelope leaves the state as it was - // rather than half replaced. resources := make(map[string]ResourceEntry, len(recorded)) stateIDs := make(map[string]string, len(recorded)) for _, res := range recorded { From 33be75184d519041442d7737ae9367d21228f096 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 12:43:46 +0200 Subject: [PATCH 156/164] remove wrapper struct for deployment ID and last version ID --- bundle/config/deployment.go | 9 --------- bundle/phases/dms.go | 5 +---- cmd/bundle/utils/process.go | 7 ++----- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/bundle/config/deployment.go b/bundle/config/deployment.go index 48218607bbb..e29051e4ff5 100644 --- a/bundle/config/deployment.go +++ b/bundle/config/deployment.go @@ -8,15 +8,6 @@ type Deployment struct { // Lock configures locking behavior on deployment. Lock Lock `json:"lock,omitempty"` - // History reports what the deployment metadata service has recorded for this - // bundle. Output only: it is read from the service for 'bundle summary' and is - // unset when the bundle does not record deployment history. - History *DeploymentHistory `json:"history,omitempty" bundle:"readonly"` -} - -// DeploymentHistory identifies the bundle's deployment in the deployment -// metadata service. -type DeploymentHistory struct { // DeploymentID is the ID the service assigned to this bundle's deployment. DeploymentID string `json:"deployment_id,omitempty"` diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 74dfc1b9690..9ceea29509d 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -89,10 +89,7 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu // A first deploy had no deployment to read at startup, so its id enters the history here. bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { - if b.Config.Bundle.Deployment.History == nil { - b.Config.Bundle.Deployment.History = &config.DeploymentHistory{} - } - b.Config.Bundle.Deployment.History.DeploymentID = deploymentID + b.Config.Bundle.Deployment.DeploymentID = deploymentID }) } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 5753c8efeef..66666ddbd7a 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -11,7 +11,6 @@ import ( "time" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" @@ -273,10 +272,8 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} if dmsDeploymentID != "" { bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { - b.Config.Bundle.Deployment.History = &config.DeploymentHistory{ - DeploymentID: dmsDeploymentID, - LatestVersionID: lastVersionID, - } + b.Config.Bundle.Deployment.DeploymentID = dmsDeploymentID + b.Config.Bundle.Deployment.LatestVersionID = lastVersionID }) muts = append(muts, metadata.AnnotateDeployment(dmsDeploymentID)) } From a4540bf058129c0105da21555deca1390edaa81c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 12:58:12 +0200 Subject: [PATCH 157/164] Regenerate the schema for the flattened deployment fields, complete the destroy version deployment_id and latest_version_id moved onto Deployment itself, so the generated schema needed regenerating and the two new fields needed descriptions rather than PLACEHOLDER - what validate-generated was failing on. bundle/dms/no-drift reads them from their new place. destroyCore's version completion was still gated on the state's features, which are gone by then: Finalize resets Data before it runs. That deferred completion and the deployment record's deletion until after files.Delete had removed the node they depend on. sameWorkspaceInfo is gone - structdiff.IsEqual compares the pointers directly and handles nil on either side, so the wrapper only restated it. Co-authored-by: Isaac --- acceptance/bundle/dms/no-drift/script | 4 ++-- bundle/internal/schema/annotations.yml | 6 ++++++ bundle/phases/destroy.go | 4 +++- bundle/schema/jsonschema.json | 8 ++++++++ libs/dms/metadata.go | 10 +--------- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index fcecb6d1f5c..1c544ce956d 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-drift-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' -trace $CLI bundle summary -o json | jq .bundle.deployment.history +trace $CLI bundle summary -o json | jq '.bundle.deployment | {deployment_id, latest_version_id}' title "The plan is a no-op now and records the deployment_history feature" dms_plan @@ -24,7 +24,7 @@ trace update_file.py databricks.yml 'serverless: true' 'serverless: true trace $CLI bundle deploy trace print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' -trace $CLI bundle summary -o json | jq .bundle.deployment.history +trace $CLI bundle summary -o json | jq '.bundle.deployment | {deployment_id, latest_version_id}' title "The plan shows the updated pipeline at serial 2 with the deployment_history feature" dms_plan diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index f6d2297a3fc..58f7154a28d 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -99,9 +99,15 @@ bundle: "markdown_description": |- The definition of the bundle deployment. For supported attributes see [\_](/dev-tools/bundles/deployment-modes.md). "$fields": + "deployment_id": + "description": |- + The ID the deployment metadata service assigned to this bundle's deployment. Output only. "fail_on_active_runs": "description": |- Whether to fail on active runs. If this is set to true a deployment that is running can be interrupted. + "latest_version_id": + "description": |- + The most recent version the deployment metadata service recorded for this deployment. Output only. "lock": "description": |- The deployment lock attributes. diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 44cc386cb70..234093ae8b6 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -159,7 +159,9 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } - if engine.IsDirect() && b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + // Unguarded beyond the engine: this runs after Finalize reset the state, so its features are no + // longer there to gate on, and CompleteVersion no-ops when no version was created. + if engine.IsDirect() { // Complete version before deleting remote files; the deployment node is under statePath. completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) if err != nil { diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index cba6063023e..b10861b65c3 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3621,10 +3621,18 @@ { "type": "object", "properties": { + "deployment_id": { + "description": "The ID the deployment metadata service assigned to this bundle's deployment. Output only.", + "$ref": "#/$defs/string" + }, "fail_on_active_runs": { "description": "Whether to fail on active runs. If this is set to true a deployment that is running can be interrupted.", "$ref": "#/$defs/bool" }, + "latest_version_id": { + "description": "The most recent version the deployment metadata service recorded for this deployment. Output only.", + "$ref": "#/$defs/int" + }, "lock": { "description": "The deployment lock attributes.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Lock" diff --git a/libs/dms/metadata.go b/libs/dms/metadata.go index e9a4a10a8e0..33b7d49f3c4 100644 --- a/libs/dms/metadata.go +++ b/libs/dms/metadata.go @@ -34,14 +34,6 @@ type Metadata struct { // among them: the service derives the deployment's from the version that carried it. var deploymentFields = []string{"display_name", "target_name", "deployment_mode", "workspace_info"} -// sameWorkspaceInfo reports whether two workspace infos carry the same paths. -func sameWorkspaceInfo(want, current *bundledeployments.WorkspaceInfo) bool { - if want == nil || current == nil { - return want == nil && current == nil - } - return structdiff.IsEqual(*want, *current) -} - // deployment renders the metadata the deployment owns. func (m Metadata) deployment() bundledeployments.Deployment { return bundledeployments.Deployment{ @@ -70,7 +62,7 @@ func (m Metadata) StaleFields(current *bundledeployments.Deployment) string { if want.DeploymentMode != current.DeploymentMode { stale = append(stale, "deployment_mode") } - if !sameWorkspaceInfo(want.WorkspaceInfo, current.WorkspaceInfo) { + if !structdiff.IsEqual(want.WorkspaceInfo, current.WorkspaceInfo) { stale = append(stale, "workspace_info") } return strings.Join(stale, ",") From 0dbffa2a0bce1616f125fb384c4b81e6531e1912 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 13:11:36 +0200 Subject: [PATCH 158/164] bundle/direct/dstate: restore the empty-shell assertion in applyDMSState It was dropped along with the comment rewrite in 83f339eb, which left its unit case failing - and unit tests run in every CI cell, so all of them went red regardless of engine. Co-authored-by: Isaac --- bundle/direct/dstate/dms.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 40034f373f5..57e10b8d4b4 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -20,6 +20,12 @@ type RecordedState struct { // resources.json does not track state. The service is queried for state for resources and then they // are filled in-place in DeploymentState.Data.State: func (db *DeploymentState) applyDMSState(recorded []dms.Resource) error { + if len(db.Data.State) > 0 { + return fmt.Errorf("internal error: state file for a recorded deployment carries %d resources, expected none", len(db.Data.State)) + } + + // Built first and assigned together, so a malformed envelope leaves the state alone rather than + // half filled. resources := make(map[string]ResourceEntry, len(recorded)) stateIDs := make(map[string]string, len(recorded)) for _, res := range recorded { From 93ac8030931c5fabe9e1ca552a9b734161ff24f1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 13:31:46 +0200 Subject: [PATCH 159/164] cmd/bundle/utils: note the half-created deployment recovery in the TODO A deployment is usable only when both the workspace node and the service's record exist. A half-created one - node present, record missing - blocks the bundle here, even though CreateDeployment already recovers from it. Records the shape proposed in review: a dms.ReadDeployment(ctx, statePath) that reports a deployment only when both halves are present, leaving the caller to call CreateDeployment to create or finalize it. Co-authored-by: Isaac --- cmd/bundle/utils/process.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 66666ddbd7a..a4824f329af 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -516,9 +516,13 @@ func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineS // Lookup and return the deployment object from ${workspace.state_path}/resources.deployment.json // -// TODO: ask the service for a new API that looks a deployment up by state path directly -// (GetDeployment(state_path)). That would replace the two round trips here: a workspace lookup to -// turn the node into an id, then a get by that id. +// TODO: a deployment is only usable when both the node and the service's record exist, and a +// half-created one - node present, record missing - blocks the bundle here even though +// CreateDeployment already recovers from it. Move this behind a dms.ReadDeployment(ctx, statePath) +// that returns an empty id and version unless both halves are there, leaving the caller to call +// CreateDeployment to create or finalize it. That call would also replace the two round trips here +// (a workspace lookup to turn the node into an id, then a get by that id) with one, if the service +// grows a lookup by state path. func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { nodePath := path.Join(statePath, dms.DeploymentNodeName) From 0883adcdc721ab3d797e8f49442365765d2d0703 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 13:55:34 +0200 Subject: [PATCH 160/164] libs/dms: record two deferred items in the code, correct the resource_id reason Probed the real service: it rejects resource_id on a delete with "state must be in update_mask when 'resource_id' is", and means state with a value. Dropping state from the mask entirely gets the same error, so no request shape carries an id alongside a cleared state. The ClearsState comment said something close but not quite that; it now quotes what the service actually answers, with a TODO to record the deleted resource's id once that is allowed. StatusPending gets the follow-up that was only on the PR: use IN_PROGRESS once the service supports it, since it describes a half-applied recreate better. Co-authored-by: Isaac --- libs/dms/fields.go | 11 +++++++---- libs/dms/operation.go | 3 +++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/libs/dms/fields.go b/libs/dms/fields.go index 05d39b387d7..e175e163aa5 100644 --- a/libs/dms/fields.go +++ b/libs/dms/fields.go @@ -19,16 +19,19 @@ const DescribesResource = FieldState | FieldErrorMessage | FieldResourceID | Fie // ClearsState is what a write that leaves no resource claims: a delete, and the delete half of // a recreate. State is named so the service reads the absent value as a clear, which is what -// drops the resource from the deployment. It cannot name resource_id either: the service counts -// only a state it was given as recording one, so an id alongside a cleared state is refused. +// drops the resource from the deployment. It cannot name resource_id: the service answers +// "state must be in update_mask when 'resource_id' is" and means state with a value, so there is +// no shape that carries an id alongside a cleared state (verified against the API). +// +// TODO: record the deleted resource's id too, once the service allows it alongside a cleared state. const ClearsState = FieldState | FieldErrorMessage | FieldStatus // KeepsState is what a failure claims: mark it failed and leave state alone. State means // the resource is as it was written; no state means a delete went through and nothing // replaced it, so the resource really is gone and the deployment should say so. // -// It cannot name resource_id: the service requires state in any mask that names the id, and -// naming state here would overwrite what the resource last recorded. +// It cannot name resource_id: the service requires state, with a value, in any mask that names +// the id, and naming state here would overwrite what the resource last recorded. const KeepsState = FieldErrorMessage | FieldStatus // wireNames pairs each field with its name on the wire, in the order a mask lists them. diff --git a/libs/dms/operation.go b/libs/dms/operation.go index 16e1c1a3588..57577f5dc21 100644 --- a/libs/dms/operation.go +++ b/libs/dms/operation.go @@ -21,6 +21,9 @@ const maxErrorMessageSize = 16 * 1024 // is where a recreate sits between deleting the old resource and creating the new one. Not in // the SDK: the enum ships server-first while it is staged DEVELOPMENT, so the generated client // carries only the terminal two. +// +// TODO: use IN_PROGRESS here once the service supports it - it describes a half-applied recreate +// better than "recorded but not applied". const StatusPending bundledeployments.OperationStatus = "OPERATION_STATUS_PENDING" // OperationUpdate is one write to an operation the version staged: the fields it claims From 12e10e7f58d8a2eb3e1e4d28c2cac7cab09b8a44 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 13:58:58 +0200 Subject: [PATCH 161/164] cmd/bundle/utils: split the two deployment-lookup TODOs Recovering a half-created deployment and collapsing the two round trips into one are independent asks; keep them as separate entries. Co-authored-by: Isaac --- cmd/bundle/utils/process.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index a4824f329af..ef1d48da4be 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -520,9 +520,10 @@ func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineS // half-created one - node present, record missing - blocks the bundle here even though // CreateDeployment already recovers from it. Move this behind a dms.ReadDeployment(ctx, statePath) // that returns an empty id and version unless both halves are there, leaving the caller to call -// CreateDeployment to create or finalize it. That call would also replace the two round trips here -// (a workspace lookup to turn the node into an id, then a get by that id) with one, if the service -// grows a lookup by state path. +// CreateDeployment to create or finalize it. +// +// TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a +// workspace lookup to turn the node into an id, then a get by that id. func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { nodePath := path.Join(statePath, dms.DeploymentNodeName) From 9233825bf04f26284d8324e423992fc361239c17 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 14:13:46 +0200 Subject: [PATCH 162/164] bundle/direct/dstate: cache whether the deployment records history Open already decides this from the state's features, so it records the answer instead of every caller re-deriving it. That also outlives the features themselves - Finalize resets Data - so CompleteVersion and its callers in the deploy and destroy phases can gate on the feature again rather than on the operation buffer. Cleared by Open rather than by reset: the same state is reopened (see bind.go), so a stale value would misroute a non-recording open, while reset is exactly what it has to survive. Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 19 +++++++++++++++---- bundle/phases/deploy.go | 8 ++++---- bundle/phases/destroy.go | 15 ++++----------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 027c6b1b725..f4fb00c6b9e 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -130,6 +130,11 @@ type DeploymentState struct { // completion after an explicit one does nothing. versionCompleted bool + // recordsHistory is whether this deployment records history, decided by Open from the state's + // features. Cached because it outlives the features themselves: Finalize resets Data, and + // CompleteVersion runs after that. + recordsHistory bool + // DeploymentID is the recorded deployment's id. Set at Open, or by the first deploy that creates // the deployment, which Open cannot know about. DeploymentID string @@ -232,8 +237,10 @@ func (db *DeploymentState) RecordingError() error { // one does nothing. Finalize has already drained the buffered operations. It reads only fields that // survive Finalize's reset and asserts nothing, so it is safe to call after the state is closed. func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (bool, error) { - // Gated on the buffer below rather than on the state's features: this runs after Finalize, which - // resets Data, so the features are gone by now. A buffer exists only for a recorded deployment. + if !db.IsDeploymentMetadataService() { + return false, nil + } + db.mu.Lock() buf := db.operationBuffer if buf == nil || db.versionCompleted { @@ -431,8 +438,7 @@ func (db *DeploymentState) GetSerial() int { // isDeploymentMetadataService is IsDeploymentMetadataService for callers already holding db.mu. func (db *DeploymentState) isDeploymentMetadataService() bool { - _, ok := db.Data.Features[FeatureDeploymentHistory] - return ok + return db.recordsHistory } // IsDeploymentMetadataService reports whether this deployment's resource state lives in the @@ -524,6 +530,9 @@ func (db *DeploymentState) reset() { func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeployment OpenDmsArgs) error { db.Path = path + // Cleared here rather than in reset, which CompleteVersion needs it to survive: the same state + // is reopened (see bind.go), and a stale value would misroute a non-recording open. + db.recordsHistory = false // The state file is the source of truth for whether this deployment records history: read it // first. @@ -603,6 +612,8 @@ To record this bundle's history, start it over as a new deployment: return ErrUnsettingRecording } + db.recordsHistory = recorded + if recorded { // The service is the source of truth for a recorded deployment; the file is a tombstone // carrying only the marker, and applyDMSState loads the resources the service holds. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f972ed42b71..425c4874283 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -203,10 +203,10 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // The version is created only after approval; CompleteVersion is deferred before // lock.Release and no-ops until then. defer func() { - // Unguarded: CompleteVersion no-ops when no version was created. It runs after Finalize has - // reset the state, so the state's features are no longer there to gate on. - if _, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { - logdiag.LogError(ctx, err) + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + if _, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 234093ae8b6..2ac8c8482d5 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -159,9 +159,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } - // Unguarded beyond the engine: this runs after Finalize reset the state, so its features are no - // longer there to gate on, and CompleteVersion no-ops when no version was created. - if engine.IsDirect() { + if engine.IsDirect() && b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { // Complete version before deleting remote files; the deployment node is under statePath. completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) if err != nil { @@ -220,9 +218,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // destroy records nothing. Deferred before lock.Release to hold the lock; a no-op once // destroyCore has completed the version. defer func() { - // Unguarded: CompleteVersion no-ops when no version was created. It runs after Finalize has - // reset the state, so the state's features are no longer there to gate on. - if engine.IsDirect() { + if engine.IsDirect() && b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)) if err != nil { logdiag.LogError(ctx, err) @@ -290,11 +286,8 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } } - // Record a destroy version now that it is approved and the state WAL is open, but only when - // the state is recorded with DMS (its marker is set). This is the single gate on the - // destroy's recording: it opens the operation buffer that destroyCore drains and completes, - // so the drain and the deployment delete follow only when a version was started here. - // Destroy never creates or updates the deployment - it is about to be deleted. + // Start the version for this destroy, now that it is approved. Everything else recording + // does follows from the buffer this opens. if engine.IsDirect() && b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { staged, err := stagedOperations(plan) if err != nil { From 792146f96737c07a5163df2f597d49d5c9a96ea1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 14:17:26 +0200 Subject: [PATCH 163/164] bundle/direct: move the plan-vs-state checks back into ValidatePlanAgainstState The checks sat in a validatePlan wrapper in cmd/bundle/utils that added nothing but a hop: everything they read is on the state. They go back to ValidatePlanAgainstState next to the state they read, and the caller calls it directly. Also drops a stale comment on templates/deployment-history: the DMS variant is not there to permit the field, the field is written by bundle init and DMS is what the test exercises. Co-authored-by: Isaac --- .../templates/deployment-history/test.toml | 2 -- bundle/direct/bundle_plan.go | 27 +++++++++++++++ cmd/bundle/utils/process.go | 34 ++----------------- 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/acceptance/bundle/templates/deployment-history/test.toml b/acceptance/bundle/templates/deployment-history/test.toml index 10d5d3aa050..3db5cdb90f5 100644 --- a/acceptance/bundle/templates/deployment-history/test.toml +++ b/acceptance/bundle/templates/deployment-history/test.toml @@ -3,6 +3,4 @@ Env.DATABRICKS_BUNDLE_INIT_DEPLOYMENT_HISTORY = "true" # What is under test is what `bundle init` writes, so one engine is enough. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# The generated project sets experimental.deployment_history; DMS=true permits it so -# the validate step below loads instead of rejecting it. EnvMatrix.DMS = ["true"] diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index d81e695fad9..48d76abe97f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -39,6 +39,33 @@ func (b *DeploymentBundle) init(client *databricks.WorkspaceClient) error { return err } +// ValidatePlanAgainstState rejects a saved plan that no longer matches the state it was built +// against: its features, its lineage and its serial. +func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan.Plan) error { + stateDB.AssertOpenedForReadOrWrite() + + // A plan is built against a set of state features, and the stamps it carries follow from + // them, so applying it to a target with a different set would deploy the wrong shape. + // The state is the target's source of truth and carries its features even on a first + // recorded deploy (unlike the version ids, which are empty then). + if !maps.Equal(plan.Features, stateDB.StateFeatures()) { + return errors.New("this plan was created for a different set of state features than the target now has; run 'bundle plan' again") + } + + // A plan taken before the first deploy carries no lineage, so both sides are empty then and this + // passes. If a deployment has happened since, the lineage no longer matches. + if plan.Lineage != stateDB.Data.Lineage { + return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) + } + + expected := stateDB.GetSerial() + if plan.Serial != expected { + return fmt.Errorf("plan serial %d does not match state serial %d; the state has been modified since the plan was created. Please run 'bundle plan' again", plan.Serial, expected) + } + + return nil +} + // InitForApply initializes the DeploymentBundle for applying a pre-computed plan. // StateDB must already be open for write before calling this function. func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index ef1d48da4be..ea29079f580 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "maps" "path" "path/filepath" "strconv" @@ -17,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/statemgmt" @@ -360,7 +360,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle log.Warnf(ctx, "Plan was created with CLI version %s but current version is %s", plan.CLIVersion, currentVersion) } - if err := validatePlan(b, plan); err != nil { + if err := direct.ValidatePlanAgainstState(&b.DeploymentBundle.StateDB, plan); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } @@ -458,36 +458,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, nil } -// validatePlan rejects a --plan file that no longer matches the target: its recording shape must -// match the target's config, and a plan that targets an existing recorded deployment must not -// predate the deployment the service now holds. It ends with the local lineage/serial guard. -// validatePlan rejects a saved plan that no longer matches the state it was built against. -func validatePlan(b *bundle.Bundle, plan *deployplan.Plan) error { - stateDB := &b.DeploymentBundle.StateDB - stateDB.AssertOpenedForReadOrWrite() - - // A plan is built against a set of state features, and the stamps it carries follow from - // them, so applying it to a target with a different set would deploy the wrong shape. - // The state is the target's source of truth and carries its features even on a first - // recorded deploy (unlike the version ids, which are empty then). - if !maps.Equal(plan.Features, stateDB.StateFeatures()) { - return errors.New("this plan was created for a different set of state features than the target now has; run 'bundle plan' again") - } - - // A plan taken before the first deploy carries no lineage, so both sides are empty then and this - // passes. If a deployment has happened since, the lineage no longer matches. - if plan.Lineage != stateDB.Data.Lineage { - return fmt.Errorf("plan lineage %q does not match state lineage %q; the state may have been modified by another process", plan.Lineage, stateDB.Data.Lineage) - } - - expected := stateDB.GetSerial() - if plan.Serial != expected { - return fmt.Errorf("plan serial %d does not match state serial %d; the state has been modified since the plan was created. Please run 'bundle plan' again", plan.Serial, expected) - } - - return nil -} - // ResolveEngineSetting determines the effective engine setting by combining bundle config and env var. // Priority: bundle.engine config > DATABRICKS_BUNDLE_ENGINE env var > engine.Default. func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineSetting, error) { From ecf560822ebf89d9a348514258076e036110ecef Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Sep 2026 14:44:41 +0200 Subject: [PATCH 164/164] acceptance/dms/successful-recreate: fix a title that claimed the wrong thing The plan runs after the recreate, so it shows nothing to do - the title said it showed the recreate action. The recreate is asserted by the recorded requests above it and the resource listing below. Co-authored-by: Isaac --- acceptance/bundle/dms/successful-recreate/output.txt | 2 +- acceptance/bundle/dms/successful-recreate/script | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 022757cde66..320369badc3 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -161,7 +161,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } } -=== The plan now shows the recreate action +=== The recreate is done: nothing left to plan, and the serial moved to 2 >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index 808b1c49b97..64aed46b84a 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -25,7 +25,7 @@ trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_ trace $CLI bundle deploy --auto-approve trace print_requests.py --dms //api/2.0/bundle -title "The plan now shows the recreate action" +title "The recreate is done: nothing left to plan, and the serial moved to 2" dms_plan title "The service lists the schema at version 2 with recreate action"