diff --git a/docs/concepts/tests.md b/docs/concepts/tests.md index a293237687..a9bd45a25a 100644 --- a/docs/concepts/tests.md +++ b/docs/concepts/tests.md @@ -463,6 +463,23 @@ You can also run tests that match a pattern or substring using a glob pathname e $ sqlmesh test tests/test_* ``` +You can pass `--local` to run tests without loading state from the configured state connection: + +``` bash +$ sqlmesh test --local +``` + +This keeps offline runs and commit hooks from opening a connection to the state backend. + +In multi-repository setups, or when running tests for only a subset of projects, models that exist only in remote state are not loaded under `--local`. Unlike [`sqlmesh lint --local`](../guides/linter.md), which reports additional errors in that situation, a test whose model is missing is **skipped with a warning and the run still succeeds**: + +``` +[WARNING] Model '"memory"."bronze"."a"' was not found at tests/test_a.yaml +.**Successfully Ran `1` Tests Against `duckdb`** +``` + +So a passing exit code alone does not mean every test you expected actually ran. Watch the output for these warnings, and keep in mind that a hook using `--local` will not fail on them. + ### Testing using notebooks You can execute tests on demand using the `%run_test` notebook magic as follows: diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 1367f8551b..1ed03535b9 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -630,6 +630,9 @@ Options: useful for debugging. --select-model TEXT Select specific models to run unit tests for. Can be specified multiple times. + --local Run tests using only locally loaded project files + without loading state. Tests whose model is not loaded + are skipped with a warning rather than failing. --help Show this message and exit. ``` diff --git a/sqlmesh/cli/main.py b/sqlmesh/cli/main.py index b6678136f0..c2c81ada9c 100644 --- a/sqlmesh/cli/main.py +++ b/sqlmesh/cli/main.py @@ -42,6 +42,8 @@ ) SKIP_CONTEXT_COMMANDS = ("init", "ui") LOCAL_ONLY_COMMANDS = ("format",) +# Commands that are local-only when they're passed --local. +OPTIONAL_LOCAL_COMMANDS = ("lint", "test") class _SQLMeshGroup(click.Group): @@ -129,8 +131,12 @@ def cli( load = True # Local-only gating must hold for any number of --paths, so it stays outside the block below. load_state = ctx.invoked_subcommand not in LOCAL_ONLY_COMMANDS - # The parent callback constructs Context before Click invokes `lint`, so inspect its parsed args here. - if ctx.invoked_subcommand == "lint" and "--local" in ctx.meta["subcommand_args"]: + # The parent callback constructs Context before Click invokes the subcommand, so inspect its + # parsed args here. + if ( + ctx.invoked_subcommand in OPTIONAL_LOCAL_COMMANDS + and "--local" in ctx.meta["subcommand_args"] + ): load_state = False if len(paths) == 1: @@ -811,6 +817,12 @@ def create_test( multiple=True, help="Select specific models to run unit tests for.", ) +@click.option( + "--local", + is_flag=True, + expose_value=False, + help="Run tests using only locally loaded project files without loading state. Tests whose model is not loaded are skipped with a warning rather than failing.", +) @click.argument("tests", nargs=-1) @click.pass_obj @error_handler diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index da73952991..b52df5394a 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -2530,6 +2530,59 @@ def test_lint_local_runs_without_state(runner: CliRunner, tmp_path: Path, mocker mock.assert_not_called() +def test_test_still_loads_state(runner: CliRunner, tmp_path: Path, mocker): + """Guard that `test` explicitly passes `load_state=True` and still reaches state sync.""" + mock = _setup_local_only_project(tmp_path, mocker) + init_spy = mocker.spy(Context, "__init__") + + runner.invoke(cli, ["--paths", str(tmp_path), "test"]) + + assert init_spy.called, "Context was never constructed" + for call in init_spy.call_args_list: + assert "load_state" in call.kwargs, ( + "CLI didn't pass load_state= explicitly; missing kwarg defaults to True silently" + ) + assert call.kwargs["load_state"] is True, ( + f"Context was constructed with load_state={call.kwargs['load_state']} for `test`" + ) + assert mock.called, "state-sync was never accessed during `test`" + + +def test_test_local_runs_without_state(runner: CliRunner, tmp_path: Path, mocker): + mock = _setup_local_only_project(tmp_path, mocker) + init_spy = mocker.spy(Context, "__init__") + + result = runner.invoke(cli, ["--paths", str(tmp_path), "test", "--local"]) + + assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}" + assert init_spy.called, "Context was never constructed" + for call in init_spy.call_args_list: + assert "load_state" in call.kwargs, ( + "CLI didn't pass load_state= explicitly; missing kwarg defaults to True silently" + ) + assert call.kwargs["load_state"] is False, ( + f"Context was constructed with load_state={call.kwargs['load_state']} for `test --local`" + ) + mock.assert_not_called() + + +def test_test_local_runs_without_state_multiple_paths( + runner: CliRunner, tmp_path: Path, mocker +) -> None: + """`--local` gating must hold for any number of --paths, matching `lint --local`.""" + project_a = tmp_path / "a" + project_b = tmp_path / "b" + _create_local_only_project(project_a, "proj_a") + _create_local_only_project(project_b, "proj_b") + mock = _patch_state_access(mocker) + + result = runner.invoke( + cli, ["--paths", str(project_a), "--paths", str(project_b), "test", "--local"] + ) + assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}" + mock.assert_not_called() + + @pytest.mark.parametrize("command", ["format"]) def test_local_only_commands_skip_state_multiple_paths( runner: CliRunner, tmp_path: Path, mocker, command: str @@ -2611,3 +2664,115 @@ def test_format_does_not_open_state_connection( result = runner.invoke(cli, ["--paths", str(tmp_path), "format"]) assert result.exit_code == 0, f"Format failed: {result.output}\nException: {result.exception}" mock.assert_not_called() + + +def test_test_local_runs_project_unit_tests(runner: CliRunner, tmp_path: Path, mocker) -> None: + """A real unit test from the project's YAML runs under `--local` without touching state.""" + create_example_project(tmp_path) + mock = _patch_state_access(mocker) + + result = runner.invoke(cli, ["--paths", str(tmp_path), "test", "--local"]) + + assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}" + assert "Successfully Ran 1 tests" in " ".join(result.output.split()) + mock.assert_not_called() + + +def test_test_local_does_not_open_state_connection( + runner: CliRunner, tmp_path: Path, mocker, monkeypatch +) -> None: + """`test --local` must not open a configured remote Postgres state connection.""" + pytest.importorskip("psycopg2") + + for var in ("PG_HOST", "PG_USER", "PG_PASSWORD", "PG_DATABASE"): + monkeypatch.delenv(var, raising=False) + + create_example_project(tmp_path) + (tmp_path / "config.yaml").write_text( + """project: cli_test + +gateways: + prod: + state_connection: + type: postgres + host: "{{ env_var('PG_HOST', 'postgres.internal.example.com') }}" + port: 5432 + user: "{{ env_var('PG_USER') }}" + password: "{{ env_var('PG_PASSWORD') }}" + database: "{{ env_var('PG_DATABASE', 'sqlmesh_state') }}" + connection: + type: duckdb + database: "warehouse.db" + +default_gateway: prod + +model_defaults: + dialect: duckdb +""", + encoding="utf-8", + ) + + mock = _patch_state_access(mocker) + + result = runner.invoke(cli, ["--paths", str(tmp_path), "test", "--local"]) + assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}" + mock.assert_not_called() + + +def test_test_local_multi_repo_partial(runner: CliRunner, copy_to_temp_path, mocker) -> None: + """Run tests for one repo of a multi-repo project whose upstream models live only in state. + + Pins the behavioral difference against `lint --local`: a model that isn't loaded produces a + warning and its test is skipped, rather than turning into an error. + """ + repo_2 = copy_to_temp_path("examples/multi")[0] / "repo_2" + + # silver.c lives in repo_2 and its upstream bronze.a is supplied as a test input. + (repo_2 / "tests" / "test_c.yaml").write_text( + """test_silver_c: + model: silver.c + inputs: + bronze.a: + rows: + - col_a: 1 + - col_a: 1 + - col_a: 2 + outputs: + query: + rows: + - col_a: 1 + - col_a: 2 +""", + encoding="utf-8", + ) + # bronze.a itself is defined in repo_1, so it is not loaded when only repo_2 is given. + (repo_2 / "tests" / "test_a.yaml").write_text( + """test_bronze_a: + model: bronze.a + outputs: + query: + rows: + - col_a: 1 +""", + encoding="utf-8", + ) + + mock = _patch_state_access(mocker) + args = ["--gateway", "memory", "--paths", str(repo_2), "test"] + + # Without --local the same run reaches the state backend. + runner.invoke(cli, args) + assert mock.called, "state-sync was never accessed during `test`" + + mock.reset_mock() + + result = runner.invoke(cli, [*args, "--local"]) + + assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}" + # Console output wraps, so compare against whitespace-normalized text. + output = " ".join(result.output.split()) + assert 'Model \'"memory"."bronze"."a"\' was not found' in output, ( + "the unloaded model should warn rather than fail" + ) + assert "Successfully Ran 1 tests" in output, "the repo_2 test should still run" + mock.assert_not_called()