Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/concepts/tests.md
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,23 @@ You can also run tests that match a pattern or substring using a glob pathname e
$ sqlmesh test tests/test_*
```

You can pass `--local` to run tests without loading state from the configured state connection:

``` bash
$ sqlmesh test --local
```

This keeps offline runs and commit hooks from opening a connection to the state backend.

In multi-repository setups, or when running tests for only a subset of projects, models that exist only in remote state are not loaded under `--local`. Unlike [`sqlmesh lint --local`](../guides/linter.md), which reports additional errors in that situation, a test whose model is missing is **skipped with a warning and the run still succeeds**:

```
[WARNING] Model '"memory"."bronze"."a"' was not found at tests/test_a.yaml
.**Successfully Ran `1` Tests Against `duckdb`**
```

So a passing exit code alone does not mean every test you expected actually ran. Watch the output for these warnings, and keep in mind that a hook using `--local` will not fail on them.

### Testing using notebooks

You can execute tests on demand using the `%run_test` notebook magic as follows:
Expand Down
3 changes: 3 additions & 0 deletions docs/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -630,6 +630,9 @@ Options:
useful for debugging.
--select-model TEXT Select specific models to run unit tests for. Can be
specified multiple times.
--local Run tests using only locally loaded project files
without loading state. Tests whose model is not loaded
are skipped with a warning rather than failing.
--help Show this message and exit.
```

Expand Down
16 changes: 14 additions & 2 deletions sqlmesh/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
)
SKIP_CONTEXT_COMMANDS = ("init", "ui")
LOCAL_ONLY_COMMANDS = ("format",)
# Commands that are local-only when they're passed --local.
OPTIONAL_LOCAL_COMMANDS = ("lint", "test")


class _SQLMeshGroup(click.Group):
Expand Down Expand Up @@ -129,8 +131,12 @@ def cli(
load = True
# Local-only gating must hold for any number of --paths, so it stays outside the block below.
load_state = ctx.invoked_subcommand not in LOCAL_ONLY_COMMANDS
# The parent callback constructs Context before Click invokes `lint`, so inspect its parsed args here.
if ctx.invoked_subcommand == "lint" and "--local" in ctx.meta["subcommand_args"]:
# The parent callback constructs Context before Click invokes the subcommand, so inspect its
# parsed args here.
if (
ctx.invoked_subcommand in OPTIONAL_LOCAL_COMMANDS
and "--local" in ctx.meta["subcommand_args"]
):
load_state = False

if len(paths) == 1:
Expand Down Expand Up @@ -811,6 +817,12 @@ def create_test(
multiple=True,
help="Select specific models to run unit tests for.",
)
@click.option(
"--local",
is_flag=True,
expose_value=False,
help="Run tests using only locally loaded project files without loading state. Tests whose model is not loaded are skipped with a warning rather than failing.",
)
@click.argument("tests", nargs=-1)
@click.pass_obj
@error_handler
Expand Down
165 changes: 165 additions & 0 deletions tests/cli/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2530,6 +2530,59 @@ def test_lint_local_runs_without_state(runner: CliRunner, tmp_path: Path, mocker
mock.assert_not_called()


def test_test_still_loads_state(runner: CliRunner, tmp_path: Path, mocker):
"""Guard that `test` explicitly passes `load_state=True` and still reaches state sync."""
mock = _setup_local_only_project(tmp_path, mocker)
init_spy = mocker.spy(Context, "__init__")

runner.invoke(cli, ["--paths", str(tmp_path), "test"])

assert init_spy.called, "Context was never constructed"
for call in init_spy.call_args_list:
assert "load_state" in call.kwargs, (
"CLI didn't pass load_state= explicitly; missing kwarg defaults to True silently"
)
assert call.kwargs["load_state"] is True, (
f"Context was constructed with load_state={call.kwargs['load_state']} for `test`"
)
assert mock.called, "state-sync was never accessed during `test`"


def test_test_local_runs_without_state(runner: CliRunner, tmp_path: Path, mocker):
mock = _setup_local_only_project(tmp_path, mocker)
init_spy = mocker.spy(Context, "__init__")

result = runner.invoke(cli, ["--paths", str(tmp_path), "test", "--local"])

assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}"
assert init_spy.called, "Context was never constructed"
for call in init_spy.call_args_list:
assert "load_state" in call.kwargs, (
"CLI didn't pass load_state= explicitly; missing kwarg defaults to True silently"
)
assert call.kwargs["load_state"] is False, (
f"Context was constructed with load_state={call.kwargs['load_state']} for `test --local`"
)
mock.assert_not_called()


def test_test_local_runs_without_state_multiple_paths(
runner: CliRunner, tmp_path: Path, mocker
) -> None:
"""`--local` gating must hold for any number of --paths, matching `lint --local`."""
project_a = tmp_path / "a"
project_b = tmp_path / "b"
_create_local_only_project(project_a, "proj_a")
_create_local_only_project(project_b, "proj_b")
mock = _patch_state_access(mocker)

result = runner.invoke(
cli, ["--paths", str(project_a), "--paths", str(project_b), "test", "--local"]
)
assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}"
mock.assert_not_called()


@pytest.mark.parametrize("command", ["format"])
def test_local_only_commands_skip_state_multiple_paths(
runner: CliRunner, tmp_path: Path, mocker, command: str
Expand Down Expand Up @@ -2611,3 +2664,115 @@ def test_format_does_not_open_state_connection(
result = runner.invoke(cli, ["--paths", str(tmp_path), "format"])
assert result.exit_code == 0, f"Format failed: {result.output}\nException: {result.exception}"
mock.assert_not_called()


def test_test_local_runs_project_unit_tests(runner: CliRunner, tmp_path: Path, mocker) -> None:
"""A real unit test from the project's YAML runs under `--local` without touching state."""
create_example_project(tmp_path)
mock = _patch_state_access(mocker)

result = runner.invoke(cli, ["--paths", str(tmp_path), "test", "--local"])

assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}"
assert "Successfully Ran 1 tests" in " ".join(result.output.split())
mock.assert_not_called()


def test_test_local_does_not_open_state_connection(
runner: CliRunner, tmp_path: Path, mocker, monkeypatch
) -> None:
"""`test --local` must not open a configured remote Postgres state connection."""
pytest.importorskip("psycopg2")

for var in ("PG_HOST", "PG_USER", "PG_PASSWORD", "PG_DATABASE"):
monkeypatch.delenv(var, raising=False)

create_example_project(tmp_path)
(tmp_path / "config.yaml").write_text(
"""project: cli_test

gateways:
prod:
state_connection:
type: postgres
host: "{{ env_var('PG_HOST', 'postgres.internal.example.com') }}"
port: 5432
user: "{{ env_var('PG_USER') }}"
password: "{{ env_var('PG_PASSWORD') }}"
database: "{{ env_var('PG_DATABASE', 'sqlmesh_state') }}"
connection:
type: duckdb
database: "warehouse.db"

default_gateway: prod

model_defaults:
dialect: duckdb
""",
encoding="utf-8",
)

mock = _patch_state_access(mocker)

result = runner.invoke(cli, ["--paths", str(tmp_path), "test", "--local"])
assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}"
mock.assert_not_called()


def test_test_local_multi_repo_partial(runner: CliRunner, copy_to_temp_path, mocker) -> None:
"""Run tests for one repo of a multi-repo project whose upstream models live only in state.

Pins the behavioral difference against `lint --local`: a model that isn't loaded produces a
warning and its test is skipped, rather than turning into an error.
"""
repo_2 = copy_to_temp_path("examples/multi")[0] / "repo_2"

# silver.c lives in repo_2 and its upstream bronze.a is supplied as a test input.
(repo_2 / "tests" / "test_c.yaml").write_text(
"""test_silver_c:
model: silver.c
inputs:
bronze.a:
rows:
- col_a: 1
- col_a: 1
- col_a: 2
outputs:
query:
rows:
- col_a: 1
- col_a: 2
""",
encoding="utf-8",
)
# bronze.a itself is defined in repo_1, so it is not loaded when only repo_2 is given.
(repo_2 / "tests" / "test_a.yaml").write_text(
"""test_bronze_a:
model: bronze.a
outputs:
query:
rows:
- col_a: 1
""",
encoding="utf-8",
)

mock = _patch_state_access(mocker)
args = ["--gateway", "memory", "--paths", str(repo_2), "test"]

# Without --local the same run reaches the state backend.
runner.invoke(cli, args)
assert mock.called, "state-sync was never accessed during `test`"

mock.reset_mock()

result = runner.invoke(cli, [*args, "--local"])

assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}"
# Console output wraps, so compare against whitespace-normalized text.
output = " ".join(result.output.split())
assert 'Model \'"memory"."bronze"."a"\' was not found' in output, (
"the unloaded model should warn rather than fail"
)
assert "Successfully Ran 1 tests" in output, "the repo_2 test should still run"
mock.assert_not_called()
Loading