From fca9a3c36af1e9e6f0b04db2a2fbf201c9ebe05f Mon Sep 17 00:00:00 2001 From: Vamsi-klu Date: Sat, 17 Oct 2026 11:06:58 +0000 Subject: [PATCH] fix(console): omit empty per-column table-diff samples Signed-off-by: Vamsi-klu --- sqlmesh/core/console.py | 2 + tests/core/test_console.py | 154 +++++++++++++++++++++++++++++++++- tests/core/test_table_diff.py | 43 ++++++++++ 3 files changed, 198 insertions(+), 1 deletion(-) diff --git a/sqlmesh/core/console.py b/sqlmesh/core/console.py index ca11b6e607..438eaa7a56 100644 --- a/sqlmesh/core/console.py +++ b/sqlmesh/core/console.py @@ -2602,6 +2602,8 @@ def show_row_diff( axis=1, ) ] + if column_table.empty: + continue # Rename the column headers for readability column_table = column_table.rename( diff --git a/tests/core/test_console.py b/tests/core/test_console.py index f899713235..f4c905a494 100644 --- a/tests/core/test_console.py +++ b/tests/core/test_console.py @@ -1,4 +1,156 @@ -from sqlmesh.core.console import MarkdownConsole +from io import StringIO + +import pytest +from rich.console import Console + +from sqlmesh.core.console import MarkdownConsole, TerminalConsole +from sqlmesh.core.table_diff import RowDiff + + +@pytest.fixture +def row_diff(): + import pandas as pd + + return RowDiff( + source="source", + target="target", + stats={ + "s_count": 2, + "t_count": 2, + "distinct_count_s": 2, + "distinct_count_t": 2, + "join_count": 2, + "full_match_count": 0, + "s_only_count": 0, + "t_only_count": 0, + "null_grain_count": 0, + }, + column_stats=pd.DataFrame( + {"pct_match": [50.0, 50.0, 100.0]}, index=["early", "late", "unchanged"] + ), + sample=pd.DataFrame(), + joined_sample=pd.DataFrame( + {"id": [1], "s__early": [10], "t__early": [99], "s__late": [20], "t__late": [20]} + ), + s_sample=pd.DataFrame(), + t_sample=pd.DataFrame(), + ) + + +@pytest.mark.parametrize("console_type", [TerminalConsole, MarkdownConsole]) +def test_show_row_diff_omits_empty_column_samples(row_diff, console_type): + output = StringIO() + console = console_type() + console.console = Console(file=output, width=120) + + console.show_row_diff(row_diff) + + rendered = output.getvalue() + assert "Column: early" in rendered + assert "99" in rendered + assert "Column: late" not in rendered + assert "Column: unchanged" not in rendered + assert "early" in rendered and "late" in rendered and "unchanged" in rendered + assert "50.0" in rendered and "100.0" in rendered + assert "PARTIAL MATCH: 2 rows (100.0%)" in rendered + + +@pytest.mark.parametrize("dtype", ["object", "Int64", "boolean", "float64"]) +def test_show_row_diff_omits_equal_null_samples(row_diff, dtype): + import pandas as pd + + nulls = pd.Series([float("nan")] if dtype == "float64" else [None], dtype=dtype) + row_diff = row_diff.copy( + update={"joined_sample": row_diff.joined_sample.assign(s__late=nulls, t__late=nulls.copy())} + ) + output = StringIO() + + TerminalConsole(console=Console(file=output)).show_row_diff(row_diff) + + assert "Column: early" in output.getvalue() + assert "Column: late" not in output.getvalue() + + +def test_show_row_diff_preserves_null_differences_aliases_keys_and_inputs(row_diff): + import pandas as pd + + row_diff = row_diff.copy( + update={ + "source_alias": "dev", + "target_alias": "prod", + "stats": { + **row_diff.stats, + "s_count": 3, + "t_count": 3, + "distinct_count_s": 3, + "distinct_count_t": 3, + "s_only_count": 1, + "t_only_count": 1, + }, + "joined_sample": pd.DataFrame( + { + "id": [1, 2], + "region": ["east", "west"], + "DEV__early": pd.Series([10, None], dtype="Int64"), + "PROD__early": pd.Series([None, 20], dtype="Int64"), + "DEV__late": pd.Series([True, None], dtype="boolean"), + "PROD__late": pd.Series([True, None], dtype="boolean"), + } + ), + "s_sample": pd.DataFrame({"id": [3], "region": ["source-only"]}), + "t_sample": pd.DataFrame({"id": [4], "region": ["target-only"]}), + } + ) + original_stats = row_diff.stats.copy() + frames = { + name: getattr(row_diff, name).copy(deep=True) + for name in ("sample", "joined_sample", "s_sample", "t_sample", "column_stats") + } + output = StringIO() + + TerminalConsole(console=Console(file=output, width=120)).show_row_diff(row_diff) + + rendered = output.getvalue() + assert "Column: early" in rendered + assert "Column: late" not in rendered + assert all(label in rendered for label in ("id", "region", "DEV", "PROD")) + assert all(value in rendered for value in ("east", "west", "10", "20", "")) + assert "DEV ONLY: 1 rows" in rendered and "PROD ONLY: 1 rows" in rendered + assert "DEV ONLY sample rows:" in rendered and "source-only" in rendered + assert "PROD ONLY sample rows:" in rendered and "target-only" in rendered + assert row_diff.stats == original_stats + for name, frame in frames.items(): + pd.testing.assert_frame_equal(getattr(row_diff, name), frame) + + +def test_show_row_diff_without_samples(row_diff): + output = StringIO() + + TerminalConsole(console=Console(file=output)).show_row_diff(row_diff, show_sample=False) + + rendered = output.getvalue() + assert "PARTIAL MATCH: 2 rows (100.0%)" in rendered + assert "column comparison stats:" in rendered + assert "late" in rendered and "unchanged" in rendered + assert "sample" not in rendered + assert "Column:" not in rendered + + +def test_show_row_diff_without_joined_sample(row_diff): + row_diff = row_diff.copy( + update={ + "joined_sample": row_diff.joined_sample.iloc[:0], + "stats": {**row_diff.stats, "full_match_count": 2}, + } + ) + output = StringIO() + + TerminalConsole(console=Console(file=output)).show_row_diff(row_diff) + + rendered = output.getvalue() + assert "FULL MATCH: 2 rows (100.0%)" in rendered + assert "All joined rows match" in rendered + assert "Column:" not in rendered def test_markdown_console_warning_block(): diff --git a/tests/core/test_table_diff.py b/tests/core/test_table_diff.py index c2e293e4c2..65fbdd6c81 100644 --- a/tests/core/test_table_diff.py +++ b/tests/core/test_table_diff.py @@ -1197,6 +1197,49 @@ def test_data_diff_sample_limit(): assert len(diff.joined_sample) == 3 +@pytest.mark.parametrize("limit", [1, 2]) +def test_data_diff_sample_omits_empty_column_tables(limit): + engine_adapter = DuckDBConnectionConfig().create_engine_adapter() + try: + engine_adapter.execute( + "CREATE TABLE src AS SELECT * FROM (VALUES (1, 10, 20, 30), (2, 10, 20, 30)) " + "AS records(id, early, late, unchanged)" + ) + engine_adapter.execute( + "CREATE TABLE target AS SELECT * FROM (VALUES (1, 99, 20, 30), (2, 10, 99, 30)) " + "AS records(id, early, late, unchanged)" + ) + table_diff = TableDiff( + adapter=engine_adapter, source="src", target="target", on=["id"], limit=limit + ) + + row_diff = table_diff.row_diff() + assert row_diff.partial_match_count == 2 + assert row_diff.column_stats["pct_match"].to_dict() == { + "early": 50.0, + "late": 50.0, + "unchanged": 100.0, + } + assert row_diff.joined_sample["id"].tolist() == list(range(1, limit + 1)) + assert "s__late" in row_diff.joined_sample + assert "s__unchanged" not in row_diff.joined_sample + original_stats = row_diff.column_stats.copy(deep=True) + original_sample = row_diff.joined_sample.copy(deep=True) + + output = strip_ansi_codes(capture_console_output("show_row_diff", row_diff=row_diff)) + + assert "Column: early" in output + assert ("Column: late" in output) == (limit == 2) + assert "Column: unchanged" not in output + assert "99" in output + assert "PARTIAL MATCH: 2 rows (100.0%)" in output + assert "unchanged" in output and "100.0" in output + pd.testing.assert_frame_equal(row_diff.column_stats, original_stats) + pd.testing.assert_frame_equal(row_diff.joined_sample, original_sample) + finally: + engine_adapter.close() + + def test_data_diff_nulls_in_some_grain_columns(): engine_adapter = DuckDBConnectionConfig().create_engine_adapter()