Add support for including and ignoring columns in table comparisons and enhance CLI options

Author: luisggoncalves

sqlcompare/compare/comparator.py

@@ -21,6 +21,59 @@ def __init__(self, connection: Any):
         self.cols_new: list[str] = []
         self.common_cols: list[str] = []
 
+    @staticmethod
+    def _resolve_compare_columns(
+        common_cols: Sequence[str],
+        include_columns: Sequence[str] | None,
+        ignore_columns: Sequence[str] | None,
+    ) -> list[str]:
+        selected_cols = list(common_cols)
+
+        if include_columns:
+            include_matches: list[str] = []
+            missing_includes: list[str] = []
+            seen: set[str] = set()
+            for raw_col in include_columns:
+                col = raw_col.strip()
+                if not col:
+                    continue
+                match = next((c for c in common_cols if c.upper() == col.upper()), None)
+                if not match:
+                    missing_includes.append(raw_col)
+                    continue
+                key = match.upper()
+                if key in seen:
+                    continue
+                seen.add(key)
+                include_matches.append(match)
+            if missing_includes:
+                raise ValueError(
+                    "Included columns not found in both tables: "
+                    + ", ".join(missing_includes)
+                )
+            selected_cols = include_matches
+
+        if ignore_columns:
+            ignore_keys: set[str] = set()
+            missing_ignores: list[str] = []
+            for raw_col in ignore_columns:
+                col = raw_col.strip()
+                if not col:
+                    continue
+                match = next((c for c in common_cols if c.upper() == col.upper()), None)
+                if not match:
+                    missing_ignores.append(raw_col)
+                    continue
+                ignore_keys.add(match.upper())
+            if missing_ignores:
+                raise ValueError(
+                    "Ignored columns not found in both tables: "
+                    + ", ".join(missing_ignores)
+                )
+            selected_cols = [c for c in selected_cols if c.upper() not in ignore_keys]
+
+        return selected_cols
+
     def _ensure_schema(self, db: DBConnection, test_schema: str) -> None:
         """Ensure the test schema exists without changing the database context permanently."""
         try:
@@ -60,15 +113,19 @@ def from_saved(
         index_cols: Sequence[str],
         cols_prev: Sequence[str],
         cols_new: Sequence[str],
+        common_cols: Sequence[str] | None = None,
     ) -> "DatabaseComparator":
         inst = cls(connection)
         inst.tables = tables
         inst.index_cols = list(index_cols)
         inst.cols_prev = list(cols_prev)
         inst.cols_new = list(cols_new)
-        inst.common_cols = [
-            c for c in inst.cols_prev if c in inst.cols_new and c not in inst.index_cols
-        ]
+        if common_cols is not None:
+            inst.common_cols = list(common_cols)
+        else:
+            inst.common_cols = [
+                c for c in inst.cols_prev if c in inst.cols_new and c not in inst.index_cols
+            ]
         return inst
 
     # ------------------------------------------------------------------
@@ -157,6 +214,8 @@ def compare(
         index_cols: Sequence[str],
         test_name: str,
         test_schema: str = "sqlcompare",
+        include_columns: Sequence[str] | None = None,
+        ignore_columns: Sequence[str] | None = None,
     ) -> str:
         """Compare two tables in the database."""
         if not index_cols:
@@ -251,9 +310,14 @@ def compare(
                 ]
             )
             self.index_cols = valid_index_cols
-            self.common_cols = [
+            all_common_cols = [
                 c for c in cols_prev if c in cols_new and c not in self.index_cols
             ]
+            self.common_cols = self._resolve_compare_columns(
+                all_common_cols,
+                include_columns=include_columns,
+                ignore_columns=ignore_columns,
+            )
 
             # Build join table with side specific columns
             select_cols = [f'p."{c}" AS "{c}_previous"' for c in cols_prev]
@@ -365,6 +429,7 @@ def compare(
             "index_cols": list(self.index_cols),
             "cols_prev": self.cols_prev,
             "cols_new": self.cols_new,
+            "common_cols": self.common_cols,
             "conn": conn_name,
         }
 

sqlcompare/inspect.py

@@ -36,10 +36,11 @@ def inspect_diff(
         idx = run.get("index_cols", [])
         cols_prev = run.get("cols_prev", [])
         cols_new = run.get("cols_new", [])
+        common_cols = run.get("common_cols")
         if "duckdb_file" in run:
             db_file = run["duckdb_file"]
             comp = DatabaseComparator.from_saved(
-                db_file, tables, idx, cols_prev, cols_new
+                db_file, tables, idx, cols_prev, cols_new, common_cols
             )
             # Use DBConnection with duckdb URL
             conn_url = f"duckdb:///{db_file}"
@@ -105,7 +106,9 @@ def inspect_diff(
                 )
             return
         conn = run["conn"]
-        comp = DatabaseComparator.from_saved(conn, tables, idx, cols_prev, cols_new)
+        comp = DatabaseComparator.from_saved(
+            conn, tables, idx, cols_prev, cols_new, common_cols
+        )
         with DBConnection(conn) as db:
             if stats:
                 q = comp.get_stats_query(column=column)

sqlcompare/table.py

@@ -17,6 +17,8 @@ def compare_table(
     index: str,
     connection: str | None,
     schema: str | None,
+    include_columns: str | None = None,
+    ignore_columns: str | None = None,
 ) -> None:
     """Compare two tables in the database.
 
@@ -64,8 +66,18 @@ def compare_table(
 
         connection = connection_id
 
-    # Parse index columns
+    # Parse index and comparison columns
     id_cols = [x.strip() for x in index.split(",")]
+    include_cols = (
+        [x.strip() for x in include_columns.split(",") if x.strip()]
+        if include_columns
+        else None
+    )
+    ignore_cols = (
+        [x.strip() for x in ignore_columns.split(",") if x.strip()]
+        if ignore_columns
+        else None
+    )
 
     # Generate a test name based on tables
     safe_t1 = "".join(c if c.isalnum() else "_" for c in table1)
@@ -74,7 +86,15 @@ def compare_table(
 
     # Run comparison
     comparator = DatabaseComparator(connection)
-    diff_id = comparator.compare(table1, table2, id_cols, test_name, schema)
+    diff_id = comparator.compare(
+        table1,
+        table2,
+        id_cols,
+        test_name,
+        schema,
+        include_columns=include_cols,
+        ignore_columns=ignore_cols,
+    )
     log.info(f"🔎 To review the diff, run: sqlcompare inspect {diff_id}")
     log.info(
         "💡 Tips: --stats for per-column counts, --missing-current/--missing-previous for row-only, "
@@ -94,6 +114,16 @@ def table_cmd(
         None, "--connection", "-c", help="Database connector name"
     ),
     schema: str | None = typer.Option(None, "--schema", help="Schema for test tables"),
+    columns: str | None = typer.Option(
+        None,
+        "--columns",
+        help="Comma-separated non-index columns to compare (default: all common columns)",
+    ),
+    ignore_columns: str | None = typer.Option(
+        None,
+        "--ignore-columns",
+        help="Comma-separated non-index columns to skip from comparison",
+    ),
 ) -> None:
     """Compare two database tables or CSV/XLSX files.
 
@@ -121,4 +151,12 @@ def table_cmd(
         2. SQLCOMPARE_CONN_DEFAULT environment variable
         3. For files: auto-creates temporary DuckDB instance
     """
-    compare_table(table1, table2, index, connection, schema)
+    compare_table(
+        table1,
+        table2,
+        index,
+        connection,
+        schema,
+        include_columns=columns,
+        ignore_columns=ignore_columns,
+    )

tests/test_cli_table.py

@@ -60,3 +60,71 @@ def test_table_command_with_files(tmp_path, monkeypatch) -> None:
     assert "inspect" in result.output
     runs = load_test_runs()
     assert len(runs) == 1
+
+
+def test_table_command_ignore_columns_option(tmp_path, monkeypatch) -> None:
+    db_path = tmp_path / "sqlcompare.duckdb"
+    seed_duckdb(db_path)
+    config_dir = tmp_path / "config"
+    set_cli_env(
+        monkeypatch,
+        config_dir,
+        "duckdb_test",
+        f"duckdb:///{db_path}",
+        schema="analysis_schema",
+    )
+    runner = CliRunner()
+
+    result = runner.invoke(
+        app,
+        [
+            "table",
+            "previous",
+            "current",
+            "id",
+            "--connection",
+            "duckdb_test",
+            "--ignore-columns",
+            "value",
+        ],
+    )
+
+    assert result.exit_code == 0, result.output
+    runs = load_test_runs()
+    assert len(runs) == 1
+    run = next(iter(runs.values()))
+    assert run["common_cols"] == ["name"]
+
+
+def test_table_command_columns_option(tmp_path, monkeypatch) -> None:
+    db_path = tmp_path / "sqlcompare.duckdb"
+    seed_duckdb(db_path)
+    config_dir = tmp_path / "config"
+    set_cli_env(
+        monkeypatch,
+        config_dir,
+        "duckdb_test",
+        f"duckdb:///{db_path}",
+        schema="analysis_schema",
+    )
+    runner = CliRunner()
+
+    result = runner.invoke(
+        app,
+        [
+            "table",
+            "previous",
+            "current",
+            "id",
+            "--connection",
+            "duckdb_test",
+            "--columns",
+            "value",
+        ],
+    )
+
+    assert result.exit_code == 0, result.output
+    runs = load_test_runs()
+    assert len(runs) == 1
+    run = next(iter(runs.values()))
+    assert run["common_cols"] == ["value"]