From 435ecb51af9a394df6d7189cb479d462f5deb38e Mon Sep 17 00:00:00 2001 From: Anand Hegde Date: Sat, 19 Sep 2026 16:54:23 +0530 Subject: [PATCH] Add psql's \crosstabview \crosstabview [colV [colH [colD [sortcolH]]]] runs the query and shows the result as a crosstab grid: the distinct values of colV down the side, of colH across the top, colD in the cells. It follows psql's crosstabview.c: columns by name (case-folded unless quoted) or number, the first three columns by default, sortcolH to order the horizontal header by an integer column (ties in name order), and psql's error messages for too few columns, ambiguous or unknown columns, and more than one value per cell. " \crosstabview" is handled in PGExecute.run(), like \G, and also ends the buffer in multi-line mode. A bare \crosstabview re-runs the last query sent to the server, as in psql. Refs #1378, dbcli/pgspecial#142 --- AUTHORS | 1 + changelog.rst | 10 ++- pgcli/crosstabview.py | 133 +++++++++++++++++++++++++++++++++++++ pgcli/main.py | 19 +++++- pgcli/pgbuffer.py | 2 + pgcli/pgexecute.py | 35 ++++++++++ tests/test_crosstabview.py | 83 +++++++++++++++++++++++ tests/test_main.py | 65 ++++++++++++++++++ tests/test_pgexecute.py | 9 +++ 9 files changed, 355 insertions(+), 2 deletions(-) create mode 100644 pgcli/crosstabview.py create mode 100644 tests/test_crosstabview.py diff --git a/AUTHORS b/AUTHORS index f235e11bc..899f6ddec 100644 --- a/AUTHORS +++ b/AUTHORS @@ -156,6 +156,7 @@ Contributors: * Chris (ChrisJr404) * Pieter Ouwerkerk (pouwerkerk) * VXNCXNX + * Anand Hegde (anandghegde) Creator: -------- diff --git a/changelog.rst b/changelog.rst index 8c4e03756..319155754 100644 --- a/changelog.rst +++ b/changelog.rst @@ -1,7 +1,15 @@ Upcoming ======== - +Features: +--------- +* Add psql's ``\crosstabview [colV [colH [colD [sortcolH]]]]``: run the query + and show the result as a crosstab grid, with the values of ``colV`` down the + side, the values of ``colH`` across the top and ``colD`` in the cells. + Columns are given by name or number and default to the first three, ``sortcolH`` + orders the columns by an integer column, and a bare ``\crosstabview`` re-runs + the last query. The errors match psql's + ([issue 1378](https://github.com/dbcli/pgcli/issues/1378)). 4.7.0 (2026-09-19) ================== diff --git a/pgcli/crosstabview.py b/pgcli/crosstabview.py new file mode 100644 index 000000000..f4f86b2a9 --- /dev/null +++ b/pgcli/crosstabview.py @@ -0,0 +1,133 @@ +r"""psql's \crosstabview: show a query result as a crosstab (pivot) grid. + +The rules and error messages follow psql's crosstabview.c. +""" + +import re + +# psql's CROSSTABVIEW_MAX_COLUMNS +MAX_COLUMNS = 1600 + +# " \crosstabview [args]". An arg is a word or a "quoted" name; a single +# quote in the tail means the \crosstabview is inside a string literal. +_TERMINATOR = re.compile(r'^(.*?)\s*\\crosstabview((?:\s+(?:"[^"]*"|[^\s"\'])+)*)\s*$', re.DOTALL) + + +class CrosstabViewError(Exception): + pass + + +def split_query(sql): + r"""Split " \crosstabview [args]" into (query, args), or None.""" + match = _TERMINATOR.match(sql) + if match is None: + return None + return match.group(1), match.group(2) + + +def parse_args(text): + """Split the arguments on whitespace, keeping "quoted" names whole.""" + return re.findall(r'(?:"[^"]*"|[^\s"])+', text or "") + + +def _dequote_downcase(arg): + """Like psql: downcase unquoted letters, strip quotes, "" means ".""" + out = [] + quoted = False + i = 0 + while i < len(arg): + c = arg[i] + if c == '"': + if quoted and arg[i + 1 : i + 2] == '"': + out.append('"') + i += 1 + else: + quoted = not quoted + else: + out.append(c if quoted or not "A" <= c <= "Z" else c.lower()) + i += 1 + return "".join(out) + + +def _column_index(arg, headers): + """Resolve a column given by number (1-based) or by name.""" + if arg.isdigit() and arg.isascii(): + idx = int(arg) - 1 + if not 0 <= idx < len(headers): + raise CrosstabViewError(f"\\crosstabview: column number {idx + 1} is out of range 1..{len(headers)}") + return idx + name = _dequote_downcase(arg) + matches = [i for i, header in enumerate(headers) if header == name] + if len(matches) > 1: + raise CrosstabViewError(f'\\crosstabview: ambiguous column name: "{name}"') + if not matches: + raise CrosstabViewError(f'\\crosstabview: column name not found: "{name}"') + return matches[0] + + +def _key(value): + # psql compares the text of the values; NULL is a value of its own. + return None if value is None else str(value) + + +def _rank(value): + # A sort value counts only when it is an integer; anything else ranks 0. + text = _key(value) + return int(text) if text is not None and re.fullmatch(r"-?[0-9]+", text) else 0 + + +def crosstabview(headers, rows, args): + """Pivot rows like psql's \\crosstabview colV colH [colD [sortcolH]]. + + Returns (rows, headers). A NULL horizontal header is returned as None; + cells with no data value are empty strings. + """ + if len(headers) < 3: + raise CrosstabViewError("\\crosstabview: query must return at least three columns") + args = list(args[:4]) + [None] * (4 - len(args[:4])) + + col_v = 0 if args[0] is None else _column_index(args[0], headers) + col_h = 1 if args[1] is None else _column_index(args[1], headers) + if col_v == col_h: + raise CrosstabViewError("\\crosstabview: vertical and horizontal headers must be different columns") + if args[2] is None: + if len(headers) != 3: + raise CrosstabViewError("\\crosstabview: data column must be specified when query returns more than three columns") + col_d = 3 - col_v - col_h # the column that is left + else: + col_d = _column_index(args[2], headers) + col_sort = None if args[3] is None else _column_index(args[3], headers) + + # Distinct header values, in order of first appearance. + h_ranks = {} + v_labels = {} + for row in rows: + h = _key(row[col_h]) + if h not in h_ranks: + h_ranks[h] = 0 if col_sort is None else _rank(row[col_sort]) + if len(h_ranks) > MAX_COLUMNS: + raise CrosstabViewError(f"\\crosstabview: maximum number of columns ({MAX_COLUMNS}) exceeded") + v_labels.setdefault(_key(row[col_v]), row[col_v]) + + h_order = list(h_ranks) + if col_sort is not None: + # Sort by rank; ties are in name order (NULL last), as in psql. + h_order.sort(key=lambda h: (h is None, h or "")) + h_order.sort(key=h_ranks.get) + h_index = {h: i for i, h in enumerate(h_order, 1)} + v_index = {v: i for i, v in enumerate(v_labels)} + + empty = object() + grid = [[label] + [empty] * len(h_order) for label in v_labels.values()] + for row in rows: + v, h = _key(row[col_v]), _key(row[col_h]) + cells = grid[v_index[v]] + if cells[h_index[h]] is not empty: + raise CrosstabViewError( + f'\\crosstabview: query result contains multiple data values for row "{"(null)" if v is None else v}", ' + f'column "{"(null)" if h is None else h}"' + ) + cells[h_index[h]] = row[col_d] + + grid = [[("" if cell is empty else cell) for cell in cells] for cells in grid] + return grid, [headers[col_v]] + h_order diff --git a/pgcli/main.py b/pgcli/main.py index 3ff820ce2..2c1edb5eb 100644 --- a/pgcli/main.py +++ b/pgcli/main.py @@ -500,6 +500,13 @@ def refresh_callback(): "Echo a string to the query output channel.", ) + self.pgspecial.register( + self.crosstabview, + "\\crosstabview", + "\\crosstabview [colV [colH [colD [sortcolH]]]]", + "Execute query (or re-run the last one) and display the result in a crosstab.", + ) + self.pgspecial.register( self.toggle_verbose_errors, "\\v", @@ -507,6 +514,11 @@ def refresh_callback(): "Toggle verbose errors.", ) + def crosstabview(self, pattern, **_): + # A bare \crosstabview re-runs the last query; " \crosstabview" + # is handled by PGExecute.run(). + return [self.pgexecute.crosstabview(None, pattern)] + def toggle_verbose_errors(self, pattern, **_): flag = pattern.strip() @@ -1296,6 +1308,10 @@ def _should_limit_output(self, sql, cur): if not is_select(sql): return False + # A \crosstabview grid is not a cursor; like psql, it is never truncated. + if not hasattr(cur, "rowcount"): + return False + return not self._has_limit(sql) and self.row_limit != 0 and cur and cur.rowcount > self.row_limit def _has_limit(self, sql): @@ -2194,7 +2210,8 @@ def format_status(cur, status): output.append(title) if cur: - headers = [] if settings.tuples_only else [case_function(x) for x in headers] + # A \crosstabview header can be NULL; show it like a NULL value. + headers = [] if settings.tuples_only else [settings.missingval if x is None else case_function(x) for x in headers] if max_width is not None: cur = list(cur) column_types = None diff --git a/pgcli/pgbuffer.py b/pgcli/pgbuffer.py index d6b5096d1..351bc17e2 100644 --- a/pgcli/pgbuffer.py +++ b/pgcli/pgbuffer.py @@ -4,6 +4,7 @@ from prompt_toolkit.enums import DEFAULT_BUFFER from prompt_toolkit.filters import Condition from prompt_toolkit.application import get_app +from .crosstabview import split_query from .packages.parseutils.utils import is_open_quote _logger = logging.getLogger(__name__) @@ -53,6 +54,7 @@ def cond(): text.startswith("\\") or text.endswith((r"\e", r"\G")) or _is_complete(text) + or split_query(text) is not None or text == "exit" or text == "quit" or text == ":q" diff --git a/pgcli/pgexecute.py b/pgcli/pgexecute.py index 1cac82eb4..600a3b788 100644 --- a/pgcli/pgexecute.py +++ b/pgcli/pgexecute.py @@ -12,6 +12,7 @@ sqlparse.engine.grouping.MAX_GROUPING_DEPTH = None sqlparse.engine.grouping.MAX_GROUPING_TOKENS = None +from .crosstabview import CrosstabViewError, crosstabview, parse_args, split_query from .packages.parseutils.meta import FunctionMetadata, ForeignKey _logger = logging.getLogger(__name__) @@ -182,6 +183,7 @@ def __init__( self.notify_callback = notify_callback self.connect(database, user, password, host, port, dsn, **kwargs) self.reset_expanded = None + self.last_sql = None def is_virtual_database(self): if self._is_virtual_database is None: @@ -387,6 +389,15 @@ def run( self.reset_expanded = True sql = sql[:-2].strip() + # " \crosstabview [args]" runs the query and pivots it. + crosstab = split_query(sql) + if crosstab and crosstab[0]: + if explain_mode: + sql = crosstab[0] + else: + yield self.crosstabview(*crosstab) + (sql, True, False) + continue + # First try to run each query as special _logger.debug("Trying a pgspecial command. sql: %r", sql) try: @@ -418,6 +429,10 @@ def run( sql = self.explain_prefix() + sql yield self.execute_normal_sql(sql) + (sql, True, False) + except CrosstabViewError as e: + yield None, None, None, str(e), sql, False, False + if not on_error_resume: + break except psycopg.DatabaseError as e: _logger.error("sql: %r, error: %r", sql, e) _logger.error("traceback: %r", traceback.format_exc()) @@ -451,6 +466,7 @@ def _must_raise(self, e): def execute_normal_sql(self, split_sql): """Returns tuple (title, rows, headers, status)""" _logger.debug("Regular sql statement. sql: %r", split_sql) + self.last_sql = split_sql title = "" @@ -485,6 +501,25 @@ def handle_notices(n): _logger.debug("No rows in result.") return title, None, None, cur.statusmessage + def crosstabview(self, sql, args): + """Run sql (or the last query, like psql) and pivot the result. + + Returns tuple (title, rows, headers, status) + """ + sql = sql or self.last_sql + if not sql: + return None, None, None, "\\crosstabview: there is no previous query to run" + args = parse_args(args) + title, cur, headers, status = self.execute_normal_sql(sql) + if not headers: + # Not a result set: show it as usual, like psql does. + return title, cur, headers, status + rows, headers = crosstabview(headers, cur.fetchall(), args) + warnings = [f'\\crosstabview: extra argument "{arg}" ignored' for arg in args[4:]] + title = "\n".join(filter(None, [title] + warnings)) + # An iterator, so an empty grid still prints its header like psql. + return title, iter(rows), headers, f"SELECT {len(rows)}" + def search_path(self): """Returns the current search path as a list of schema names""" diff --git a/tests/test_crosstabview.py b/tests/test_crosstabview.py new file mode 100644 index 000000000..eea5487ec --- /dev/null +++ b/tests/test_crosstabview.py @@ -0,0 +1,83 @@ +import pytest + +from pgcli.crosstabview import CrosstabViewError, crosstabview, parse_args, split_query + +HEADERS = ["first", "second", "gt2"] +ROWS = [(1, "one", False), (2, "two", False), (3, "three", True), (4, "four", True)] + + +def test_split_query(): + assert split_query("select 1") is None + assert split_query("select 1 \\crosstabview") == ("select 1", "") + assert split_query("select 1\n\\crosstabview a \"B c\"") == ("select 1", ' a "B c"') + assert split_query("\\crosstabview 1 2") == ("", " 1 2") + # inside a string literal + assert split_query("select 'x \\crosstabview y'") is None + + +def test_parse_args(): + assert parse_args(' a "B c" 3 "fO""o"') == ["a", '"B c"', "3", '"fO""o"'] + assert parse_args("") == [] + + +def test_defaults(): + # first column vertical, second horizontal, third data + rows, headers = crosstabview(HEADERS, ROWS, []) + assert headers == ["first", "one", "two", "three", "four"] + assert rows == [ + [1, False, "", "", ""], + [2, "", False, "", ""], + [3, "", "", True, ""], + [4, "", "", "", True], + ] + + +def test_columns_by_name_and_number(): + rows, headers = crosstabview(HEADERS, ROWS, ["second", "1", "GT2"]) + assert headers == ["second", "1", "2", "3", "4"] + assert rows[0] == ["one", False, "", "", ""] + + +def test_quoted_names_are_case_sensitive(): + headers = ["Foo", "b", "c"] + assert crosstabview(headers, [(1, 2, 3)], ['"Foo"'])[1] == ["Foo", "2"] + with pytest.raises(CrosstabViewError, match=r'column name not found: "foo"'): + crosstabview(headers, [(1, 2, 3)], ["Foo"]) + # a quoted number is a name, and "" is an escaped quote + assert crosstabview(["1", 'fO"o', "c"], [(1, 2, 3)], ['"fO""o"', '"1"'])[1] == ['fO"o', "1"] + + +def test_sort_column(): + headers = ["a", "b", "c", "s"] + rows = [("x", "b", 1, "z"), ("x", "a", 2, "-3"), ("y", "c", 3, None), ("y", "d", 3, "1.5"), ("y", "e", 4, "-1")] + grid, headers = crosstabview(headers, rows, ["a", "b", "c", "s"]) + # integer ranks sort; anything else ranks 0, ties in name order + assert headers == ["a", "a", "e", "b", "c", "d"] + assert grid == [["x", 2, "", 1, "", ""], ["y", "", 4, "", 3, 3]] + + +def test_nulls(): + rows = [(1, 2, None), (1, None, 4), (None, 3, 5), (2, 3, 6)] + grid, headers = crosstabview(["a", "b", "c"], rows, []) + # A NULL data value stays None; a cell without data is empty. + assert headers == ["a", "2", None, "3"] + assert grid == [[1, None, 4, ""], [None, "", "", 5], [2, "", "", 6]] + + +@pytest.mark.parametrize( + "headers, rows, args, message", + [ + (["a", "b"], [], [], "query must return at least three columns"), + (["a", "b", "c", "d"], [], [], "data column must be specified when query returns more than three columns"), + (["a", "b", "c"], [], ["a", "a"], "vertical and horizontal headers must be different columns"), + (["a", "b", "c"], [], ["5"], r"column number 5 is out of range 1\.\.3"), + (["a", "b", "c"], [], ["0"], r"column number 0 is out of range 1\.\.3"), + (["a", "a", "c"], [], ["a"], 'ambiguous column name: "a"'), + (["a", "b", "c"], [(1, 2, 3), (1, 2, 4)], [], 'multiple data values for row "1", column "2"'), + (["a", "b", "c"], [(1, None, 3), (1, None, 4)], [], r'multiple data values for row "1", column "\(null\)"'), + (["a", "b", "c"], [(1, i, 1) for i in range(1601)], [], r"maximum number of columns \(1600\) exceeded"), + ], +) +def test_errors(headers, rows, args, message): + with pytest.raises(CrosstabViewError, match=message): + crosstabview(headers, rows, args) diff --git a/tests/test_main.py b/tests/test_main.py index f18319312..4a1f216b4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1058,3 +1058,68 @@ def test_connect_timeout_config_value_must_be_a_number(tmpdir): f.write("[main]\nconnect_timeout = soon\n") with pytest.raises(ValueError): PGCli(pgclirc_file=rc) + + +CROSSTAB_QUERY = "SELECT * FROM (VALUES (1, 'one', false), (2, 'two', false), (3, 'three', true)) v(first, second, gt2)" + + +def crosstab(cli, text): + output, query = cli._evaluate_command(text) + return [COLOR_CODE_REGEX.sub("", line) for line in output], query + + +@dbtest +def test_crosstabview(executor): + cli = PGCli(pgexecute=executor) + output, query = crosstab(cli, CROSSTAB_QUERY + " \\crosstabview first second") + assert query.successful + assert output == [ + "+-------+-------+-------+-------+", + "| first | one | two | three |", + "|-------+-------+-------+-------|", + "| 1 | False | | |", + "| 2 | | False | |", + "| 3 | | | True |", + "+-------+-------+-------+-------+", + "SELECT 3", + ] + + +@dbtest +def test_crosstabview_reruns_last_query(executor): + cli = PGCli(pgexecute=executor) + output, _ = crosstab(cli, "\\crosstabview") + assert output == ["\\crosstabview: there is no previous query to run"] + + cli._evaluate_command(CROSSTAB_QUERY) + cli._evaluate_command("\\dt") # not a query sent to the server + output, query = crosstab(cli, "\\crosstabview 2 1") + assert query.successful + assert output[1] == "| second | 1 | 2 | 3 |" + + # the same, in one go + output, _ = crosstab(cli, CROSSTAB_QUERY + "; \\crosstabview 2 1") + assert output[-7] == "| second | 1 | 2 | 3 |" + + +@dbtest +def test_crosstabview_error(executor): + cli = PGCli(pgexecute=executor) + output, query = crosstab(cli, "SELECT 1, 2 \\crosstabview") + assert not query.successful + assert output == ["\\crosstabview: query must return at least three columns"] + + +@dbtest +def test_crosstabview_not_a_result_set(executor): + cli = PGCli(pgexecute=executor) + output, query = crosstab(cli, "SET search_path = public \\crosstabview") + assert query.successful + assert output == ["SET"] + + +@dbtest +def test_crosstabview_is_not_row_limited(executor): + cli = PGCli(pgexecute=executor, row_limit=2) + output, _ = crosstab(cli, CROSSTAB_QUERY + " \\crosstabview") + assert output[-1] == "SELECT 3" diff --git a/tests/test_pgexecute.py b/tests/test_pgexecute.py index c5fcaa2cd..a213da1ea 100644 --- a/tests/test_pgexecute.py +++ b/tests/test_pgexecute.py @@ -811,6 +811,15 @@ def test_explain_mode_strips_G_suffix(executor, pgspecial): assert "\\G" not in sent +@dbtest +def test_explain_mode_strips_crosstabview(executor, pgspecial): + """`select ... \\crosstabview` explains the query in explain mode.""" + with patch.object(executor, "execute_normal_sql", return_value=("", None, None, "")) as normal_sql: + list(executor.run("select 1, 2, 3 \\crosstabview", pgspecial=pgspecial, explain_mode=True)) + + assert normal_sql.call_args.args[0] == executor.explain_prefix() + "select 1, 2, 3" + + @dbtest def test_exit_without_active_connection(executor): quit_handler = MagicMock()