From a4e2695b7701b861d88ff264b51acfcb8ebb4911 Mon Sep 17 00:00:00 2001 From: Shockwave Date: Tue, 13 Sep 2022 20:18:10 +0800 Subject: [PATCH 1/5] Sub: Add sql_output_adapter Body: ==== End ==== --- .gitignore | 4 + .../tabular_output/sql_output_adapter.py | 89 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 cli_helpers/tabular_output/sql_output_adapter.py diff --git a/.gitignore b/.gitignore index 213f266..7132e97 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ __pycache__ /cli_helpers_dev .idea/ .cache/ +.vscode/ +**/.ropeproject/ +*.swp + diff --git a/cli_helpers/tabular_output/sql_output_adapter.py b/cli_helpers/tabular_output/sql_output_adapter.py new file mode 100644 index 0000000..589b2d6 --- /dev/null +++ b/cli_helpers/tabular_output/sql_output_adapter.py @@ -0,0 +1,89 @@ +# coding=utf-8 + +supported_formats = ( + "sql-insert", + "sql-update", + "sql-update-1", + "sql-update-2", +) + +preprocessors = () + + +def escape_for_sql_statement(value): + if isinstance(value, bytes): + return f"X'{value.hex()}'" + else: + return "'{}'".format(value) + + +def adapter(data, headers, table_format=None, **kwargs): + """ + This function registers supported_formats to default TabularOutputFormatter + + Parameters: + data: query result + headers: columns + table_format: values from supported_formats + kwargs: + tables: parsed from clis + delimeter: Character surrounds table name or column name when it conflicts with sql keywords. + For example, mysql uses ` and postgres uses " + """ + # tables = extract_tables(formatter.query) + tables = kwargs.get("tables") + delimeter = kwargs.get("delimeter") + if not isinstance(delimeter, str): + delimeter = '"' + + if isinstance(tables, list) and len(tables) > 0: + table = tables[0] + if table[0]: + table_name = "{}.{}".format(*table[:2]) + else: + table_name = table[1] + else: + table_name = '{delimeter}DUAL{delimeter}'.format(delimeter=delimeter) + + header_joiner = '{delimeter}, {delimeter}'.format(delimeter=delimeter) + if table_format == "sql-insert": + h = header_joiner.join(headers) + yield 'INSERT INTO {delimeter}{table_name}{delimeter} ({delimeter}{header}{delimeter}) VALUES'.format( + table_name=table_name, header=h, delimeter=delimeter) + prefix = " " + for d in data: + values = ", ".join(escape_for_sql_statement(v) for i, v in enumerate(d)) + yield "{}({})".format(prefix, values) + if prefix == " ": + prefix = ", " + yield ";" + if table_format.startswith("sql-update"): + s = table_format.split("-") + keys = 1 + if len(s) > 2: + keys = int(s[-1]) + for d in data: + yield 'UPDATE {delimeter}{table_name}{delimeter} SET'.format(table_name=table_name, delimeter=delimeter) + prefix = " " + for i, v in enumerate(d[keys:], keys): + yield '{prefix}{delimeter}{column}{delimeter} = {value}'.format( + prefix=prefix, delimeter=delimeter, column=headers[i], value=escape_for_sql_statement(v) + ) + if prefix == " ": + prefix = ", " + f = '{delimeter}{column}{delimeter}" = {value}' + where = ( + f.format(delimeter=delimeter, column=headers[i], value=escape_for_sql_statement(d[i])) + for i in range(keys) + ) + yield "WHERE {};".format(" AND ".join(where)) + + + +def register_new_formatter(TabularOutputFormatter): + global formatter + formatter = TabularOutputFormatter + for sql_format in supported_formats: + TabularOutputFormatter.register_new_formatter( + sql_format, adapter, preprocessors, {"table_format": sql_format} + ) From 67c4dc33fb5dce0c28a6cb6c358c794040db930a Mon Sep 17 00:00:00 2001 From: Shockwave Date: Mon, 19 Sep 2022 19:56:13 +0800 Subject: [PATCH 2/5] Sub: Add unit test for sql adapter Body: ==== End ==== --- .../tabular_output/sql_output_adapter.py | 44 +++--- .../tabular_output/test_sql_output_adapter.py | 147 ++++++++++++++++++ 2 files changed, 172 insertions(+), 19 deletions(-) create mode 100644 tests/tabular_output/test_sql_output_adapter.py diff --git a/cli_helpers/tabular_output/sql_output_adapter.py b/cli_helpers/tabular_output/sql_output_adapter.py index 589b2d6..f144f85 100644 --- a/cli_helpers/tabular_output/sql_output_adapter.py +++ b/cli_helpers/tabular_output/sql_output_adapter.py @@ -1,4 +1,4 @@ -# coding=utf-8 +# -*- coding: utf-8 -*- supported_formats = ( "sql-insert", @@ -26,30 +26,30 @@ def adapter(data, headers, table_format=None, **kwargs): headers: columns table_format: values from supported_formats kwargs: - tables: parsed from clis - delimeter: Character surrounds table name or column name when it conflicts with sql keywords. + tables: tuple parsed from clis. Example: (TableReference(schema=None, name='user', alias='"user"', is_function=False),) + delimiter: Character surrounds table name or column name when it conflicts with sql keywords. For example, mysql uses ` and postgres uses " """ # tables = extract_tables(formatter.query) tables = kwargs.get("tables") - delimeter = kwargs.get("delimeter") - if not isinstance(delimeter, str): - delimeter = '"' + delimiter = kwargs.get("delimiter") + if not isinstance(delimiter, str): + delimiter = '"' - if isinstance(tables, list) and len(tables) > 0: + if tables is not None and len(tables) > 0: table = tables[0] if table[0]: table_name = "{}.{}".format(*table[:2]) else: table_name = table[1] else: - table_name = '{delimeter}DUAL{delimeter}'.format(delimeter=delimeter) + table_name = 'DUAL'.format(delimiter=delimiter) - header_joiner = '{delimeter}, {delimeter}'.format(delimeter=delimeter) + header_joiner = '{delimiter}, {delimiter}'.format(delimiter=delimiter) if table_format == "sql-insert": h = header_joiner.join(headers) - yield 'INSERT INTO {delimeter}{table_name}{delimeter} ({delimeter}{header}{delimeter}) VALUES'.format( - table_name=table_name, header=h, delimeter=delimeter) + yield 'INSERT INTO {delimiter}{table_name}{delimiter} ({delimiter}{header}{delimiter}) VALUES'.format( + table_name=table_name, header=h, delimiter=delimiter) prefix = " " for d in data: values = ", ".join(escape_for_sql_statement(v) for i, v in enumerate(d)) @@ -63,27 +63,33 @@ def adapter(data, headers, table_format=None, **kwargs): if len(s) > 2: keys = int(s[-1]) for d in data: - yield 'UPDATE {delimeter}{table_name}{delimeter} SET'.format(table_name=table_name, delimeter=delimeter) + yield 'UPDATE {delimiter}{table_name}{delimiter} SET'.format(table_name=table_name, delimiter=delimiter) prefix = " " for i, v in enumerate(d[keys:], keys): - yield '{prefix}{delimeter}{column}{delimeter} = {value}'.format( - prefix=prefix, delimeter=delimeter, column=headers[i], value=escape_for_sql_statement(v) + yield '{prefix}{delimiter}{column}{delimiter} = {value}'.format( + prefix=prefix, delimiter=delimiter, column=headers[i], value=escape_for_sql_statement(v) ) if prefix == " ": prefix = ", " - f = '{delimeter}{column}{delimeter}" = {value}' + f = '{delimiter}{column}{delimiter} = {value}' where = ( - f.format(delimeter=delimeter, column=headers[i], value=escape_for_sql_statement(d[i])) + f.format(delimiter=delimiter, column=headers[i], value=escape_for_sql_statement(d[i])) for i in range(keys) ) yield "WHERE {};".format(" AND ".join(where)) - -def register_new_formatter(TabularOutputFormatter): +def register_new_formatter(TabularOutputFormatter, **kwargs): + """ + Parameters: + TabularOutputFormatter: default TabularOutputFormatter imported from cli_helpers + kwargs: dict required, with key delimiter and tables required. + For example {"delimiter": "`", "tables": ["table_name"]} + """ global formatter formatter = TabularOutputFormatter for sql_format in supported_formats: + kwargs["table_format"] = sql_format TabularOutputFormatter.register_new_formatter( - sql_format, adapter, preprocessors, {"table_format": sql_format} + sql_format, adapter, preprocessors, kwargs ) diff --git a/tests/tabular_output/test_sql_output_adapter.py b/tests/tabular_output/test_sql_output_adapter.py new file mode 100644 index 0000000..ff832fa --- /dev/null +++ b/tests/tabular_output/test_sql_output_adapter.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- + +from collections import namedtuple + +from cli_helpers.tabular_output import TabularOutputFormatter +from cli_helpers.tabular_output.sql_output_adapter import escape_for_sql_statement, adapter, register_new_formatter + +TableReference = namedtuple( + "TableReference", ["schema", "name", "alias", "is_function"] +) + +TableReference.ref = property( + lambda self: self.alias + or ( + self.name + if self.name.islower() or self.name[0] == '"' + else '"' + self.name + '"' + ) +) + + +def test_escape_for_sql_statement_bytes(): + bts = b"837124ab3e8dc0f" + escaped_bytes = escape_for_sql_statement(bts) + assert escaped_bytes == "X'383337313234616233653864633066'" + + +def test_output_sql_insert(): + global formatter + formatter = TabularOutputFormatter + register_new_formatter(formatter) + data = [ + [ + 1, + "Jackson", + "jackson_test@gmail.com", + "132454789", + "", + "2022-09-09 19:44:32.712343+08", + "2022-09-09 19:44:32.712343+08", + ] + ] + header = ["id", "name", "email", "phone", "description", "created_at", "updated_at"] + table_format = "sql-insert" + table_refs = (TableReference(schema=None, name='user', alias='"user"', is_function=False),) + kwargs = { + "column_types": [int, str, str, str, str, str, str], + "sep_title": "RECORD {n}", + "sep_character": "-", + "sep_length": (1, 25), + "missing_value": "", + "integer_format": "", + "float_format": "", + "disable_numparse": True, + "preserve_whitespace": True, + "max_field_width": 500, + "tables": table_refs, + } + + formatter.query = 'SELECT * FROM "user";' + # For postgresql + kwargs["delimiter"] = '"' + output = adapter(data, header, table_format=table_format, **kwargs) + output_list = [l for l in output] + expected = [ + 'INSERT INTO "user" ("id", "name", "email", "phone", "description", "created_at", "updated_at") VALUES', + " ('1', 'Jackson', 'jackson_test@gmail.com', '132454789', '', " + + "'2022-09-09 19:44:32.712343+08', '2022-09-09 19:44:32.712343+08')", + ";", + ] + assert expected == output_list + + # For mysql + kwargs["delimiter"] = "`" + output = adapter(data, header, table_format=table_format, **kwargs) + output_list = [l for l in output] + expected = [ + 'INSERT INTO `user` (`id`, `name`, `email`, `phone`, `description`, `created_at`, `updated_at`) VALUES', + " ('1', 'Jackson', 'jackson_test@gmail.com', '132454789', '', " + + "'2022-09-09 19:44:32.712343+08', '2022-09-09 19:44:32.712343+08')", + ";", + ] + assert expected == output_list + + +def test_output_sql_update_pg(): + global formatter + formatter = TabularOutputFormatter + register_new_formatter(formatter) + data = [ + [ + 1, + "Jackson", + "jackson_test@gmail.com", + "132454789", + "", + "2022-09-09 19:44:32.712343+08", + "2022-09-09 19:44:32.712343+08", + ] + ] + header = ["id", "name", "email", "phone", "description", "created_at", "updated_at"] + table_format = "sql-update" + table_refs = (TableReference(schema=None, name='user', alias='"user"', is_function=False),) + kwargs = { + "column_types": [int, str, str, str, str, str, str], + "sep_title": "RECORD {n}", + "sep_character": "-", + "sep_length": (1, 25), + "missing_value": "", + "integer_format": "", + "float_format": "", + "disable_numparse": True, + "preserve_whitespace": True, + "max_field_width": 500, + "tables": table_refs, + } + formatter.query = 'SELECT * FROM "user";' + # For postgresql + kwargs["delimiter"] = '"' + output = adapter(data, header, table_format=table_format, **kwargs) + output_list = [l for l in output] + expected = [ + 'UPDATE "user" SET', + ' "name" = \'Jackson\'', + ', "email" = \'jackson_test@gmail.com\'', + ', "phone" = \'132454789\'', + ', "description" = \'\'', + ', "created_at" = \'2022-09-09 19:44:32.712343+08\'', + ', "updated_at" = \'2022-09-09 19:44:32.712343+08\'', + 'WHERE "id" = \'1\';'] + assert expected == output_list + + # For mysql + kwargs["delimiter"] = "`" + output = adapter(data, header, table_format=table_format, **kwargs) + output_list = [l for l in output] + print(output_list) + expected = [ + 'UPDATE `user` SET', + " `name` = 'Jackson'", + ", `email` = 'jackson_test@gmail.com'", + ", `phone` = '132454789'", + ", `description` = ''", + ", `created_at` = '2022-09-09 19:44:32.712343+08'", + ", `updated_at` = '2022-09-09 19:44:32.712343+08'", + "WHERE `id` = '1';"] + assert expected == output_list From 97df810fb055c596a027251c7cc94f5ee9e7f38b Mon Sep 17 00:00:00 2001 From: Shockwave Date: Sat, 22 Oct 2022 15:09:39 +0800 Subject: [PATCH 3/5] Sub: Change kwargs for adapter, try to pass func Body: ==== End ==== --- .../tabular_output/sql_output_adapter.py | 11 +++++++---- .../tabular_output/test_sql_output_adapter.py | 18 +++++++++++++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/cli_helpers/tabular_output/sql_output_adapter.py b/cli_helpers/tabular_output/sql_output_adapter.py index f144f85..16fec07 100644 --- a/cli_helpers/tabular_output/sql_output_adapter.py +++ b/cli_helpers/tabular_output/sql_output_adapter.py @@ -26,12 +26,15 @@ def adapter(data, headers, table_format=None, **kwargs): headers: columns table_format: values from supported_formats kwargs: - tables: tuple parsed from clis. Example: (TableReference(schema=None, name='user', alias='"user"', is_function=False),) + extract_tables: extract_tables function. For example, in pgcli.packages.parseutils.tables there is a function extract_tables delimiter: Character surrounds table name or column name when it conflicts with sql keywords. For example, mysql uses ` and postgres uses " """ - # tables = extract_tables(formatter.query) - tables = kwargs.get("tables") + extract_table_func = kwargs.get('extract_tables') + if not extract_table_func: + raise ValueError('extract_tables function should be registered first') + + tables = extract_table_func(formatter.query) delimiter = kwargs.get("delimiter") if not isinstance(delimiter, str): delimiter = '"' @@ -84,7 +87,7 @@ def register_new_formatter(TabularOutputFormatter, **kwargs): Parameters: TabularOutputFormatter: default TabularOutputFormatter imported from cli_helpers kwargs: dict required, with key delimiter and tables required. - For example {"delimiter": "`", "tables": ["table_name"]} + For example {"delimiter": "`", "extact_tables": extract_tables} """ global formatter formatter = TabularOutputFormatter diff --git a/tests/tabular_output/test_sql_output_adapter.py b/tests/tabular_output/test_sql_output_adapter.py index ff832fa..0e4c5c9 100644 --- a/tests/tabular_output/test_sql_output_adapter.py +++ b/tests/tabular_output/test_sql_output_adapter.py @@ -25,6 +25,19 @@ def test_escape_for_sql_statement_bytes(): assert escaped_bytes == "X'383337313234616233653864633066'" +def __mock_extract_tables(sql): + """ + mock function for extract tables + in mycli, pass `mycli.packages.parseutils.extract_tables` + in pgcli, pass `pgcli.packages.parseutils.extract_tables` + + :param sql: sql query + :return: + """ + table_refs = (TableReference(schema=None, name='user', alias='"user"', is_function=False),) + return table_refs + + def test_output_sql_insert(): global formatter formatter = TabularOutputFormatter @@ -42,7 +55,6 @@ def test_output_sql_insert(): ] header = ["id", "name", "email", "phone", "description", "created_at", "updated_at"] table_format = "sql-insert" - table_refs = (TableReference(schema=None, name='user', alias='"user"', is_function=False),) kwargs = { "column_types": [int, str, str, str, str, str, str], "sep_title": "RECORD {n}", @@ -54,7 +66,7 @@ def test_output_sql_insert(): "disable_numparse": True, "preserve_whitespace": True, "max_field_width": 500, - "tables": table_refs, + "extract_tables": __mock_extract_tables, } formatter.query = 'SELECT * FROM "user";' @@ -112,7 +124,7 @@ def test_output_sql_update_pg(): "disable_numparse": True, "preserve_whitespace": True, "max_field_width": 500, - "tables": table_refs, + "extract_tables": __mock_extract_tables, } formatter.query = 'SELECT * FROM "user";' # For postgresql From c1038de912fbb815a1232024c6b5f3c8e14c940b Mon Sep 17 00:00:00 2001 From: Shockwave Date: Sat, 22 Oct 2022 15:48:23 +0800 Subject: [PATCH 4/5] Sub: Update changelog and AUTHORS, and `black` is done Body: ==== End ==== --- AUTHORS | 1 + CHANGELOG | 5 +++ .../tabular_output/sql_output_adapter.py | 32 ++++++++++++------- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/AUTHORS b/AUTHORS index 40f2b90..d3aae74 100644 --- a/AUTHORS +++ b/AUTHORS @@ -25,6 +25,7 @@ This project receives help from these awesome contributors: - Mel Dafert - Andrii Kohut - Roland Walker +- Liu Zhao (astroshot) Thanks ------ diff --git a/CHANGELOG b/CHANGELOG index da2b436..3663614 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,11 @@ Changelog ========= +Features: +--------- + +* New formatter is added to export query result to sql format (such as sql-insert, sql-update). + TBD ------------- * don't escape newlines, etc. in ascii tables, and add ascii_escaped table format diff --git a/cli_helpers/tabular_output/sql_output_adapter.py b/cli_helpers/tabular_output/sql_output_adapter.py index 16fec07..eb673cf 100644 --- a/cli_helpers/tabular_output/sql_output_adapter.py +++ b/cli_helpers/tabular_output/sql_output_adapter.py @@ -30,9 +30,9 @@ def adapter(data, headers, table_format=None, **kwargs): delimiter: Character surrounds table name or column name when it conflicts with sql keywords. For example, mysql uses ` and postgres uses " """ - extract_table_func = kwargs.get('extract_tables') + extract_table_func = kwargs.get("extract_tables") if not extract_table_func: - raise ValueError('extract_tables function should be registered first') + raise ValueError("extract_tables function should be registered first") tables = extract_table_func(formatter.query) delimiter = kwargs.get("delimiter") @@ -46,13 +46,14 @@ def adapter(data, headers, table_format=None, **kwargs): else: table_name = table[1] else: - table_name = 'DUAL'.format(delimiter=delimiter) + table_name = "DUAL".format(delimiter=delimiter) - header_joiner = '{delimiter}, {delimiter}'.format(delimiter=delimiter) + header_joiner = "{delimiter}, {delimiter}".format(delimiter=delimiter) if table_format == "sql-insert": h = header_joiner.join(headers) - yield 'INSERT INTO {delimiter}{table_name}{delimiter} ({delimiter}{header}{delimiter}) VALUES'.format( - table_name=table_name, header=h, delimiter=delimiter) + yield "INSERT INTO {delimiter}{table_name}{delimiter} ({delimiter}{header}{delimiter}) VALUES".format( + table_name=table_name, header=h, delimiter=delimiter + ) prefix = " " for d in data: values = ", ".join(escape_for_sql_statement(v) for i, v in enumerate(d)) @@ -66,17 +67,26 @@ def adapter(data, headers, table_format=None, **kwargs): if len(s) > 2: keys = int(s[-1]) for d in data: - yield 'UPDATE {delimiter}{table_name}{delimiter} SET'.format(table_name=table_name, delimiter=delimiter) + yield "UPDATE {delimiter}{table_name}{delimiter} SET".format( + table_name=table_name, delimiter=delimiter + ) prefix = " " for i, v in enumerate(d[keys:], keys): - yield '{prefix}{delimiter}{column}{delimiter} = {value}'.format( - prefix=prefix, delimiter=delimiter, column=headers[i], value=escape_for_sql_statement(v) + yield "{prefix}{delimiter}{column}{delimiter} = {value}".format( + prefix=prefix, + delimiter=delimiter, + column=headers[i], + value=escape_for_sql_statement(v), ) if prefix == " ": prefix = ", " - f = '{delimiter}{column}{delimiter} = {value}' + f = "{delimiter}{column}{delimiter} = {value}" where = ( - f.format(delimiter=delimiter, column=headers[i], value=escape_for_sql_statement(d[i])) + f.format( + delimiter=delimiter, + column=headers[i], + value=escape_for_sql_statement(d[i]), + ) for i in range(keys) ) yield "WHERE {};".format(" AND ".join(where)) From 7e9ca0a6852e013ffb5ccdc622fd3e63f05e6fb1 Mon Sep 17 00:00:00 2001 From: Shockwave Date: Sat, 22 Oct 2022 16:11:45 +0800 Subject: [PATCH 5/5] Sub: Optimize code style Body: ==== End ==== --- CHANGELOG | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 3663614..faa759c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,9 +1,8 @@ Changelog ========= -Features: ---------- - +Features +------------- * New formatter is added to export query result to sql format (such as sql-insert, sql-update). TBD