From 96d6930bd98656ba917eeb80833b1ee0d03a3fe8 Mon Sep 17 00:00:00 2001 From: Andrey Tatarinov Date: Sat, 23 Apr 2022 00:19:50 +0300 Subject: [PATCH 1/5] feat: connection_string config property use case: heroku postgres --- tap_postgres/__init__.py | 49 ++++++++++++++++++++++++++---------- tap_postgres/config_utils.py | 15 +++++++++++ 2 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 tap_postgres/config_utils.py diff --git a/tap_postgres/__init__.py b/tap_postgres/__init__.py index 5c7c2ac1..0bea32da 100644 --- a/tap_postgres/__init__.py +++ b/tap_postgres/__init__.py @@ -1,6 +1,7 @@ import argparse import itertools import copy +import urllib.parse import psycopg2 import psycopg2.extras import psycopg2.extensions @@ -20,15 +21,21 @@ from tap_postgres.stream_utils import ( dump_catalog, clear_state_on_replication_change, is_selected_via_metadata, refresh_streams_schema, any_logical_streams) +from tap_postgres.config_utils import check_config_for_key_groups LOGGER = singer.get_logger('tap_postgres') -REQUIRED_CONFIG_KEYS = [ - 'dbname', - 'host', - 'port', - 'user', - 'password' +REQUIRED_CONFIG_KEYS_GROUPS = [ + [ + 'dbname', + 'host', + 'port', + 'user', + 'password' + ], + [ + 'connection_string' + ] ] @@ -381,7 +388,7 @@ def parse_args(required_config_keys): setattr(args, 'catalog_path', args.catalog) args.catalog = Catalog.load(args.catalog) - utils.check_config(args.config, required_config_keys) + check_config_for_key_groups(args.config, required_config_keys) return args @@ -390,14 +397,30 @@ def main_impl(): """ Main method """ - args = parse_args(REQUIRED_CONFIG_KEYS) + args = parse_args(REQUIRED_CONFIG_KEYS_GROUPS) + + if 'connection_string' in args.config: + p = urllib.parse.urlparse(args.config['connection_string']) + + host = p.hostname + dbname = p.path[1:] # Strip starting "/" + user = p.username + password = p.password + port = p.port + else: + host = args.config['host'] + dbname = args.config['dbname'] + user = args.config['user'] + password = args.config['password'] + port = args.config['port'] + conn_config = { # Required config keys - 'host': args.config['host'], - 'user': args.config['user'], - 'password': args.config['password'], - 'port': args.config['port'], - 'dbname': args.config['dbname'], + 'host': host, + 'user': user, + 'password': password, + 'port': port, + 'dbname': dbname, # Optional config keys 'tap_id': args.config.get('tap_id'), diff --git a/tap_postgres/config_utils.py b/tap_postgres/config_utils.py new file mode 100644 index 00000000..7c3a05dc --- /dev/null +++ b/tap_postgres/config_utils.py @@ -0,0 +1,15 @@ +def check_config_for_key_groups(config, required_key_groups): + missing_key_groups = [] + + for required_keys in required_key_groups: + missing_keys = [key for key in required_keys if key not in config] + if missing_keys: + missing_key_groups.append(missing_keys) + + # At least one group is complete no need to check further + return + + if missing_key_groups: + missing_keys_msg = ' or '.join(str(i) for i in missing_key_groups) + raise Exception( + "Config is missing required keys: {}".format(missing_keys_msg)) From e4ca552e5fa5ad4bdc6de9bcb00e569746bd942e Mon Sep 17 00:00:00 2001 From: Andrey Tatarinov Date: Sat, 23 Apr 2022 00:28:18 +0300 Subject: [PATCH 2/5] fix linting errors --- tap_postgres/__init__.py | 12 ++++++------ tap_postgres/config_utils.py | 11 +++++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tap_postgres/__init__.py b/tap_postgres/__init__.py index 0bea32da..5e3ba079 100644 --- a/tap_postgres/__init__.py +++ b/tap_postgres/__init__.py @@ -400,13 +400,13 @@ def main_impl(): args = parse_args(REQUIRED_CONFIG_KEYS_GROUPS) if 'connection_string' in args.config: - p = urllib.parse.urlparse(args.config['connection_string']) + parsed_conn_str = urllib.parse.urlparse(args.config['connection_string']) - host = p.hostname - dbname = p.path[1:] # Strip starting "/" - user = p.username - password = p.password - port = p.port + host = parsed_conn_str.hostname + dbname = parsed_conn_str.path[1:] # Strip starting "/" + user = parsed_conn_str.username + password = parsed_conn_str.password + port = parsed_conn_str.port else: host = args.config['host'] dbname = args.config['dbname'] diff --git a/tap_postgres/config_utils.py b/tap_postgres/config_utils.py index 7c3a05dc..fd17ce47 100644 --- a/tap_postgres/config_utils.py +++ b/tap_postgres/config_utils.py @@ -1,15 +1,18 @@ def check_config_for_key_groups(config, required_key_groups): + """ + Checks if any required group of keys is present in config. + If none of the groups is present in full - raise Exception. + """ missing_key_groups = [] for required_keys in required_key_groups: missing_keys = [key for key in required_keys if key not in config] if missing_keys: missing_key_groups.append(missing_keys) - + # At least one group is complete no need to check further return if missing_key_groups: - missing_keys_msg = ' or '.join(str(i) for i in missing_key_groups) - raise Exception( - "Config is missing required keys: {}".format(missing_keys_msg)) + missing_keys_msg = " or ".join(str(i) for i in missing_key_groups) + raise Exception("Config is missing required keys: {}".format(missing_keys_msg)) From 28c960c6b944dcf688fe438d3cb73e1b888f2742 Mon Sep 17 00:00:00 2001 From: Andrey Tatarinov Date: Sat, 23 Apr 2022 00:30:43 +0300 Subject: [PATCH 3/5] fix linting errors --- tap_postgres/config_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_postgres/config_utils.py b/tap_postgres/config_utils.py index fd17ce47..4a1e3b9a 100644 --- a/tap_postgres/config_utils.py +++ b/tap_postgres/config_utils.py @@ -15,4 +15,4 @@ def check_config_for_key_groups(config, required_key_groups): if missing_key_groups: missing_keys_msg = " or ".join(str(i) for i in missing_key_groups) - raise Exception("Config is missing required keys: {}".format(missing_keys_msg)) + raise Exception(f"Config is missing required keys: {missing_keys_msg}") From 4718e34f13f5b47ffe9c5b8591a4aaa94a5beb35 Mon Sep 17 00:00:00 2001 From: Andrey Tatarinov Date: Sat, 23 Apr 2022 00:40:53 +0300 Subject: [PATCH 4/5] exclude main_impl from code cov --- tap_postgres/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_postgres/__init__.py b/tap_postgres/__init__.py index 5e3ba079..7d7b74d7 100644 --- a/tap_postgres/__init__.py +++ b/tap_postgres/__init__.py @@ -393,7 +393,7 @@ def parse_args(required_config_keys): return args -def main_impl(): +def main_impl(): # pragma: no cover """ Main method """ From 0c5a3b5fa036f1fc3bc978ee8c96b6a8dc17f0e4 Mon Sep 17 00:00:00 2001 From: Andrey Tatarinov Date: Sat, 23 Apr 2022 01:22:04 +0300 Subject: [PATCH 5/5] rename connection_string -> connection_uri --- tap_postgres/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_postgres/__init__.py b/tap_postgres/__init__.py index 7d7b74d7..a378f9b3 100644 --- a/tap_postgres/__init__.py +++ b/tap_postgres/__init__.py @@ -399,8 +399,8 @@ def main_impl(): # pragma: no cover """ args = parse_args(REQUIRED_CONFIG_KEYS_GROUPS) - if 'connection_string' in args.config: - parsed_conn_str = urllib.parse.urlparse(args.config['connection_string']) + if 'connection_uri' in args.config: + parsed_conn_str = urllib.parse.urlparse(args.config['connection_uri']) host = parsed_conn_str.hostname dbname = parsed_conn_str.path[1:] # Strip starting "/"