From 28ff6d08afffe2afe1314e9c928d5ba54a305d4e Mon Sep 17 00:00:00 2001 From: daminichopra Date: Tue, 11 Jan 2022 08:35:23 +0000 Subject: [PATCH 01/13] added ql version in metadata --- src/translators/sql_translator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/translators/sql_translator.py b/src/translators/sql_translator.py index 115136bb..7c1947ec 100644 --- a/src/translators/sql_translator.py +++ b/src/translators/sql_translator.py @@ -712,7 +712,8 @@ def _update_metadata_table(self, table_name, metadata): # Attempt to re-create metadata table self._create_metadata_table() persisted_metadata = {} - + + metadata['ql_version'] = (ql_version ,"Text") diff = metadata.keys() - persisted_metadata.keys() if diff: # we update using the difference to "not" corrupt the metadata From b273ade7c974e3cff4a581f0fbceaeb0f794a7e6 Mon Sep 17 00:00:00 2001 From: Damini Date: Tue, 11 Jan 2022 14:10:49 +0530 Subject: [PATCH 02/13] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f3604184..f77f8dad 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -5,6 +5,7 @@ ### New features - Removed subscription API (#493) +- Added quantum leap version in metadata (#651) ### Bug fixes From f84096f8195dc6df17df640daad7d40b8375f15f Mon Sep 17 00:00:00 2001 From: Damini Date: Tue, 11 Jan 2022 18:53:55 +0530 Subject: [PATCH 03/13] Update sql_translator.py --- src/translators/sql_translator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/translators/sql_translator.py b/src/translators/sql_translator.py index 7c1947ec..253e344e 100644 --- a/src/translators/sql_translator.py +++ b/src/translators/sql_translator.py @@ -713,7 +713,7 @@ def _update_metadata_table(self, table_name, metadata): self._create_metadata_table() persisted_metadata = {} - metadata['ql_version'] = (ql_version ,"Text") + metadata['ql_version'] = (ql_version, "Text") diff = metadata.keys() - persisted_metadata.keys() if diff: # we update using the difference to "not" corrupt the metadata From f17d172d46ac9d0777184adfd216c5060700891a Mon Sep 17 00:00:00 2001 From: Damini Date: Tue, 11 Jan 2022 21:50:58 +0530 Subject: [PATCH 04/13] Update sql_translator.py --- src/translators/sql_translator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/translators/sql_translator.py b/src/translators/sql_translator.py index 253e344e..df2c7019 100644 --- a/src/translators/sql_translator.py +++ b/src/translators/sql_translator.py @@ -35,6 +35,7 @@ TENANT_PREFIX = 'mt' TYPE_PREFIX = 'et' TIME_INDEX = 'timeindex' +ql_version = "0.8.3" VALID_AGGR_METHODS = ['count', 'sum', 'avg', 'min', 'max'] VALID_AGGR_PERIODS = ['year', 'month', 'day', 'hour', 'minute', 'second'] # The name of the column where we store the original JSON entity received From e92bd48866fe9561016ad1b216aefcbbfde2e596 Mon Sep 17 00:00:00 2001 From: Damini Date: Wed, 12 Jan 2022 03:08:02 +0530 Subject: [PATCH 05/13] Update test_timescale_insert.py --- src/translators/tests/test_timescale_insert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/translators/tests/test_timescale_insert.py b/src/translators/tests/test_timescale_insert.py index 31aad8c9..684c8cb9 100644 --- a/src/translators/tests/test_timescale_insert.py +++ b/src/translators/tests/test_timescale_insert.py @@ -104,6 +104,7 @@ def expected_entity_attrs_meta(): return { 'entity_id': ['id', 'Text'], 'entity_type': ['type', 'Text'], + 'ql_version': ['0.8.3', 'Text'], TIME_INDEX_NAME: ['time_index', 'DateTime'], 'a_number': ['a_number', 'Number'], 'an_integer': ['an_integer', 'Integer'], From 74ef5b1652c285f515d4faef1b1f47b898467662 Mon Sep 17 00:00:00 2001 From: daminichopra Date: Tue, 18 Jan 2022 19:21:21 +0530 Subject: [PATCH 06/13] Incoperated review comments --- src/_version.py | 6 ++++++ src/reporter/version.py | 4 ++-- src/translators/sql_translator.py | 4 ++-- src/translators/tests/test_timescale_insert.py | 3 ++- 4 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 src/_version.py diff --git a/src/_version.py b/src/_version.py new file mode 100644 index 00000000..f48ededb --- /dev/null +++ b/src/_version.py @@ -0,0 +1,6 @@ +# Most recent official release +__version_info__ = ('0', '8', '3') +__version__ = '.'.join(__version_info__) + +# Current dev version for the next release +__dev_version__ = '0.9.0-dev' diff --git a/src/reporter/version.py b/src/reporter/version.py index 20ea9efa..ef0ce07a 100644 --- a/src/reporter/version.py +++ b/src/reporter/version.py @@ -1,6 +1,6 @@ - +from _version import __dev_version__ def version(): return { - 'version': '0.9.0-dev' + 'version': __dev_version__ } diff --git a/src/translators/sql_translator.py b/src/translators/sql_translator.py index df2c7019..3c7eb1cf 100644 --- a/src/translators/sql_translator.py +++ b/src/translators/sql_translator.py @@ -7,6 +7,7 @@ from utils.common import iter_entity_attrs from utils.jsondict import safe_get_value from utils.maybe import maybe_map +from _version import __version__ import logging from geocoding.slf import SlfQuery import dateutil.parser @@ -35,7 +36,6 @@ TENANT_PREFIX = 'mt' TYPE_PREFIX = 'et' TIME_INDEX = 'timeindex' -ql_version = "0.8.3" VALID_AGGR_METHODS = ['count', 'sum', 'avg', 'min', 'max'] VALID_AGGR_PERIODS = ['year', 'month', 'day', 'hour', 'minute', 'second'] # The name of the column where we store the original JSON entity received @@ -714,7 +714,7 @@ def _update_metadata_table(self, table_name, metadata): self._create_metadata_table() persisted_metadata = {} - metadata['ql_version'] = (ql_version, "Text") + metadata['ql_version'] = (__version__, "Text") diff = metadata.keys() - persisted_metadata.keys() if diff: # we update using the difference to "not" corrupt the metadata diff --git a/src/translators/tests/test_timescale_insert.py b/src/translators/tests/test_timescale_insert.py index 684c8cb9..74f456b0 100644 --- a/src/translators/tests/test_timescale_insert.py +++ b/src/translators/tests/test_timescale_insert.py @@ -2,6 +2,7 @@ import pg8000 import pytest import random +from _version import __version__ from time import sleep from geocoding.geojson.wktcodec import decode_wkb_hexstr @@ -104,8 +105,8 @@ def expected_entity_attrs_meta(): return { 'entity_id': ['id', 'Text'], 'entity_type': ['type', 'Text'], - 'ql_version': ['0.8.3', 'Text'], TIME_INDEX_NAME: ['time_index', 'DateTime'], + 'ql_version': [__version__, 'Text'], 'a_number': ['a_number', 'Number'], 'an_integer': ['an_integer', 'Integer'], 'a_bool': ['a_bool', 'Boolean'], From 2f75e0cbcc884789e961698c85c0bca17e35ca21 Mon Sep 17 00:00:00 2001 From: Damini Date: Tue, 18 Jan 2022 21:30:06 +0530 Subject: [PATCH 07/13] Update test --- src/reporter/tests/test_version.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/reporter/tests/test_version.py b/src/reporter/tests/test_version.py index 75f484f8..85975ba0 100644 --- a/src/reporter/tests/test_version.py +++ b/src/reporter/tests/test_version.py @@ -1,4 +1,5 @@ from conftest import QL_BASE_URL +from _version import __dev_version__ import requests @@ -7,5 +8,5 @@ def test_version(): r = requests.get('{}'.format(version_url)) assert r.status_code == 200, r.text assert r.json() == { - "version": "0.9.0-dev" + "version": __dev_version__ } From ff6060ffdba4b2fe59fffa37fdcfe1ec1859a922 Mon Sep 17 00:00:00 2001 From: Damini Date: Mon, 7 Feb 2022 22:26:00 +0530 Subject: [PATCH 08/13] Updating test --- src/translators/tests/test_insert.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/translators/tests/test_insert.py b/src/translators/tests/test_insert.py index d312d658..264b1428 100644 --- a/src/translators/tests/test_insert.py +++ b/src/translators/tests/test_insert.py @@ -6,6 +6,7 @@ from translators.base_translator import BaseTranslator from translators.sql_translator import NGSI_TEXT, NGSI_DATETIME, NGSI_STRUCTURED_VALUE from utils.common import * +from src._version import __version__ from utils.tests.common import * from datetime import datetime, timezone @@ -774,3 +775,24 @@ def test_ngsi_ld(translator, ngsi_ld): assert ngsi_ld['location']['value'] == loaded[0]['location']['values'][0] translator.clean() + +def expected_entity_attrs_meta_version(): + return { + 'entity_id': ['id', 'Text'], + 'entity_type': ['type', 'Text'], + TIME_INDEX_NAME: ['time_index', 'DateTime'], + 'ql_version': [__version__, 'Text'], + 'a_number': ['a_number', 'Number'], + 'an_integer': ['an_integer', 'Integer'], + 'a_bool': ['a_bool', 'Boolean'], + 'a_datetime': ['a_datetime', 'DateTime'], + 'a_point': ['a_point', 'geo:point'], + 'a_geom': ['a_geom', 'geo:json'], + 'a_text': ['a_text', 'Text'], + 'an_obj': ['an_obj', 'Custom'], + 'an_array': ['an_array', 'StructuredValue'] + } + +def assert_entity_attrs_meta_version(translator, entity): + data = select_entity_attrs_meta_version(translator, entity) + assert data == expected_entity_attrs_meta_version() From 88622bb27417b59124563260ca5616eb73bbf783 Mon Sep 17 00:00:00 2001 From: Damini Date: Tue, 15 Feb 2022 12:33:33 +0530 Subject: [PATCH 09/13] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 1 - 1 file changed, 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index b24bfa0e..179f78c9 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,7 +6,6 @@ - Removed subscription API (#493) - ### Bug fixes ### Continuous Integration From dc0a59803c24004a25fb8abb213d9d634b48cca1 Mon Sep 17 00:00:00 2001 From: Pooja Pathak <33315652+pooja1pathak@users.noreply.github.com> Date: Tue, 15 Mar 2022 00:22:08 +0530 Subject: [PATCH 10/13] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 410a55ec..2339ca6b 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -5,7 +5,14 @@ ### New features - Removed subscription API (#493) - +- Replaced geocoder with [geopy](https://geopy.readthedocs.io/en/stable/) (#610) +- Bumped pillow from 8.4.0 to 9.0.0 +- Aligned missing Fiware-servicePath behaviour with the one of Orion + Context Broker (#564). This is a breaking change! Before no value for + Fiware-servicePath was interpreted as python None, from now on, None + will be replaced with /. This affects only users that manually injected + data, since Orion, assume / when no servicePath is passed. +- Added quantum leap version in metadata (#615) ### Bug fixes From 6c0ef64259ab8cba248fbb43fd60261c4d4e2775 Mon Sep 17 00:00:00 2001 From: pooja1pathak Date: Tue, 15 Mar 2022 05:22:34 +0000 Subject: [PATCH 11/13] updated for lint --- src/reporter/version.py | 1 + src/translators/sql_translator.py | 2 +- src/translators/tests/test_insert.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/reporter/version.py b/src/reporter/version.py index ef0ce07a..6d29722f 100644 --- a/src/reporter/version.py +++ b/src/reporter/version.py @@ -1,5 +1,6 @@ from _version import __dev_version__ + def version(): return { 'version': __dev_version__ diff --git a/src/translators/sql_translator.py b/src/translators/sql_translator.py index 1cbb946c..870e6fdb 100644 --- a/src/translators/sql_translator.py +++ b/src/translators/sql_translator.py @@ -712,7 +712,7 @@ def _update_metadata_table(self, table_name, metadata): # Attempt to re-create metadata table self._create_metadata_table() persisted_metadata = {} - + metadata['ql_version'] = (__version__, "Text") diff = metadata.keys() - persisted_metadata.keys() if diff: diff --git a/src/translators/tests/test_insert.py b/src/translators/tests/test_insert.py index 264b1428..c818be56 100644 --- a/src/translators/tests/test_insert.py +++ b/src/translators/tests/test_insert.py @@ -776,6 +776,7 @@ def test_ngsi_ld(translator, ngsi_ld): translator.clean() + def expected_entity_attrs_meta_version(): return { 'entity_id': ['id', 'Text'], @@ -793,6 +794,7 @@ def expected_entity_attrs_meta_version(): 'an_array': ['an_array', 'StructuredValue'] } + def assert_entity_attrs_meta_version(translator, entity): data = select_entity_attrs_meta_version(translator, entity) assert data == expected_entity_attrs_meta_version() From c3e2ba79231448ac93050dfaca3d9a1663ebb17c Mon Sep 17 00:00:00 2001 From: pooja1pathak Date: Wed, 23 Mar 2022 07:52:11 +0000 Subject: [PATCH 12/13] updated for test cases --- src/translators/sql_translator.py | 16 +++++++++++++++ src/translators/tests/test_insert.py | 30 ++++++++++------------------ 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/translators/sql_translator.py b/src/translators/sql_translator.py index 870e6fdb..56059978 100644 --- a/src/translators/sql_translator.py +++ b/src/translators/sql_translator.py @@ -730,6 +730,22 @@ def _update_metadata_table(self, table_name, metadata): # This implementation paves # the way to lost updates... + def query_metadata_table(self): + # Bring translation table! + stmt = 'select entity_attrs from "{}" limit 100'.format( + METADATA_TABLE_NAME) + # By design, one entry per table_name + try: + res = self.cursor.execute(stmt) + row = self.cursor.fetchall() + persisted_metadata = row[0][0] if row else {} + except Exception as e: + self.sql_error_handler(e) + # Metadata table still not created + logging.debug(str(e), exc_info=True) + persisted_metadata = {} + return persisted_metadata + def _store_metadata(self, table_name, persisted_metadata): raise NotImplementedError diff --git a/src/translators/tests/test_insert.py b/src/translators/tests/test_insert.py index c818be56..e8803537 100644 --- a/src/translators/tests/test_insert.py +++ b/src/translators/tests/test_insert.py @@ -777,24 +777,16 @@ def test_ngsi_ld(translator, ngsi_ld): translator.clean() -def expected_entity_attrs_meta_version(): - return { - 'entity_id': ['id', 'Text'], - 'entity_type': ['type', 'Text'], - TIME_INDEX_NAME: ['time_index', 'DateTime'], - 'ql_version': [__version__, 'Text'], - 'a_number': ['a_number', 'Number'], - 'an_integer': ['an_integer', 'Integer'], - 'a_bool': ['a_bool', 'Boolean'], - 'a_datetime': ['a_datetime', 'DateTime'], - 'a_point': ['a_point', 'geo:point'], - 'a_geom': ['a_geom', 'geo:json'], - 'a_text': ['a_text', 'Text'], - 'an_obj': ['an_obj', 'Custom'], - 'an_array': ['an_array', 'StructuredValue'] - } +@pytest.mark.parametrize("translator", translators, ids=["crate", "timescale"]) +def test_entity_meta_version(translator): + entities = create_random_entities(1, 2, 3, use_time=True, use_geo=True) + result = translator.insert(entities) + assert result.rowcount > 0 + table_name = "et0" + data = translator.query_metadata_table() + observed = data['ql_version'] + expected = [__version__, 'Text'] -def assert_entity_attrs_meta_version(translator, entity): - data = select_entity_attrs_meta_version(translator, entity) - assert data == expected_entity_attrs_meta_version() + assert observed == expected + translator.clean() From bf03f1209bb14f3cf57d5a5addb7a76ceb2c6487 Mon Sep 17 00:00:00 2001 From: pooja1pathak Date: Mon, 16 May 2022 08:24:05 +0000 Subject: [PATCH 13/13] updated as per comment --- src/translators/crate.py | 8 ++++---- src/translators/sql_translator.py | 11 +++++------ src/translators/tests/test_insert.py | 6 +++--- src/translators/timescale.py | 8 ++++---- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/translators/crate.py b/src/translators/crate.py index 0aecc820..4644a052 100644 --- a/src/translators/crate.py +++ b/src/translators/crate.py @@ -241,19 +241,19 @@ def _build_original_data_value(self, entity: dict, def _create_metadata_table(self): stmt = "create table if not exists {} " \ - "(table_name string primary key, entity_attrs object) " \ + "(table_name string primary key, version text, entity_attrs object) " \ "with (" \ "number_of_replicas = '2-all', " \ "column_policy = 'dynamic')" op = stmt.format(METADATA_TABLE_NAME) self.cursor.execute(op) - def _store_metadata(self, table_name, persisted_metadata): - stmt = "insert into {} (table_name, entity_attrs) values (?,?) " \ + def _store_metadata(self, table_name, version, persisted_metadata): + stmt = "insert into {} (table_name, version, entity_attrs) values (?,?,?) " \ "on conflict(table_name) " \ "DO UPDATE SET entity_attrs = excluded.entity_attrs" stmt = stmt.format(METADATA_TABLE_NAME) - self.cursor.execute(stmt, (table_name, persisted_metadata)) + self.cursor.execute(stmt, (table_name, version, persisted_metadata)) def _compute_db_specific_type(self, attr_t, attr): """ diff --git a/src/translators/sql_translator.py b/src/translators/sql_translator.py index 56059978..133f7989 100644 --- a/src/translators/sql_translator.py +++ b/src/translators/sql_translator.py @@ -713,14 +713,13 @@ def _update_metadata_table(self, table_name, metadata): self._create_metadata_table() persisted_metadata = {} - metadata['ql_version'] = (__version__, "Text") diff = metadata.keys() - persisted_metadata.keys() if diff: # we update using the difference to "not" corrupt the metadata # by previous insert update = dict((k, metadata[k]) for k in diff if k in metadata) persisted_metadata.update(update) - self._store_metadata(table_name, persisted_metadata) + self._store_metadata(table_name, __version__, persisted_metadata) self._cache(self.dbCacheName, table_name, [[persisted_metadata]], @@ -730,9 +729,9 @@ def _update_metadata_table(self, table_name, metadata): # This implementation paves # the way to lost updates... - def query_metadata_table(self): - # Bring translation table! - stmt = 'select entity_attrs from "{}" limit 100'.format( + def query_version_meta_table(self): + # Bring version from metadata table! + stmt = 'select version from "{}" limit 100'.format( METADATA_TABLE_NAME) # By design, one entry per table_name try: @@ -746,7 +745,7 @@ def query_metadata_table(self): persisted_metadata = {} return persisted_metadata - def _store_metadata(self, table_name, persisted_metadata): + def _store_metadata(self, table_name, version, persisted_metadata): raise NotImplementedError def _get_et_table_names(self, fiware_service=None): diff --git a/src/translators/tests/test_insert.py b/src/translators/tests/test_insert.py index e8803537..3adc0080 100644 --- a/src/translators/tests/test_insert.py +++ b/src/translators/tests/test_insert.py @@ -784,9 +784,9 @@ def test_entity_meta_version(translator): result = translator.insert(entities) assert result.rowcount > 0 table_name = "et0" - data = translator.query_metadata_table() - observed = data['ql_version'] - expected = [__version__, 'Text'] + data = translator.query_version_meta_table() + observed = data + expected = __version__ assert observed == expected translator.clean() diff --git a/src/translators/timescale.py b/src/translators/timescale.py index 54d4397b..fd099d79 100644 --- a/src/translators/timescale.py +++ b/src/translators/timescale.py @@ -269,20 +269,20 @@ def _should_insert_original_entities(self, def _create_metadata_table(self): def do_create(): stmt = "create table if not exists {} " \ - "(table_name text primary key, entity_attrs jsonb)" + "(table_name text primary key, version text, entity_attrs jsonb)" op = stmt.format(METADATA_TABLE_NAME) self.cursor.execute(op) self.with_connection_guard(do_create) - def _store_metadata(self, table_name, persisted_metadata): + def _store_metadata(self, table_name, version, persisted_metadata): def do_store(): - stmt = "insert into {} (table_name, entity_attrs) values (?, ?)" \ + stmt = "insert into {} (table_name, version, entity_attrs) values (?, ?, ?)" \ " on conflict (table_name)" \ " do update set entity_attrs = ?" stmt = stmt.format(METADATA_TABLE_NAME) entity_attrs_value = _encode_to_json_string(persisted_metadata) - self.cursor.execute(stmt, (table_name, entity_attrs_value, + self.cursor.execute(stmt, (table_name, version, entity_attrs_value, entity_attrs_value)) self.with_connection_guard(do_store)