From 417631dfce6f237c10cdf1e163759ee30742cc4a Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Tue, 28 May 2024 12:11:46 +0100 Subject: [PATCH 1/6] release 0.61.0 --- CHANGELOG.md | 4 ++++ setup.py | 4 ++-- singer-connectors/tap-mongodb/requirements.txt | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be957fc4a..b3b378a2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +0.61.0 (2024-05-28) +------------------- +- Bump `pipelinewise-tap-mongodb`from `1.3.0` to `1.4.0` + 0.60.0 (2024-05-10) ------------------- - Bump `pipelinewise-tap-mysql`from `1.5.6` to `1.6.0` diff --git a/setup.py b/setup.py index 517ddcbd7..930a27e08 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup(name='pipelinewise', python_requires='>=3.7,<3.10', - version='0.60.0', + version='0.60.1', description='PipelineWise', long_description=LONG_DESCRIPTION, long_description_content_type='text/markdown', @@ -33,7 +33,7 @@ 'google-cloud-bigquery==3.11.1', 'pipelinewise-singer-python==1.*', 'python-pidfile==3.0.0', - 'pymongo==3.12.3', + 'pymongo==4.7.*', 'tzlocal>=2.0,<4.1', 'slackclient==2.9.4', 'sqlparse==0.4.4', diff --git a/singer-connectors/tap-mongodb/requirements.txt b/singer-connectors/tap-mongodb/requirements.txt index 162130640..8ce903d9d 100644 --- a/singer-connectors/tap-mongodb/requirements.txt +++ b/singer-connectors/tap-mongodb/requirements.txt @@ -1 +1 @@ -pipelinewise-tap-mongodb==1.3.0 \ No newline at end of file +pipelinewise-tap-mongodb==1.4.0 \ No newline at end of file From 77190c0511187331cdc0f0db21b2f562df086c1f Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Tue, 11 Jun 2024 09:57:18 +0100 Subject: [PATCH 2/6] fix e2e test for mongo to postgres --- pipelinewise/fastsync/commons/tap_mongodb.py | 9 ++++++++- tests/end_to_end/test_target_postgres.py | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pipelinewise/fastsync/commons/tap_mongodb.py b/pipelinewise/fastsync/commons/tap_mongodb.py index 1a6c951a8..0f37a9a20 100644 --- a/pipelinewise/fastsync/commons/tap_mongodb.py +++ b/pipelinewise/fastsync/commons/tap_mongodb.py @@ -41,7 +41,7 @@ def serialize_document(document: Dict) -> Dict: return { key: transform_value(val, [key]) for key, val in document.items() - if not isinstance(val, (bson.min_key.MinKey, bson.max_key.MaxKey)) + if not isinstance(val, (bson.min_key.MinKey, bson.max_key.MaxKey, bson.binary.Binary)) } @@ -148,6 +148,13 @@ def transform_value(value: Any, path) -> Any: } if isinstance(value, tuple(conversion.keys())): + try: + a = conversion[type(value)](value, path) + except Exception as e: + print(f'======>>> {e}') + print(f'-=-=-=->> {conversion[type(value)]}') + print(f'----->>> {type(value)} , {value}, {path}') + raise e return conversion[type(value)](value, path) return value diff --git a/tests/end_to_end/test_target_postgres.py b/tests/end_to_end/test_target_postgres.py index be7e14a3e..c128f1a6f 100644 --- a/tests/end_to_end/test_target_postgres.py +++ b/tests/end_to_end/test_target_postgres.py @@ -430,17 +430,17 @@ def assert_row_counts_equal(target_schema, table, count_in_source): { 'age': randint(10, 30), 'id': 1001, - 'uuid': uuid.uuid4(), + 'uuid': str(uuid.uuid4()), 'ts': Timestamp(12030, 500), }, { 'date': datetime.utcnow(), 'id': 1002, - 'uuid': uuid.uuid4(), + 'uuid': str(uuid.uuid4()), 'regex': bson.Regex(r'^[A-Z]\\w\\d{2,6}.*$'), }, { - 'uuid': uuid.uuid4(), + 'uuid': str(uuid.uuid4()), 'id': 1003, 'decimal': bson.Decimal128( decimal.Decimal('5.64547548425446546546644') From 3ded12616199d5ffa0ecf7d96852d2c8c8f9ed7d Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Tue, 11 Jun 2024 10:01:29 +0100 Subject: [PATCH 3/6] fix e2e test for mongo to postgres --- pipelinewise/fastsync/commons/tap_mongodb.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pipelinewise/fastsync/commons/tap_mongodb.py b/pipelinewise/fastsync/commons/tap_mongodb.py index 0f37a9a20..8900f33c0 100644 --- a/pipelinewise/fastsync/commons/tap_mongodb.py +++ b/pipelinewise/fastsync/commons/tap_mongodb.py @@ -148,13 +148,6 @@ def transform_value(value: Any, path) -> Any: } if isinstance(value, tuple(conversion.keys())): - try: - a = conversion[type(value)](value, path) - except Exception as e: - print(f'======>>> {e}') - print(f'-=-=-=->> {conversion[type(value)]}') - print(f'----->>> {type(value)} , {value}, {path}') - raise e return conversion[type(value)](value, path) return value From d6c0fb4d6fd646802b24c2eb4c38fba52569d3a9 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Tue, 11 Jun 2024 16:54:37 +0100 Subject: [PATCH 4/6] fix e2e test for mongo to postgres --- pipelinewise/fastsync/commons/tap_mongodb.py | 3 ++- setup.py | 2 +- singer-connectors/tap-mongodb/requirements.txt | 2 +- tests/end_to_end/test_target_postgres.py | 6 +++--- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pipelinewise/fastsync/commons/tap_mongodb.py b/pipelinewise/fastsync/commons/tap_mongodb.py index 8900f33c0..71b98cc23 100644 --- a/pipelinewise/fastsync/commons/tap_mongodb.py +++ b/pipelinewise/fastsync/commons/tap_mongodb.py @@ -41,7 +41,7 @@ def serialize_document(document: Dict) -> Dict: return { key: transform_value(val, [key]) for key, val in document.items() - if not isinstance(val, (bson.min_key.MinKey, bson.max_key.MaxKey, bson.binary.Binary)) + if not isinstance(val, (bson.min_key.MinKey, bson.max_key.MaxKey)) } @@ -139,6 +139,7 @@ def transform_value(value: Any, path) -> Any: datetime.datetime: lambda val, _: class_to_string(val, 'datetime'), bson.decimal128.Decimal128: lambda val, _: val.to_decimal(), bson.regex.Regex: lambda val, _: dict(pattern=val.pattern, flags=val.flags), + bson.binary.Binary: lambda val, _: class_to_string(val, 'bytes'), bson.code.Code: lambda val, _: dict(value=str(val), scope=str(val.scope)) if val.scope else str(val), diff --git a/setup.py b/setup.py index 930a27e08..467efd356 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ 'google-cloud-bigquery==3.11.1', 'pipelinewise-singer-python==1.*', 'python-pidfile==3.0.0', - 'pymongo==4.7.*', + 'pymongo==4.7.3', 'tzlocal>=2.0,<4.1', 'slackclient==2.9.4', 'sqlparse==0.4.4', diff --git a/singer-connectors/tap-mongodb/requirements.txt b/singer-connectors/tap-mongodb/requirements.txt index 8ce903d9d..162130640 100644 --- a/singer-connectors/tap-mongodb/requirements.txt +++ b/singer-connectors/tap-mongodb/requirements.txt @@ -1 +1 @@ -pipelinewise-tap-mongodb==1.4.0 \ No newline at end of file +pipelinewise-tap-mongodb==1.3.0 \ No newline at end of file diff --git a/tests/end_to_end/test_target_postgres.py b/tests/end_to_end/test_target_postgres.py index c128f1a6f..73267cb2f 100644 --- a/tests/end_to_end/test_target_postgres.py +++ b/tests/end_to_end/test_target_postgres.py @@ -430,17 +430,17 @@ def assert_row_counts_equal(target_schema, table, count_in_source): { 'age': randint(10, 30), 'id': 1001, - 'uuid': str(uuid.uuid4()), + 'uuid': bson.Binary.from_uuid(uuid.uuid4()), 'ts': Timestamp(12030, 500), }, { 'date': datetime.utcnow(), 'id': 1002, - 'uuid': str(uuid.uuid4()), + 'uuid': bson.Binary.from_uuid(uuid.uuid4()), 'regex': bson.Regex(r'^[A-Z]\\w\\d{2,6}.*$'), }, { - 'uuid': str(uuid.uuid4()), + 'uuid': bson.Binary.from_uuid(uuid.uuid4()), 'id': 1003, 'decimal': bson.Decimal128( decimal.Decimal('5.64547548425446546546644') From 02b3276ff69c44ecf442a3cda890bce098434fa9 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Tue, 11 Jun 2024 17:06:12 +0100 Subject: [PATCH 5/6] fix e2e test for mongodb --- .../tap_mongodb/test_replicate_mongodb_to_sf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/end_to_end/target_snowflake/tap_mongodb/test_replicate_mongodb_to_sf.py b/tests/end_to_end/target_snowflake/tap_mongodb/test_replicate_mongodb_to_sf.py index 79746d614..c9501ec3a 100644 --- a/tests/end_to_end/target_snowflake/tap_mongodb/test_replicate_mongodb_to_sf.py +++ b/tests/end_to_end/target_snowflake/tap_mongodb/test_replicate_mongodb_to_sf.py @@ -90,17 +90,17 @@ def test_replicate_mongodb_to_sf(self): { 'age': randint(10, 30), 'id': 1001, - 'uuid': uuid.uuid4(), + 'uuid': bson.Binary.from_uuid(uuid.uuid4()), 'ts': bson.Timestamp(12030, 500), }, { 'date': datetime.utcnow(), 'id': 1002, - 'uuid': uuid.uuid4(), + 'uuid': bson.Binary.from_uuid(uuid.uuid4()), 'regex': bson.Regex(r'^[A-Z]\\w\\d{2,6}.*$'), }, { - 'uuid': uuid.uuid4(), + 'uuid': bson.Binary.from_uuid(uuid.uuid4()), 'id': 1003, 'decimal': bson.Decimal128( decimal.Decimal('5.64547548425446546546644') From 0000346505b2478daf0ce801f320ed5b35b991c6 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Tue, 11 Jun 2024 17:44:51 +0100 Subject: [PATCH 6/6] fix e2e test for mongodb --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3b378a2f..02ce9c491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ 0.61.0 (2024-05-28) ------------------- -- Bump `pipelinewise-tap-mongodb`from `1.3.0` to `1.4.0` +- Bump `pymongo`from `3.12.3` to `4.7.3` 0.60.0 (2024-05-10) -------------------