From 77190c0511187331cdc0f0db21b2f562df086c1f Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Tue, 11 Jun 2024 09:57:18 +0100 Subject: [PATCH] fix e2e test for mongo to postgres --- pipelinewise/fastsync/commons/tap_mongodb.py | 9 ++++++++- tests/end_to_end/test_target_postgres.py | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pipelinewise/fastsync/commons/tap_mongodb.py b/pipelinewise/fastsync/commons/tap_mongodb.py index 1a6c951a8..0f37a9a20 100644 --- a/pipelinewise/fastsync/commons/tap_mongodb.py +++ b/pipelinewise/fastsync/commons/tap_mongodb.py @@ -41,7 +41,7 @@ def serialize_document(document: Dict) -> Dict: return { key: transform_value(val, [key]) for key, val in document.items() - if not isinstance(val, (bson.min_key.MinKey, bson.max_key.MaxKey)) + if not isinstance(val, (bson.min_key.MinKey, bson.max_key.MaxKey, bson.binary.Binary)) } @@ -148,6 +148,13 @@ def transform_value(value: Any, path) -> Any: } if isinstance(value, tuple(conversion.keys())): + try: + a = conversion[type(value)](value, path) + except Exception as e: + print(f'======>>> {e}') + print(f'-=-=-=->> {conversion[type(value)]}') + print(f'----->>> {type(value)} , {value}, {path}') + raise e return conversion[type(value)](value, path) return value diff --git a/tests/end_to_end/test_target_postgres.py b/tests/end_to_end/test_target_postgres.py index be7e14a3e..c128f1a6f 100644 --- a/tests/end_to_end/test_target_postgres.py +++ b/tests/end_to_end/test_target_postgres.py @@ -430,17 +430,17 @@ def assert_row_counts_equal(target_schema, table, count_in_source): { 'age': randint(10, 30), 'id': 1001, - 'uuid': uuid.uuid4(), + 'uuid': str(uuid.uuid4()), 'ts': Timestamp(12030, 500), }, { 'date': datetime.utcnow(), 'id': 1002, - 'uuid': uuid.uuid4(), + 'uuid': str(uuid.uuid4()), 'regex': bson.Regex(r'^[A-Z]\\w\\d{2,6}.*$'), }, { - 'uuid': uuid.uuid4(), + 'uuid': str(uuid.uuid4()), 'id': 1003, 'decimal': bson.Decimal128( decimal.Decimal('5.64547548425446546546644')