finish implementing test

airbytehq · Feb 21, 2024 · 5a5254c · 5a5254c
1 parent b9cab96
commit 5a5254c
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 5 deletions.
diff --git a/...tegrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java b/...tegrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java
@@ -225,6 +225,7 @@ public void testExtractedAtUtcTimezoneMigration() throws Exception {
             .withSyncMode(SyncMode.INCREMENTAL)
             .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP)
             .withPrimaryKey(List.of(List.of("id1"), List.of("id2")))
+            .withCursorField(List.of("updated_at"))
             .withStream(new AirbyteStream()
                 .withNamespace(streamNamespace)
                 .withName(streamName)
@@ -234,11 +235,9 @@ public void testExtractedAtUtcTimezoneMigration() throws Exception {
     final List<AirbyteMessage> messages1 = readMessages("dat/sync1_messages.jsonl");
     runSync(catalog, messages1, "airbyte/destination-snowflake:3.5.11");
 
-    // TODO verify that these records are in UTC-8
-    // this probably requires a new pair of fixture files
-//    final List<JsonNode> expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_raw.jsonl");
-//    final List<JsonNode> expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_dedup_final.jsonl");
-//    verifySyncResult(expectedRawRecords1, expectedFinalRecords1, disableFinalTableComparison());
+    final List<JsonNode> expectedRawRecords1 = readRecords("dat/ltz_extracted_at_sync1_expectedrecords_raw.jsonl");
+    final List<JsonNode> expectedFinalRecords1 = readRecords("dat/ltz_extracted_at_sync1_expectedrecords_dedup_final.jsonl");
+    verifySyncResult(expectedRawRecords1, expectedFinalRecords1, disableFinalTableComparison());
 
     // Second sync
     final List<AirbyteMessage> messages2 = readMessages("dat/sync2_messages.jsonl");

diff --git a/...c/test-integration/resources/dat/ltz_extracted_at_sync1_expectedrecords_dedup_final.jsonl b/...c/test-integration/resources/dat/ltz_extracted_at_sync1_expectedrecords_dedup_final.jsonl
@@ -0,0 +1,5 @@
+// Keep the Alice record with more recent UPDATED_AT
+{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:01:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}}
+{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}}
+{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"}
+{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}
diff --git a/...flake/src/test-integration/resources/dat/ltz_extracted_at_sync1_expectedrecords_raw.jsonl b/...flake/src/test-integration/resources/dat/ltz_extracted_at_sync1_expectedrecords_raw.jsonl
@@ -0,0 +1,7 @@
+{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}}
+// Note the duplicate record. In this sync mode, we don't dedup anything.
+{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}
+{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}
+// Invalid data is still allowed in the raw table.
+{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}
+{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}}