Merge pull request #16 from keboola/adamvyborny-remove-types-from-map…

…ping Remove types which are omitted in relaxed mode of extended JSON retur…
keboola · Nov 30, 2022 · d0aa466 · d0aa466
2 parents 175abeb + c75d1b3
commit d0aa466
Showing 31 changed files with 283 additions and 8 deletions.
diff --git a/src/Config/ExportOptions.php b/src/Config/ExportOptions.php
@@ -5,6 +5,7 @@
 namespace MongoExtractor\Config;
 
 use Keboola\Component\UserException;
+use MongoExtractor\ExportHelper;
 
 class ExportOptions
 {
@@ -40,7 +41,12 @@ public function __construct(array $exportOptions)
         $this->incrementalFetching = $exportOptions['incremental'] ?? false;
         $this->name = $exportOptions['tableName'] ?? $exportOptions['name'];
         $this->mode = $exportOptions['mode'];
-        $this->mapping = $exportOptions['mapping'] ?? [];
+        if (isset($exportOptions['mapping'])) {
+            ExportHelper::removeTypesInMappingKeys($exportOptions['mapping']);
+        } else {
+            $exportOptions['mapping'] = [];
+        }
+        $this->mapping = $exportOptions['mapping'];
         $this->limit = !empty($exportOptions['limit']) ? (int) $exportOptions['limit'] : null;
         $this->query = $exportOptions['query'] ?? null;
         $this->collection = $exportOptions['collection'] ?? null;

diff --git a/src/ExportHelper.php b/src/ExportHelper.php
@@ -127,4 +127,22 @@ public static function addQuotesToJsonKeys(string $input): string
 
         return $output;
     }
+
+    /**
+     * @param array<mixed, mixed> $mapping
+     */
+    public static function removeTypesInMappingKeys(array &$mapping): void
+    {
+        $final = [];
+        foreach ($mapping as $k => &$v) {
+            if (is_string($k)) {
+                $k = preg_replace('/(\.\$(numberDouble|numberInt|numberLong))$/', '', $k);
+            }
+            if (is_array($v)) {
+                self::removeTypesInMappingKeys($v);
+            }
+            $final[$k] = $v;
+        }
+        $mapping = $final;
+    }
 }
diff --git a/tests/datasets/dataset-number-long.json b/tests/datasets/dataset-number-long.json
@@ -0,0 +1,23 @@
+[
+  {
+    "_id": {
+      "$oid": "5716054bee6e764c94fa7ddd"
+    },
+    "numberLong": 2090845886852,
+    "numberLongInObject": {"$numberLong": "2090845886852"}
+  },
+  {
+    "_id": {
+      "$oid": "5716054bee6e764c94fa8181"
+    },
+    "numberLong": 2000000000000,
+    "numberLongInObject": {"$numberLong": "2000000000000"}
+  },
+  {
+    "_id": {
+      "$oid": "5716054bee6e764c94fa8213"
+    },
+    "numberLong": 1050845886852,
+    "numberLongInObject": {"$numberLong": "1050845886852"}
+  }
+]
diff --git a/tests/functional/export-remove-types-omitted-in-relax-mode/expected-code b/tests/functional/export-remove-types-omitted-in-relax-mode/expected-code
@@ -0,0 +1 @@
+0
diff --git a/tests/functional/export-remove-types-omitted-in-relax-mode/expected-stdout b/tests/functional/export-remove-types-omitted-in-relax-mode/expected-stdout
@@ -0,0 +1,3 @@
+Connected to mongodb://mongodb:27017/test
+Exporting "number-long"
+Done "number-long", parsed 3 records in total
diff --git a/tests/functional/export-remove-types-omitted-in-relax-mode/expected/data/out/files/.gitkeep b/tests/functional/export-remove-types-omitted-in-relax-mode/expected/data/out/files/.gitkeep
diff --git a/tests/functional/export-remove-types-omitted-in-relax-mode/expected/data/out/tables/.gitkeep b/tests/functional/export-remove-types-omitted-in-relax-mode/expected/data/out/tables/.gitkeep
diff --git a/...tional/export-remove-types-omitted-in-relax-mode/expected/data/out/tables/number-long.csv b/...tional/export-remove-types-omitted-in-relax-mode/expected/data/out/tables/number-long.csv
@@ -0,0 +1,4 @@
+"id","numberLong","numberLongInObject"
+"5716054bee6e764c94fa7ddd","2090845886852","2090845886852"
+"5716054bee6e764c94fa8181","2000000000000","2000000000000"
+"5716054bee6e764c94fa8213","1050845886852","1050845886852"
diff --git a/...port-remove-types-omitted-in-relax-mode/expected/data/out/tables/number-long.csv.manifest b/...port-remove-types-omitted-in-relax-mode/expected/data/out/tables/number-long.csv.manifest
@@ -0,0 +1 @@
+{"primary_key":null,"incremental":false}
diff --git a/tests/functional/export-remove-types-omitted-in-relax-mode/setUp.php b/tests/functional/export-remove-types-omitted-in-relax-mode/setUp.php
@@ -0,0 +1,10 @@
+<?php
+
+declare(strict_types = 1);
+
+use MongoExtractor\FunctionalTests\DatadirTest;
+use MongoExtractor\Tests\Traits\ImportDatasetTrait;
+
+return static function (DatadirTest $test): void {
+    (new class { use ImportDatasetTrait; })::importDatatasetNoAuthDb('number-long', 'dataset-number-long.json');
+};
diff --git a/tests/functional/export-remove-types-omitted-in-relax-mode/source/data/config.json b/tests/functional/export-remove-types-omitted-in-relax-mode/source/data/config.json
@@ -0,0 +1,23 @@
+{
+  "parameters": {
+    "db": {
+      "host": "mongodb",
+      "port": 27017,
+      "database": "test"
+    },
+    "collection": "number-long",
+    "tableName": "number-long",
+    "mapping": {
+      "_id.$oid": {
+        "type": "column",
+        "mapping": {
+          "destination": "id"
+        }
+      },
+      "numberLong.$numberLong": "numberLong",
+      "numberLongInObject.$numberLong": "numberLongInObject"
+    },
+    "mode": "mapping",
+    "sort": "{_id: 1}"
+  }
+}
diff --git a/tests/functional/incremental-fetching-number-long-state/expected-code b/tests/functional/incremental-fetching-number-long-state/expected-code
@@ -0,0 +1 @@
+0
diff --git a/tests/functional/incremental-fetching-number-long-state/expected-stdout b/tests/functional/incremental-fetching-number-long-state/expected-stdout
@@ -0,0 +1,3 @@
+Connected to mongodb://mongodb:27017/test
+Exporting "incremental"
+Done "incremental", parsed 1 record in total
diff --git a/tests/functional/incremental-fetching-number-long-state/expected/data/out/files/.gitkeep b/tests/functional/incremental-fetching-number-long-state/expected/data/out/files/.gitkeep
diff --git a/tests/functional/incremental-fetching-number-long-state/expected/data/out/state.json b/tests/functional/incremental-fetching-number-long-state/expected/data/out/state.json
@@ -0,0 +1 @@
+{"lastFetchedRow":2090845886852}
diff --git a/tests/functional/incremental-fetching-number-long-state/expected/data/out/tables/.gitkeep b/tests/functional/incremental-fetching-number-long-state/expected/data/out/tables/.gitkeep
diff --git a/...unctional/incremental-fetching-number-long-state/expected/data/out/tables/incremental.csv b/...unctional/incremental-fetching-number-long-state/expected/data/out/tables/incremental.csv
@@ -0,0 +1,2 @@
+"id","numberLong","numberLongInObject"
+"5716054bee6e764c94fa7ddd","2090845886852","2090845886852"
diff --git a/.../incremental-fetching-number-long-state/expected/data/out/tables/incremental.csv.manifest b/.../incremental-fetching-number-long-state/expected/data/out/tables/incremental.csv.manifest
@@ -0,0 +1 @@
+{"primary_key":null,"incremental":true}
diff --git a/tests/functional/incremental-fetching-number-long-state/setUp.php b/tests/functional/incremental-fetching-number-long-state/setUp.php
@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types = 1);
+
+use MongoExtractor\FunctionalTests\DatadirTest;
+use MongoExtractor\Tests\Traits\ImportDatasetTrait;
+
+return static function (DatadirTest $test): void {
+    (new class {
+        use ImportDatasetTrait;
+    })::importDatatasetNoAuthDb('incremental', 'dataset-number-long.json');
+};
diff --git a/tests/functional/incremental-fetching-number-long-state/source/data/config.json b/tests/functional/incremental-fetching-number-long-state/source/data/config.json
@@ -0,0 +1,24 @@
+{
+  "parameters": {
+    "db": {
+      "host": "mongodb",
+      "port": 27017,
+      "database": "test"
+    },
+    "tableName": "incremental",
+    "collection": "incremental",
+    "incremental": true,
+    "incrementalFetchingColumn": "numberLongInObject",
+    "mapping": {
+      "_id.$oid": {
+        "type": "column",
+        "mapping": {
+          "destination": "id"
+        }
+      },
+      "numberLong.$numberLong": "numberLong",
+      "numberLongInObject.$numberLong": "numberLongInObject"
+    },
+    "mode": "mapping"
+  }
+}
diff --git a/tests/functional/incremental-fetching-number-long-state/source/data/in/state.json b/tests/functional/incremental-fetching-number-long-state/source/data/in/state.json
@@ -0,0 +1 @@
+{"lastFetchedRow":2090845886852}
diff --git a/tests/functional/incremental-fetching-number-long/expected-code b/tests/functional/incremental-fetching-number-long/expected-code
@@ -0,0 +1 @@
+0
diff --git a/tests/functional/incremental-fetching-number-long/expected-stdout b/tests/functional/incremental-fetching-number-long/expected-stdout
@@ -0,0 +1,3 @@
+Connected to mongodb://mongodb:27017/test
+Exporting "incremental"
+Done "incremental", parsed 3 records in total
diff --git a/tests/functional/incremental-fetching-number-long/expected/data/out/files/.gitkeep b/tests/functional/incremental-fetching-number-long/expected/data/out/files/.gitkeep
diff --git a/tests/functional/incremental-fetching-number-long/expected/data/out/state.json b/tests/functional/incremental-fetching-number-long/expected/data/out/state.json
@@ -0,0 +1 @@
+{"lastFetchedRow":2090845886852}
diff --git a/tests/functional/incremental-fetching-number-long/expected/data/out/tables/.gitkeep b/tests/functional/incremental-fetching-number-long/expected/data/out/tables/.gitkeep
diff --git a/tests/functional/incremental-fetching-number-long/expected/data/out/tables/incremental.csv b/tests/functional/incremental-fetching-number-long/expected/data/out/tables/incremental.csv
@@ -0,0 +1,4 @@
+"id","numberLong","numberLongInObject"
+"5716054bee6e764c94fa8213","1050845886852","1050845886852"
+"5716054bee6e764c94fa8181","2000000000000","2000000000000"
+"5716054bee6e764c94fa7ddd","2090845886852","2090845886852"
diff --git a/...tional/incremental-fetching-number-long/expected/data/out/tables/incremental.csv.manifest b/...tional/incremental-fetching-number-long/expected/data/out/tables/incremental.csv.manifest
@@ -0,0 +1 @@
+{"primary_key":null,"incremental":true}
diff --git a/tests/functional/incremental-fetching-number-long/setUp.php b/tests/functional/incremental-fetching-number-long/setUp.php
@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types = 1);
+
+use MongoExtractor\FunctionalTests\DatadirTest;
+use MongoExtractor\Tests\Traits\ImportDatasetTrait;
+
+return static function (DatadirTest $test): void {
+    (new class {
+        use ImportDatasetTrait;
+    })::importDatatasetNoAuthDb('incremental', 'dataset-number-long.json');
+};
diff --git a/tests/functional/incremental-fetching-number-long/source/data/config.json b/tests/functional/incremental-fetching-number-long/source/data/config.json
@@ -0,0 +1,24 @@
+{
+  "parameters": {
+    "db": {
+      "host": "mongodb",
+      "port": 27017,
+      "database": "test"
+    },
+    "tableName": "incremental",
+    "collection": "incremental",
+    "incremental": true,
+    "incrementalFetchingColumn": "numberLongInObject",
+    "mapping": {
+      "_id.$oid": {
+        "type": "column",
+        "mapping": {
+          "destination": "id"
+        }
+      },
+      "numberLong.$numberLong": "numberLong",
+      "numberLongInObject.$numberLong": "numberLongInObject"
+    },
+    "mode": "mapping"
+  }
+}
diff --git a/tests/phpunit/ExportHelperTest.php b/tests/phpunit/ExportHelperTest.php
@@ -4,6 +4,7 @@
 
 namespace MongoExtractor\Tests\Unit;
 
+use Generator;
 use MongoExtractor\ExportHelper;
 use PHPUnit\Framework\Assert;
 use PHPUnit\Framework\TestCase;
@@ -26,6 +27,17 @@ public function testFixIsoDateInGteQuery(string $input, string $expectedOutput):
         Assert::assertSame($expectedOutput, ExportHelper::fixIsoDateInGteQuery($input));
     }
 
+    /**
+     * @dataProvider getMappingsProvider
+     * @param array<mixed, mixed> $input
+     * @param array<mixed, mixed> $expected
+     */
+    public function testRemoveTypeFromMappingKeys(array $input, array $expected): void
+    {
+        ExportHelper::removeTypesInMappingKeys($input);
+        Assert::assertSame($expected, $input);
+    }
+
     public function testConvertObjectIdToString(): void
     {
         $input = '{ "id": 1, "_id": {"$oid": "61f8e6b99e2986a522ebb90f"}, "string":  "testString"}';
@@ -71,17 +83,17 @@ public function getConvertDatesDataProvider(): array
                 false,
                 '{ "id" : 1, "date": "2020-05-18T16:00:00Z", "string":  "testString"}',
             ],
-            'prefix' =>[
+            'prefix' => [
                 '{ "id" : 1, "date": {"$date": "2020-05-18T16:00:00Z"}, "string":  "testString"}',
                 true,
                 '{ "id" : 1, "date": "ISODate(\"2020-05-18T16:00:00Z\")", "string":  "testString"}',
             ],
-            'escaping' =>[
+            'escaping' => [
                 '{ "id" : 1, "date": {"$date": "2020-05-\"\"\"18T16:00:00Z"}, "string":  "testString"}',
                 false,
                 '{ "id" : 1, "date": "2020-05-\"\"\"18T16:00:00Z", "string":  "testString"}',
             ],
-            'escaping-prefix' =>[
+            'escaping-prefix' => [
                 '{ "id" : 1, "date": {"$date": "2020-05-\"\"\"18T16:00:00Z"}, "string":  "testString"}',
                 true,
                 '{ "id" : 1, "date": "ISODate(\"2020-05-\\\\\"\\\\\"\\\\\"18T16:00:00Z\")", "string":  "testString"}',
@@ -91,7 +103,7 @@ public function getConvertDatesDataProvider(): array
                 false,
                 '{"id":1,"date":"2020-05-18T16:00:00Z","string":"testString"}',
             ],
-            'no-spaces-prefix' =>[
+            'no-spaces-prefix' => [
                 '{"id":1,"date":{"$date":"2020-05-18T16:00:00Z"},"string":"testString"}',
                 true,
                 '{"id":1,"date":"ISODate(\"2020-05-18T16:00:00Z\")","string":"testString"}',
@@ -101,7 +113,7 @@ public function getConvertDatesDataProvider(): array
                 false,
                 '{ "id" : 1, "date": "", "string":  "testString"}',
             ],
-            'empty-prefix' =>[
+            'empty-prefix' => [
                 '{ "id" : 1, "date": {"$date": ""}, "string":  "testString"}',
                 true,
                 '{ "id" : 1, "date": "ISODate(\"\")", "string":  "testString"}',
@@ -111,7 +123,7 @@ public function getConvertDatesDataProvider(): array
                 false,
                 '{ "id" : 1, "date": {"$date": "2020-05-18T16:00:00Z}, "string":  "testString"}',
             ],
-            'invalid-1-prefix' =>[
+            'invalid-1-prefix' => [
                 '{ "id" : 1, "date": {"$date": "2020-05-18T16:00:00Z}, "string":  "testString"}',
                 true,
                 '{ "id" : 1, "date": {"$date": "2020-05-18T16:00:00Z}, "string":  "testString"}',
@@ -121,7 +133,7 @@ public function getConvertDatesDataProvider(): array
                 false,
                 '{ "id" : 1, "date": {"$date": 1234, "string":  "testString"}',
             ],
-            'invalid-2-prefix' =>[
+            'invalid-2-prefix' => [
                 '{ "id" : 1, "date": {"$date": 1234, "string":  "testString"}',
                 true,
                 '{ "id" : 1, "date": {"$date": 1234, "string":  "testString"}',
@@ -183,4 +195,87 @@ public function testAddQuotesToJsonKeys(): void
             )
         );
     }
+
+    public function getMappingsProvider(): Generator
+    {
+        yield [
+            'inputJson' => json_decode('
+            {
+      "_id.$oid": {
+        "type": "column",
+        "mapping": {
+          "destination": "id"
+        }
+      },
+      "numberLong.$numberLong": "numberLong",
+      "numberLongInObject.$numberLong": "numberLongInObject"
+    }', true),
+            'expected' => [
+                '_id.$oid' => [
+                    'type' => 'column',
+                    'mapping' => [
+                        'destination' => 'id',
+                    ],
+                ],
+                'numberLong' => 'numberLong',
+                'numberLongInObject' => 'numberLongInObject',
+            ],
+        ];
+
+        yield [
+            'inputJson' => json_decode('
+            {
+      "_id.$oid": {
+        "type": "column",
+        "mapping": {
+          "destination": "id"
+        }
+      },
+      "numberLong": "numberLong",
+      "date.$date": "date"
+    }', true),
+            'expected' => [
+                '_id.$oid' => [
+                    'type' => 'column',
+                    'mapping' => [
+                        'destination' => 'id',
+                    ],
+                ],
+                'numberLong' => 'numberLong',
+                'date.$date' => 'date',
+            ],
+        ];
+
+        yield [
+            'inputJson' => json_decode('
+            {
+      "_id.$oid": {
+        "type": "column",
+        "mapping": {
+          "destination": "id"
+        }
+      },
+      "numberLong.$numberLong": "numberLong",
+      "numberInt.$numberInt": "numberInt",
+      "numberDouble.$numberDouble": "numberDouble",
+      "date.$date": "date",
+      "numberDecimal.$numberDecimal": "numberDecimal",
+      "binary.$binary.base64": "binary"
+    }', true),
+            'expected' => [
+                '_id.$oid' => [
+                    'type' => 'column',
+                    'mapping' => [
+                        'destination' => 'id',
+                    ],
+                ],
+                'numberLong' => 'numberLong',
+                'numberInt' => 'numberInt',
+                'numberDouble' => 'numberDouble',
+                'date.$date' => 'date',
+                'numberDecimal.$numberDecimal' => 'numberDecimal',
+                'binary.$binary.base64' => 'binary',
+            ],
+        ];
+    }
 }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		"id","numberLong","numberLongInObject"
		"5716054bee6e764c94fa7ddd","2090845886852","2090845886852"