Merge pull request #28 from keboola/update-CF-3202/proper-date-conver…

…sion-before-1970

UPDATE/CF-3202 - Proper date conversion before 1970

README.md

@@ -200,7 +200,8 @@ Clone this repository and init the workspace with following command:
 ```
 git clone https://github.com/keboola/ex-mongodb.git
 cd ex-mongodb
-docker compose build #On M1 Mac build image with "--platform linux/amd64" flag
+cp .env.dist .env
+docker compose build #On M1 Mac build image: "DOCKER_DEFAULT_PLATFORM=linux/amd64 docker compose build"
 docker compose run --rm dev composer install --no-scripts
 ```
 

src/DataNormalizer.php

@@ -0,0 +1,13 @@
+<?php
+
+declare(strict_types=1);
+
+namespace MongoExtractor;
+
+interface DataNormalizer
+{
+    /**
+     * @param array<string, mixed> $data
+     */
+    public function normalize(array &$data): void;
+}

src/DateNormalizer.php

@@ -0,0 +1,56 @@
+<?php
+
+declare(strict_types=1);
+
+namespace MongoExtractor;
+
+use DateTimeImmutable;
+use DateTimeInterface;
+use MongoDB\BSON\UTCDateTime;
+
+final class DateNormalizer implements DataNormalizer
+{
+    /**
+     * @param array<string, mixed> $mapping
+     */
+    public function __construct(
+        private array $mapping = [],
+    ) {
+    }
+
+    /**
+     * @inheritDoc
+     */
+    public function normalize(array &$data): void
+    {
+        foreach ($data as &$item) {
+            foreach ($this->mapping as $path => $mapping) {
+                if (!isset($mapping['type']) || $mapping['type'] !== 'date') {
+                    continue;
+                }
+
+                $keys = explode('.', $path);
+                $current = &$item;
+
+                foreach ($keys as $key) {
+                    if (property_exists($current, $key)) {
+                        $current = &$current->{$key};
+                    }
+                }
+
+                if (is_string($current)) {
+                    $current = (new DateTimeImmutable($current))->format(DateTimeInterface::ATOM);
+
+                    return;
+                }
+
+                if (property_exists($current, '$numberLong')) {
+                    $current = (new UTCDateTime((int) $current->{'$numberLong'}))
+                        ->toDateTime()->format(DateTimeInterface::ATOM);
+
+                    return;
+                }
+            }
+        }
+    }
+}

src/Parse.php

@@ -96,6 +96,7 @@ protected function getParser(): ParserInterface
                 $this->mapping,
                 $this->exportOptions->isIncludeParentInPK(),
                 $this->path,
+                new DateNormalizer($this->mapping),
             );
         }
 

src/Parser/Mapping.php

@@ -9,6 +9,7 @@
 use Keboola\CsvMap\Exception\BadConfigException;
 use Keboola\CsvMap\Exception\BadDataException;
 use Keboola\CsvMap\Mapper;
+use MongoExtractor\DataNormalizer;
 use Nette\Utils\Strings;
 use Symfony\Component\Filesystem\Filesystem;
 use Throwable;
@@ -25,6 +26,8 @@ class Mapping implements ParserInterface
     /** @var array<string, array{path: string, primaryKey: array<int, string>, columns: array<int, string>}> */
     private array $manifestData = [];
 
+    private DataNormalizer $dataNormalizer;
+
     /**
      * @param array<string, mixed> $mapping
      */
@@ -33,12 +36,14 @@ public function __construct(
         array $mapping,
         bool $includeParentInPK,
         string $outputPath,
+        DataNormalizer $dataNormalizer,
     ) {
         $this->name = $name;
         $this->mapping = $mapping;
         $this->includeParentInPK = $includeParentInPK;
         $this->path = $outputPath;
         $this->filesystem = new Filesystem();
+        $this->dataNormalizer = $dataNormalizer;
     }
 
     /**
@@ -52,10 +57,12 @@ public function parse(array $data): void
         $userData = $this->includeParentInPK ? ['parentId' => md5(serialize($data))] : [];
         $mapper = new Mapper($this->mapping, false, $this->name);
         try {
+            $this->dataNormalizer->normalize($data);
+
             $mapper->parse($data, $userData);
         } catch (BadConfigException|BadDataException $e) {
             throw new UserException(sprintf('Invalid mapping configuration: %s', $e->getMessage()));
-        } catch (TypeError) { // @phpstan-ignore-line
+        } catch (TypeError) {
             throw new UserException('CSV writing error. Header and mapped documents must be scalar values.');
         }
 

tests/datasets/dataset-date.json

@@ -0,0 +1,14 @@
+[
+  {
+    "_id": {
+      "$oid": "675042b13a59410a6c6c44b6"
+    },
+    "birthDate": {"$date": "1907-03-31T03:01:56Z"}
+  },
+  {
+    "_id": {
+      "$oid": "6750431c3a59410a6c6c44b7"
+    },
+    "birthDate": {"$date": "2024-03-04T11:11:11Z"}
+  }
+]

tests/functional/export-date-values/expected-code

@@ -0,0 +1 @@
+0

tests/functional/export-date-values/expected-stdout

@@ -0,0 +1,3 @@
+Connected to mongodb://mongodb:27017/test
+Exporting "export-all"
+Done "export-all", parsed 2 records in total

tests/functional/export-date-values/expected/data/out/tables/export-all.csv

@@ -0,0 +1,2 @@
+"675042b13a59410a6c6c44b6","-1980449884"
+"6750431c3a59410a6c6c44b7","1709550671"

tests/functional/export-date-values/expected/data/out/tables/export-all.csv.manifest

@@ -0,0 +1 @@
+{"incremental":false,"columns":["id","timeColumn"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"string"}],"timeColumn":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"string"}]}}

tests/functional/export-date-values/setUp.php

@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types=1);
+
+use MongoExtractor\FunctionalTests\DatadirTest;
+use MongoExtractor\Tests\Traits\ImportDatasetTrait;
+
+return static function (DatadirTest $test): void {
+    (new class { use ImportDatasetTrait;
+
+    })::importDatatasetNoAuthDb('restaurants', 'dataset-date.json');
+};

tests/functional/export-date-values/source/data/config.json

@@ -0,0 +1,27 @@
+{
+  "parameters": {
+    "db": {
+      "host": "mongodb",
+      "port": 27017,
+      "database": "test"
+    },
+    "collection": "restaurants",
+    "tableName": "export-all",
+    "mapping": {
+      "_id.$oid": {
+        "type": "column",
+        "mapping": {
+          "destination": "id"
+        }
+      },
+      "birthDate.$date": {
+        "type": "date",
+        "mapping": {
+          "destination": "timeColumn"
+        }
+      }
+    },
+    "mode": "mapping",
+    "sort": "{_id: 1}"
+  }
+}

tests/functional/test-connection-action-fail-wrong-host/expected-stderr

@@ -1 +1 @@
-No suitable servers found (`serverSelectionTryOnce` set): [Failed to resolve 'locahost']
+No suitable servers found (`serverSelectionTryOnce` set): [Failed to resolve 'locahost']. Topology type: Single