Merge pull request #17 from keboola/adamvyborny-CM-388-memory-exceede…

…d-user-error Make QueryExceededMemoryLimitNoDiskUseAllowed as user exception
keboola · Jan 24, 2023 · 3e3beb8 · 3e3beb8
2 parents d0aa466 + 0678d4e
commit 3e3beb8
Show file tree

Hide file tree

Showing 3 changed files with 87 additions and 1 deletion.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM php:8-cli
+FROM php:8.1-cli
 
 ARG COMPOSER_FLAGS="--prefer-dist --no-interaction"
 ARG DEBIAN_FRONTEND=noninteractive

diff --git a/src/Export.php b/src/Export.php
@@ -107,6 +107,17 @@ protected function handleMongoExportFails(Throwable $e): void
             ));
         }
 
+        if (str_contains($e->getMessage(), 'QueryExceededMemoryLimitNoDiskUseAllowed')) {
+            throw new UserException('Sort exceeded memory limit, but did not opt in to ' .
+                'external sorting. The field should be set as an index, so there will be no sorting in the ' .
+                'incremental fetching query, because the index will be used');
+        }
+
+        if (str_contains($e->getMessage(), 'dial tcp: i/o timeout')) {
+            throw new UserException('Could not connect to server: connection() error occurred during ' .
+                'connection handshake: dial tcp: i/o timeout');
+        }
+
         if (preg_match('/query \'\\[[^\\]]*\\]\' is not valid JSON/i', $e->getMessage())) {
             throw new UserException(sprintf(
                 'Export "%s" failed. Query "' . $this->exportOptions->getQuery() . '" is not valid JSON',

diff --git a/tests/phpunit/HandleMongoExportFailsTest.php b/tests/phpunit/HandleMongoExportFailsTest.php
@@ -0,0 +1,75 @@
+<?php
+
+declare(strict_types=1);
+
+namespace MongoExtractor\Tests\Unit;
+
+use Generator;
+use Keboola\Component\UserException;
+use MongoExtractor\Config\ExportOptions;
+use MongoExtractor\Export;
+use MongoExtractor\ExportCommandFactory;
+use MongoExtractor\UriFactory;
+use PHPUnit\Framework\TestCase;
+use ReflectionClass;
+use Symfony\Component\Process\Exception\ProcessFailedException;
+use Symfony\Component\Process\Process;
+
+class HandleMongoExportFailsTest extends TestCase
+{
+    /**
+     * @dataProvider exceptionsProvider
+     * @throws \ReflectionException
+     * @throws \Keboola\Component\UserException
+     */
+    public function testHandleMongoExportFails(
+        ProcessFailedException $mongoException,
+        UserException $expectedException
+    ): void {
+        $this->expectException(get_class($expectedException));
+        $this->expectExceptionMessage($expectedException->getMessage());
+
+        $class = new ReflectionClass(Export::class);
+        $method = $class->getMethod('handleMongoExportFails');
+        $exportOptions = new ExportOptions(['name' => '', 'mode' => '']);
+        $exportClass = new Export(new ExportCommandFactory(new UriFactory(), false), [], $exportOptions);
+        $method->invoke($exportClass, $mongoException);
+    }
+
+    public function exceptionsProvider(): Generator
+    {
+        yield 'dial tcp: i/o timeout' => [
+            new ProcessFailedException($this->createMockInstanceOfProcess('2023-01-23T17:02:32.685+0000\t' .
+                'could not connect to server: connection() error occured during connection handshake: dial tcp: i/o ' .
+                'timeout')),
+            new UserException('Could not connect to server: connection() error occurred during ' .
+                'connection handshake: dial tcp: i/o timeout'),
+        ];
+
+        yield 'QueryExceededMemoryLimitNoDiskUseAllowed' => [
+            new ProcessFailedException($this->createMockInstanceOfProcess('2023-01-23T17:02:32.685+0000\t' .
+                '(QueryExceededMemoryLimitNoDiskUseAllowed) Sort exceeded memory limit of 104857600 bytes, but did ' .
+                'not opt in to external sorting.')),
+            new UserException('Sort exceeded memory limit, but did not opt in to ' .
+                'external sorting. The field should be set as an index, so there will be no sorting in the ' .
+                'incremental fetching query, because the index will be used'),
+        ];
+    }
+
+    private function createMockInstanceOfProcess(string $errorOutput): Process
+    {
+        $mockProcess = $this->createMock(Process::class);
+        $mockProcess->method('isSuccessful')->willReturn(false);
+        $mockProcess->method('getCommandLine')->willReturn('mongoexport --uri ' .
+            '\'mongodb://user:pass@mongo/mongodb\' --collection \'transactions\' ' .
+            '--query \'{\"_id\":{\"$gte\":{\"$oid\": \"63ceb66a967f8f0017ceed64\"}}}\' --sort \'{\"_id\":1}\' ' .
+            '--type \'json\'');
+        $mockProcess->method('getExitCode')->willReturn(1);
+        $mockProcess->method('getExitCodeText')->willReturn('General error');
+        $mockProcess->method('getWorkingDirectory')->willReturn('/code');
+        $mockProcess->method('getOutput')->willReturn('');
+        $mockProcess->method('getErrorOutput')->willReturn($errorOutput);
+
+        return $mockProcess;
+    }
+}