Skip to content

Commit

Permalink
fix(PT-12973): Fixed log file flooding with continue on error enabled (
Browse files Browse the repository at this point in the history
…#58)

* fix(PT-12973): Fixed log file flooding with continue on error enabled

* fix(PT-12973): Fixed log file flooding with continue on error enabled

* fix(PT-12973): Fixed log file flooding with continue on error enabled

* fix(PT-12973): Fixed log file flooding with continue on error enabled
  • Loading branch information
Dominik Rathmer authored Jul 26, 2023
1 parent 363e889 commit eeab0d5
Show file tree
Hide file tree
Showing 6 changed files with 667 additions and 14 deletions.
8 changes: 4 additions & 4 deletions Components/Service/DataWorkflow.php
Original file line number Diff line number Diff line change
Expand Up @@ -159,10 +159,6 @@ public function import(ImportRequest $request, Session $session): array
// inserts/update data into the database
$dataIo->write($data, $defaultValues);

// writes into database log table
$profileName = $request->profileEntity->getName();
$dataIo->writeLog($request->inputFile, $profileName);

$session->progress($batchSize);

if ($dataIo->supportsUnprocessedData()) {
Expand All @@ -172,6 +168,10 @@ public function import(ImportRequest $request, Session $session): array
}

if ($session->getState() === Session::SESSION_FINISHED) {
// writes into database log table
$profileName = $request->profileEntity->getName();
$dataIo->writeLog($request->inputFile, $profileName);

$session->close();
}

Expand Down
79 changes: 69 additions & 10 deletions Tests/Functional/Components/Service/DataWorkflowTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,31 @@
use Doctrine\DBAL\Exception;
use PHPUnit\Framework\TestCase;
use Shopware\Tests\Functional\Traits\DatabaseTransactionBehaviour;
use SwagImportExport\Components\DbAdapters\ProductsImagesDbAdapter;
use SwagImportExport\Components\DbAdapters\ProductsPricesDbAdapter;
use SwagImportExport\Components\Factories\DataTransformerFactory;
use SwagImportExport\Components\Factories\ProfileFactory;
use SwagImportExport\Components\Logger\Logger;
use SwagImportExport\Components\FileIO\FileReader;
use SwagImportExport\Components\Profile\Profile;
use SwagImportExport\Components\Providers\DataProvider;
use SwagImportExport\Components\Providers\FileIOProvider;
use SwagImportExport\Components\Service\DataWorkflow;
use SwagImportExport\Components\Session\Session;
use SwagImportExport\Components\Session\SessionService;
use SwagImportExport\Components\Structs\ImportRequest;
use SwagImportExport\Components\Transformers\DataTransformerChain;
use SwagImportExport\Models\Profile as ProfileModel;
use SwagImportExport\Tests\Functional\Components\Service\Mock\FileIoMock;
use SwagImportExport\Tests\Functional\Components\Service\Mock\FileIoMockWithBrokenRecords;
use SwagImportExport\Tests\Functional\Components\Service\Mock\LogMock;
use SwagImportExport\Tests\Functional\Components\Service\Mock\TransformerChainMock;
use SwagImportExport\Tests\Functional\Components\Service\Mock\TransformerChainWithBrokenRecordsMock;
use SwagImportExport\Tests\Helper\ContainerTrait;
use SwagImportExport\Tests\Helper\ReflectionHelperTrait;

class DataWorkflowTest extends TestCase
{
use ReflectionHelperTrait;
use ContainerTrait;
use DatabaseTransactionBehaviour;

Expand Down Expand Up @@ -72,9 +79,9 @@ public function testImportDoesNotOverwritePrices(): void

$session = $this->getMockBuilder(Session::class)->disableOriginalConstructor()->getMock();
$session->setTotalCount(4);
$session->method('getState')->willReturn('active');
$session->method('getState')->willReturn('new', 'active', 'finished');

$workflow = $this->getWorkflow();
$workflow = $this->getWorkflow(ProductsPricesDbAdapter::class, new FileIoMock(), new TransformerChainMock(), new LogMock());

$workflow->import($importRequest, $session);

Expand All @@ -93,7 +100,7 @@ public function testSaveUnprocessedDataShouldWriteHeader(): void
static::assertIsResource($handle);
\ftruncate($handle, 0);
\fclose($handle);
$workflow = $this->getWorkflow();
$workflow = $this->getWorkflow(ProductsPricesDbAdapter::class, new FileIoMock(), new TransformerChainMock(), new LogMock());
$workflow->saveUnprocessedData([], 'someProfileName', $file, 'old');

$expected = 'new | empty | header | test' . \PHP_EOL . 'just | another | return | value';
Expand All @@ -102,18 +109,70 @@ public function testSaveUnprocessedDataShouldWriteHeader(): void
static::assertSame($expected, $result);
}

private function getWorkflow(): DataWorkflow
public function testLogFileShouldHaveDistinctErrorMessages(): void
{
$dbAdapter = $this->getContainer()->get(ProductsPricesDbAdapter::class);
$productsImagesDbAdapter = $this->getContainer()->get(ProductsImagesDbAdapter::class);
$value = $this->getReflectionProperty(ProductsImagesDbAdapter::class, 'importExportErrorMode');
$value->setValue($productsImagesDbAdapter, true);

$customerGroupInsert = __DIR__ . '/../_fixtures/customerGroupInsert.sql';
$tree = __DIR__ . '/../_fixtures/tree.json';
$file = __DIR__ . '/../_fixtures/emptyFile.csv';

static::assertFileExists($file);
static::assertFileExists($tree);
static::assertFileExists($customerGroupInsert);

$conn = $this->getConnection();

$conn->executeQuery((string) file_get_contents($customerGroupInsert));

$profileModel = new ProfileModel();
$profileModel->setType('articlesImages');
$profileModel->setTree((string) file_get_contents($tree));
$profileModel->setName('default_article_images');
$profileEntity = new Profile($profileModel);

$importRequest = new ImportRequest();
$dataSet = [
['profileEntity' => $profileEntity],
['format' => 'csv'],
['inputFile' => $file],
];

foreach ($dataSet as $data) {
$importRequest->setData($data);
}

$session = $this->getMockBuilder(Session::class)->disableOriginalConstructor()->getMock();
$session->setTotalCount(4);
$session->method('getState')->willReturn('new', 'active', 'finished');

$logMock = new LogMock();
$workflow = $this->getWorkflow(ProductsImagesDbAdapter::class, new FileIoMockWithBrokenRecords(), new TransformerChainWithBrokenRecordsMock(), $logMock);

$workflow->import($importRequest, $session);

$logs = $logMock->getLogs();

static::assertStringContainsString('Kann file:///test/testme.png nicht zum Lesen öffnen', $logs[0][1]);
static::assertStringNotContainsString('Kann file:///test/help.jpg nicht zum Lesen öffnen', $logs[0][1]);

$value->setValue($productsImagesDbAdapter, false);
}

private function getWorkflow(string $adapterName, FileReader $fileReader, DataTransformerChain $transformerChain, LogMock $logMock): DataWorkflow
{
$dbAdapter = $this->getContainer()->get($adapterName);
$dataProvider = $this->getMockBuilder(DataProvider::class)->disableOriginalConstructor()->getMock();
$dataProvider->method('createDbAdapter')->willReturn($dbAdapter);
$dataTransformationFactory = $this->getMockBuilder(DataTransformerFactory::class)->disableOriginalConstructor()->getMock();
$dataTransformationFactory->method('createDataTransformerChain')->willReturn(new TransformerChainMock());
$dataTransformationFactory->method('createDataTransformerChain')->willReturn($transformerChain);
$fileIOProvider = $this->getMockBuilder(FileIOProvider::class)->disableOriginalConstructor()->getMock();
$fileIOProvider->method('getFileWriter')->willReturn(new FileIoMock());
$fileIOProvider->method('getFileReader')->willReturn(new FileIoMock());
$fileIOProvider->method('getFileWriter')->willReturn($fileReader);
$fileIOProvider->method('getFileReader')->willReturn($fileReader);
$sessionService = $this->getMockBuilder(SessionService::class)->disableOriginalConstructor()->getMock();
$logger = $this->getMockBuilder(Logger::class)->disableOriginalConstructor()->getMock();
$logger = $logMock;
$profileFactory = $this->getMockBuilder(ProfileFactory::class)->disableOriginalConstructor()->getMock();
$profileFactory->method('loadHiddenProfile')->willReturn(new Profile(new ProfileModel()));

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
<?php

declare(strict_types=1);
/**
* (c) shopware AG <[email protected]>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/

namespace SwagImportExport\Tests\Functional\Components\Service\Mock;

use SwagImportExport\Components\FileIO\FileReader;
use SwagImportExport\Components\FileIO\FileWriter;
use SwagImportExport\Tests\Functional\Components\_fixtures\DataSetBrokenProductImages;

class FileIoMockWithBrokenRecords implements FileWriter, FileReader
{
public function __construct()
{
}

/**
* @param mixed|null $headerData
*/
public function writeHeader(string $fileName, $headerData): void
{
\file_put_contents($fileName, $headerData);
}

/**
* @param mixed|null $treeData
*/
public function writeRecords(string $fileName, $treeData): void
{
\file_put_contents($fileName, $treeData, \FILE_APPEND);
}

public function getTotalCount(string $fileName): int
{
return 0;
}

/**
* @param array<string> $tree
*/
public function setTree(array $tree): void
{
}

/**
* @return array<string, array<int, array<string, string>>>
*/
public function readRecords(string $fileName, int $position, int $step): array
{
return DataSetBrokenProductImages::getDataSet();
}

public function supports(string $format): bool
{
return true;
}

public function writeFooter(string $fileName, ?array $footerData): void
{
}

public function hasTreeStructure(): bool
{
return false;
}
}
57 changes: 57 additions & 0 deletions Tests/Functional/Components/Service/Mock/LogMock.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
<?php

declare(strict_types=1);
/**
* (c) shopware AG <[email protected]>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/

namespace SwagImportExport\Tests\Functional\Components\Service\Mock;

use SwagImportExport\Components\Logger\LogDataStruct;
use SwagImportExport\Components\Logger\LoggerInterface;
use SwagImportExport\Components\Session\Session;

class LogMock implements LoggerInterface
{
/**
* @var array<array<string>|string>
*/
private array $logs;

/**
* @param array<string> $messages
*/
public function write(array $messages, string $status, Session $session): void
{
$this->logs[] = $messages;
}

public function logProcessing(string $writeStatus, string $filename, string $profileName, string $logMessage, string $status, Session $session): void
{
$logDataStruct = new LogDataStruct(
\date('Y-m-d H:i:s'),
$filename,
$profileName,
$logMessage,
$status
);

$this->writeToFile($logDataStruct);
}

public function writeToFile(LogDataStruct $logDataStruct): void
{
$this->logs[] = $logDataStruct->getMessages();
}

/**
* @return array<array<string>|string>
*/
public function getLogs(): array
{
return $this->logs;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
<?php
declare(strict_types=1);
/**
* (c) shopware AG <[email protected]>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/

namespace SwagImportExport\Tests\Functional\Components\Service\Mock;

use SwagImportExport\Components\Transformers\DataTransformerChain;
use SwagImportExport\Tests\Functional\Components\_fixtures\DataSetBrokenProductImages;

class TransformerChainWithBrokenRecordsMock extends DataTransformerChain
{
public function __construct()
{
// DO NOTHING
}

/**
* @return array<string>
*/
public function composeHeader(): array
{
return ['new | empty | header | test'];
}

/**
* @param array<string, array<int, mixed>> $data
*
* @return array<string>
*/
public function transformForward($data): array
{
return [\PHP_EOL . 'just | another | return | value'];
}

/**
* @param array<string, mixed> $data
*
* @return array<string, array<int, array<string, string>>>
*/
public function transformBackward(array $data): array
{
return DataSetBrokenProductImages::getFixedDataSet();
}
}
Loading

0 comments on commit eeab0d5

Please sign in to comment.