Skip to content

Commit

Permalink
Merge pull request #22 from ttreeagency/task-improvement
Browse files Browse the repository at this point in the history 
TASK: The road to 3.1
  • Loading branch information
@dfeyer
dfeyer authored Jul 26, 2017
2 parents 97dd6d3 + d0a7e93 commit e206ac6
Show file tree
Hide file tree
Showing 40 changed files with 735 additions and 243 deletions.
4 changes: 0 additions & 4 deletions Classes/Aspect/EventLogAspect.php
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
<?php
namespace Ttree\ContentRepositoryImporter\Aspect;

/*
* This script belongs to the Neos Flow package "Ttree.ContentRepositoryImporter".
*/

use Ttree\ContentRepositoryImporter\Domain\Service\ImportService;
use Ttree\ContentRepositoryImporter\Importer\ImporterInterface;
use Neos\Flow\Annotations as Flow;
Expand Down
171 changes: 137 additions & 34 deletions Classes/Command/ImportCommandController.php
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
<?php
namespace Ttree\ContentRepositoryImporter\Command;

/*
* This script belongs to the Neos Flow package "Ttree.ContentRepositoryImporter".
*/

use Ttree\ContentRepositoryImporter\DataProvider\DataProviderInterface;
use Ttree\ContentRepositoryImporter\Domain\Model\PresetPartDefinition;
use Ttree\ContentRepositoryImporter\Domain\Model\RecordMapping;
use Ttree\ContentRepositoryImporter\Domain\Repository\EventRepository;
use Ttree\ContentRepositoryImporter\Domain\Repository\RecordMappingRepository;
use Ttree\ContentRepositoryImporter\Domain\Service\ImportService;
use Ttree\ContentRepositoryImporter\Exception\ImportAlreadyExecutedException;
use Ttree\ContentRepositoryImporter\Importer\AbstractImporter;
Expand All @@ -19,6 +17,7 @@
use Neos\Flow\ObjectManagement\ObjectManagerInterface;
use Neos\Utility\Arrays;
use Neos\Neos\EventLog\Domain\Service\EventEmittingService;
use Ttree\ContentRepositoryImporter\Service\Vault;

/**
* Import Command Controller
Expand All @@ -39,6 +38,12 @@ class ImportCommandController extends CommandController
*/
protected $eventLogRepository;

/**
* @Flow\Inject
* @var RecordMappingRepository
*/
protected $recordMapperRepository;

/**
* @Flow\InjectConfiguration(package="Neos.Flow")
* @var array
Expand Down Expand Up @@ -92,6 +97,55 @@ public function injectSettings(array $settings)
$this->settings = $settings;
}

/**
* Reset the mapping between external identifier and local nodes
*
* @param string $preset
* @param string $parts
*/
public function initCommand($preset, $parts = null)
{
$parts = Arrays::trimExplode(',', $parts);
$presetSettings = $this->loadPreset($preset);
array_walk($presetSettings['parts'], function ($partSetting, $partName) use ($preset, $parts) {
$this->outputLine();
$this->outputPartTitle($partSetting, $partName);

if ($parts !== array() && !in_array($partName, $parts)) {
$this->outputLine('<error>~</error> Skipped');
return;
}

if (!isset($partSetting['importerClassName'])) {
$this->outputLine('<error>Missing importerClassName in the current preset part (%s/%s), check your settings</error>', [$preset, $partName]);
return;
}

$identifier = $partSetting['importerClassName'] . '@' . $preset . '/' . $partName;
/** @var RecordMapping $recordMapper */
foreach ($this->recordMapperRepository->findByImporterClassName($identifier) as $recordMapper) {
$this->recordMapperRepository->remove($recordMapper);
}
});
$vault = new Vault($preset);
$vault->flush();
}

/**
* Show the different pars of the preset
*
* @param string $preset
* @param string $parts
*/
public function showCommand($preset)
{
$presetSettings = $this->loadPreset($preset);
array_walk($presetSettings['parts'], function ($partSetting, $partName) use ($preset) {
$this->outputLine();
$this->outputPartTitle($partSetting, $partName);
});
}

/**
* Run batch import
*
Expand All @@ -113,14 +167,14 @@ public function injectSettings(array $settings)
* @param string $preset Name of the preset which holds the configuration for the import
* @param string $parts Optional comma separated names of parts. If no parts are specified, all parts will be imported.
* @param integer $batchSize Number of records to import at a time. If not specified, the batch size defined in the preset will be used.
* @param string $externalImportIdentifier External identifier which is used for checking if an import of the same data has already been executed earlier.
* @param string $identifier External identifier which is used for checking if an import of the same data has already been executed earlier.
* @param boolean $force If set, an import will even be executed if it ran earlier with the same external import identifier.
* @return void
*/
public function batchCommand($preset, $parts = null, $batchSize = null, $externalImportIdentifier = null, $force = false)
public function batchCommand($preset, $parts = null, $batchSize = null, $identifier = null, $force = false)
{
try {
$this->importService->start($externalImportIdentifier, $force);
$this->importService->start($identifier, $force);
} catch (ImportAlreadyExecutedException $e) {
$this->outputLine($e->getMessage());
$this->outputLine('Import skipped. You can force running this import again by specifying --force.');
Expand All @@ -130,31 +184,25 @@ public function batchCommand($preset, $parts = null, $batchSize = null, $externa
$this->startTime = microtime(true);
$parts = Arrays::trimExplode(',', $parts);

$this->outputLine('Start import ...');
$presetSettings = Arrays::getValueByPath($this->settings, array('presets', $preset));
if (!is_array($presetSettings)) {
$this->outputLine(sprintf('Preset "%s" not found ...', $preset));
$this->quit(1);
}

$this->checkForPartsSettingsOrQuit($presetSettings, $preset);

$identifier = $this->importService->getCurrentImportIdentifier();
$this->outputLine('Start import with identifier <b>%s</b>', [$identifier]);

array_walk($presetSettings['parts'], function ($partSetting, $partName) use ($preset, $parts, $batchSize) {
$presetSettings = $this->loadPreset($preset);
array_walk($presetSettings['parts'], function ($partSetting, $partName) use ($preset, $parts, $batchSize, $identifier) {
$this->elapsedTime = 0;
$this->batchCounter = 0;
$this->outputLine();
$this->outputFormatted(sprintf('<b>%s</b>', $partSetting['label']));
$this->outputPartTitle($partSetting, $partName);

$partSetting['__currentPresetName'] = $preset;
$partSetting['__currentPartName'] = $partName;
if ($batchSize !== null) {
$partSetting['batchSize'] = $batchSize;
}

$partSetting = new PresetPartDefinition($partSetting, $this->importService->getCurrentImportIdentifier());
$partSetting = new PresetPartDefinition($partSetting, $identifier);
if ($parts !== array() && !in_array($partName, $parts)) {
$this->outputLine('Skipped');
$this->outputLine('<error>~</error> Skipped');
return;
}

Expand All @@ -173,14 +221,31 @@ public function batchCommand($preset, $parts = null, $batchSize = null, $externa
$import = $this->importService->getLastImport();

$this->outputLine();
$this->outputLine('Import finished.');
$this->outputLine(sprintf(' Started %s', $import->getStartTime()->format(DATE_RFC2822)));
$this->outputLine(sprintf(' Finished %s', $import->getEndTime()->format(DATE_RFC2822)));
$this->outputLine(sprintf(' Runtime %d seconds', $import->getElapsedTime()));
$this->outputLine('<b>Import finished</b>');
$this->outputLine(sprintf('<info>-</info> Started %s', $import->getStartTime()->format(DATE_RFC2822)));
$this->outputLine(sprintf('<info>-</info> Finished %s', $import->getEndTime()->format(DATE_RFC2822)));
$this->outputLine(sprintf('<info>-</info> Runtime %d seconds', $import->getElapsedTime()));
$this->outputLine();
$this->outputLine('See log for more details and possible errors.');
}

/**
* @param string $preset
* @return array
*/
protected function loadPreset($preset)
{
$presetSettings = Arrays::getValueByPath($this->settings, ['presets', $preset]);
if (!is_array($presetSettings)) {
$this->outputLine(sprintf('Preset "%s" not found ...', $preset));
$this->quit(1);
}

$this->checkForPartsSettingsOrQuit($presetSettings, $preset);

return $presetSettings;
}

/**
* Execute a sub process which imports a batch as specified by the part definition.
*
Expand All @@ -196,25 +261,46 @@ protected function executeCommand(PresetPartDefinition $partSetting)
$startTime = microtime(true);

++$this->batchCounter;
ob_start();
$status = Scripts::executeCommand('ttree.contentrepositoryimporter:import:executebatch', $this->flowSettings, true, $partSetting->getCommandArguments());
ob_start(NULL, 1<<20);
$commandIdentifier = 'ttree.contentrepositoryimporter:import:executebatch';
$status = Scripts::executeCommand($commandIdentifier, $this->flowSettings, true, $partSetting->getCommandArguments());
if ($status !== true) {
throw new Exception('Sub command failed', 1426767159);
throw new Exception(\vsprintf('Command: %s with parameters: %s', [$commandIdentifier, \json_encode($partSetting->getCommandArguments())]), 1426767159);
}
$count = (integer)ob_get_clean();
if ($count < 1) {
return 0;
$output = explode(\PHP_EOL, ob_get_clean());
if (count($output) > 1) {
$this->outputLine('<error>+</error> <b>Command "%s"</b>', [$commandIdentifier]);
$this->outputLine('<error>+</error> <comment> with parameters:</comment>');
foreach ($partSetting->getCommandArguments() as $argumentName => $argumentValue) {
$this->outputLine('<error>+</error> %s: %s', [$argumentName, $argumentValue]);
}
foreach ($output as $line) {
$line = trim($line);
if ($line === '') {
continue;
}
$this->outputLine('<error>+</error> %s', [$line]);
}
}
$count = (int)\array_pop($output);
$count = $count < 1 ? 0 : $count;

$elapsedTime = (microtime(true) - $startTime) * 1000;
$this->elapsedTime += $elapsedTime;
$this->outputLine(' #%d %d records in %dms, %d ms per record, %d ms per batch (avg)', [$partSetting->getCurrentBatch(), $count, $elapsedTime, $elapsedTime / $count, $this->elapsedTime / $this->batchCounter]);
$this->outputLine('<info>+</info> #%d %d records in %dms, %d ms per record, %d ms per batch (avg)', [
$partSetting->getCurrentBatch(),
$count,
$elapsedTime,
($count > 0 ? $elapsedTime / $count : $elapsedTime),
($this->batchCounter > 0 ? $this->elapsedTime / $this->batchCounter : $this->elapsedTime)
]);
$this->importService->addEvent(sprintf('%s:Ended', $partSetting->getEventType()), null, $partSetting->getCommandArguments());
$this->importService->persistEntities();
return $count;
} catch (\Exception $exception) {
$this->logger->logException($exception);
$this->outputLine("Error, please check your logs ...", [$partSetting->getLabel()]);
$this->outputLine('Error in parts "%s", please check your logs for more details', [$partSetting->getLabel()]);
$this->outputLine('<error>%s</error>', [$exception->getMessage()]);
$this->importService->addEvent(sprintf('%s:Failed', $partSetting->getEventType()), null, $partSetting->getCommandArguments());
$this->quit(1);
}
Expand All @@ -239,23 +325,31 @@ protected function executeCommand(PresetPartDefinition $partSetting)
public function executeBatchCommand($presetName, $partName, $dataProviderClassName, $importerClassName, $currentImportIdentifier, $offset = null, $batchSize = null)
{
try {
$vault = new Vault($presetName);

$dataProviderOptions = Arrays::getValueByPath($this->settings, implode('.', ['presets', $presetName, 'parts', $partName, 'dataProviderOptions']));
$dataProviderOptions['__presetName'] = $presetName;
$dataProviderOptions['__partName'] = $partName;

/** @var DataProviderInterface $dataProvider */
$dataProvider = $dataProviderClassName::create(is_array($dataProviderOptions) ? $dataProviderOptions : [], $offset, $batchSize);

$importerOptions = Arrays::getValueByPath($this->settings, ['presets', $presetName, 'parts', $partName, 'importerOptions']);

/** @var AbstractImporter $importer */
$importer = $this->objectManager->get($importerClassName, is_array($importerOptions) ? $importerOptions : [], $currentImportIdentifier);
$importerOptions = is_array($importerOptions) ? $importerOptions : [];
$importerOptions['__presetName'] = $presetName;
$importerOptions['__partName'] = $partName;
$importer = $this->objectManager->get($importerClassName, $importerOptions, $currentImportIdentifier, $vault);
$importer->getImportService()->addEventMessage(sprintf('%s:Batch:Started', $importerClassName), sprintf('%s batch started (%s)', $importerClassName, $dataProviderClassName));
$importer->initialize($dataProvider);
$importer->process();
$importer->getImportService()->addEventMessage(sprintf('%s:Batch:Ended', $importerClassName), sprintf('%s batch ended (%s)', $importerClassName, $dataProviderClassName));
$this->output($importer->getProcessedRecords());
} catch (\Exception $exception) {
$this->logger->logException($exception);
$this->quit(1);
$this->outputLine('<error>%s</error>', [$exception->getMessage()]);
$this->sendAndExit(1);
}
}

Expand All @@ -271,6 +365,15 @@ public function flushEventLogCommand()
$this->eventLogRepository->removeAll();
}

/**
* @param array $partSetting
* @param string $partName
*/
protected function outputPartTitle(array $partSetting, $partName)
{
$this->outputFormatted(sprintf('<info>+</info> <b>%s</b> (%s)', $partSetting['label'], $partName));
}

/**
* Checks if the preset settings contain a "parts" segment and quits if it does not.
*
Expand Down
29 changes: 23 additions & 6 deletions Classes/DataProvider/AbstractDataProvider.php
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
<?php
namespace Ttree\ContentRepositoryImporter\DataProvider;

/*
* This script belongs to the Neos Flow package "Ttree.ContentRepositoryImporter".
*/

use Neos\Cache\Frontend\VariableFrontend;
use Ttree\ContentRepositoryImporter\Service\ProcessedNodeService;
use Neos\Flow\Annotations as Flow;
use Neos\Flow\Log\SystemLoggerInterface;
use Ttree\ContentRepositoryImporter\Service\Vault;

/**
* Abstract Data Provider
Expand All @@ -30,6 +28,11 @@ abstract class AbstractDataProvider implements DataProviderInterface
*/
protected $processedNodeService;

/**
* @var Vault
*/
protected $vault;

/**
* @var integer
* @api
Expand All @@ -55,13 +58,27 @@ abstract class AbstractDataProvider implements DataProviderInterface
*/
protected $options = [];

/**
* @var string
*/
protected $presetName;

/**
* @var string
*/
protected $partName;

/**
* @param array $options
* @param Vault|null $vault
* @api
*/
public function __construct(array $options)
public function __construct(array $options, Vault $vault)
{
$this->options = $options;
$this->presetName = $options['__presetName'];
$this->partName = $options['__partName'];
$this->vault = $vault;
}

/**
Expand All @@ -75,7 +92,7 @@ public function __construct(array $options)
*/
public static function create(array $options = [], $offset = null, $limit = null)
{
$dataProvider = new static($options);
$dataProvider = new static($options, new Vault($options['__presetName']));
$dataProvider->setOffset($offset);
$dataProvider->setLimit($limit);

Expand Down
4 changes: 0 additions & 4 deletions Classes/DataProvider/AbstractDatabaseDataProvider.php
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
<?php
namespace Ttree\ContentRepositoryImporter\DataProvider;

/*
* This script belongs to the Neos Flow package "Ttree.ContentRepositoryImporter".
*/

use Doctrine\DBAL\Configuration;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\DriverManager;
Expand Down
Loading

0 comments on commit e206ac6

Please sign in to comment.