From a268785ecc183f6543b15370991833933b76eacb Mon Sep 17 00:00:00 2001 From: Mathieu Ledru Date: Sun, 13 Oct 2024 02:53:14 +0200 Subject: [PATCH] :sparkles: Add php-etl pipeline --- composer.json | 1 + composer.lock | 284 +++++++++++++++++++++++++++++++++- src/Command/PhpEtlCommand.php | 122 +++++++++++++++ tools/phpstan/composer.json | 3 +- tools/phpstan/composer.lock | 284 +++++++++++++++++++++++++++++++++- 5 files changed, 691 insertions(+), 3 deletions(-) create mode 100644 src/Command/PhpEtlCommand.php diff --git a/composer.json b/composer.json index 59d1edd..e363209 100644 --- a/composer.json +++ b/composer.json @@ -22,6 +22,7 @@ "doctrine/doctrine-bundle": "^2.12", "doctrine/doctrine-migrations-bundle": "^3.3", "doctrine/orm": "^3.2", + "php-etl/pipeline": "^0.6.1", "phpdocumentor/reflection-docblock": "^5.4", "phpstan/phpdoc-parser": "^1.29", "react/async": "^4.3", diff --git a/composer.lock b/composer.lock index 19bfba0..e4fea91 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "64ed88553514526932414a91784ce9f4", + "content-hash": "15e7e669c49cf73114dbfbcf3ea52b64", "packages": [ { "name": "amphp/amp", @@ -1761,6 +1761,288 @@ ], "time": "2024-06-28T09:40:51+00:00" }, + { + "name": "php-etl/bucket", + "version": "v0.4.0", + "source": { + "type": "git", + "url": "https://github.com/php-etl/bucket.git", + "reference": "8b3f318398ceaca1fb25154ff8a06adf175b1c31" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-etl/bucket/zipball/8b3f318398ceaca1fb25154ff8a06adf175b1c31", + "reference": "8b3f318398ceaca1fb25154ff8a06adf175b1c31", + "shasum": "" + }, + "require": { + "php": "^8.2", + "php-etl/bucket-contracts": "0.3.*" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.38", + "phpspec/phpspec": "^7.3", + "phpstan/phpstan": "^1.10", + "rector/rector": "^0.15" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "0.4.x-dev" + } + }, + "autoload": { + "psr-4": { + "Kiboko\\Component\\Bucket\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Kiboko SAS", + "homepage": "http://kiboko.fr" + }, + { + "name": "Grégory Planchat", + "email": "gregory@kiboko.fr" + } + ], + "description": "This library implements the Extract-Transform-Load pattern asynchronously in PHP with the help of iterators and generators", + "support": { + "issues": "https://github.com/php-etl/bucket/issues", + "source": "https://github.com/php-etl/bucket/tree/v0.4.0" + }, + "time": "2023-11-16T15:32:18+00:00" + }, + { + "name": "php-etl/bucket-contracts", + "version": "v0.3.0", + "source": { + "type": "git", + "url": "https://github.com/php-etl/bucket-contracts.git", + "reference": "64170754cb1f5e556c8e526177be51d905c3b15f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-etl/bucket-contracts/zipball/64170754cb1f5e556c8e526177be51d905c3b15f", + "reference": "64170754cb1f5e556c8e526177be51d905c3b15f", + "shasum": "" + }, + "require": { + "php": "^8.2" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.0", + "phpstan/phpstan": "^1.10", + "rector/rector": "^0.15" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "0.3.x-dev" + } + }, + "autoload": { + "psr-4": { + "Kiboko\\Contract\\Bucket\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Kiboko SAS", + "homepage": "http://kiboko.fr" + }, + { + "name": "Grégory Planchatn rules", + "email": "gregory@kiboko.fr" + } + ], + "description": "This library describes contracts for the data-interchange buckets.", + "support": { + "issues": "https://github.com/php-etl/bucket-contracts/issues", + "source": "https://github.com/php-etl/bucket-contracts/tree/v0.3.0" + }, + "time": "2023-11-14T14:39:36+00:00" + }, + { + "name": "php-etl/pipeline", + "version": "v0.6.1", + "source": { + "type": "git", + "url": "https://github.com/php-etl/pipeline.git", + "reference": "46621f5208cb2b530309c737e846c37cfb64078c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-etl/pipeline/zipball/46621f5208cb2b530309c737e846c37cfb64078c", + "reference": "46621f5208cb2b530309c737e846c37cfb64078c", + "shasum": "" + }, + "require": { + "php": "^8.2", + "php-etl/bucket": "*", + "php-etl/bucket-contracts": "0.3.*", + "php-etl/pipeline-contracts": "0.5.*", + "psr/log": "^3.0" + }, + "provide": { + "php-etl/pipeline-implementation": "0.5.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.38", + "infection/infection": "^0.26.18", + "phpstan/phpstan": "^1.10", + "phpunit/php-invoker": "^4.0", + "phpunit/phpunit": "^10.0", + "rector/rector": "^0.15" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "0.6.x-dev" + } + }, + "autoload": { + "psr-4": { + "Kiboko\\Component\\Pipeline\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Kiboko SAS", + "homepage": "http://kiboko.fr" + }, + { + "name": "Grégory Planchat", + "email": "gregory@kiboko.fr" + } + ], + "description": "This library implements the Extract-Transform-Load pattern asynchronously in PHP with the help of iterators and generators", + "support": { + "issues": "https://github.com/php-etl/pipeline/issues", + "source": "https://github.com/php-etl/pipeline/tree/v0.6.1" + }, + "time": "2023-11-16T15:32:18+00:00" + }, + { + "name": "php-etl/pipeline-contracts", + "version": "v0.5.1", + "source": { + "type": "git", + "url": "https://github.com/php-etl/pipeline-contracts.git", + "reference": "24c71241d16b7deca56d67ea0f1d591c8ae52f09" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-etl/pipeline-contracts/zipball/24c71241d16b7deca56d67ea0f1d591c8ae52f09", + "reference": "24c71241d16b7deca56d67ea0f1d591c8ae52f09", + "shasum": "" + }, + "require": { + "php": "^8.2", + "php-etl/bucket-contracts": "0.3.*", + "php-etl/satellite-contracts": "0.1.*" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.38", + "phpstan/phpstan": "^1.10", + "rector/rector": "^0.15" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "0.5.x-dev" + } + }, + "autoload": { + "psr-4": { + "Kiboko\\Contract\\Pipeline\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Kiboko SAS", + "homepage": "http://kiboko.fr" + }, + { + "name": "Grégory Planchat", + "email": "gregory@kiboko.fr" + } + ], + "description": "This library describes contracts for the Extract-Transform-Load pattern.", + "support": { + "issues": "https://github.com/php-etl/pipeline-contracts/issues", + "source": "https://github.com/php-etl/pipeline-contracts/tree/v0.5.1" + }, + "time": "2023-11-20T10:48:56+00:00" + }, + { + "name": "php-etl/satellite-contracts", + "version": "v0.1.1", + "source": { + "type": "git", + "url": "https://github.com/php-etl/satellite-contracts.git", + "reference": "d2be591800f42e460a59d864888aedf00f111dd7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-etl/satellite-contracts/zipball/d2be591800f42e460a59d864888aedf00f111dd7", + "reference": "d2be591800f42e460a59d864888aedf00f111dd7", + "shasum": "" + }, + "require": { + "php": "^8.2" + }, + "require-dev": { + "rector/rector": "^0.15" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "0.1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Kiboko\\Contract\\Satellite\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Kiboko SAS", + "homepage": "http://kiboko.fr" + }, + { + "name": "Grégory Planchat", + "email": "gregory@kiboko.fr" + } + ], + "description": "This library describes contracts for defining satellite formats", + "support": { + "issues": "https://github.com/php-etl/satellite-contracts/issues", + "source": "https://github.com/php-etl/satellite-contracts/tree/v0.1.1" + }, + "time": "2023-11-20T10:48:56+00:00" + }, { "name": "php-webdriver/webdriver", "version": "1.15.1", diff --git a/src/Command/PhpEtlCommand.php b/src/Command/PhpEtlCommand.php new file mode 100644 index 0000000..6a8a9f8 --- /dev/null +++ b/src/Command/PhpEtlCommand.php @@ -0,0 +1,122 @@ +extract( + StepCode::fromString('extractor'), + new class($data) implements ExtractorInterface { + /** @param array $data */ + public function __construct(private array $data) {} + + public function extract(): iterable + { + foreach ($this->data as $item) { + yield new AcceptanceResultBucket([$item]); + } + } + }, + new NullStepRejection(), + new NullStepState() + ); + + return $pipeline; + }; + yield static function (Pipeline $pipeline) { + $pipeline->transform( + StepCode::fromString('transformer'), + new class() implements TransformerInterface, + FlushableInterface { + /** @return Generator> */ + public function transform(): Generator + { + $line = yield; + $line = yield new AcceptanceResultBucket(array_map(static fn (string $item) => str_rot13($item), $line)); + $line = yield new AcceptanceResultBucket(array_map(static fn (string $item) => str_rot13($item), $line)); + yield new AcceptanceResultBucket(array_map(static fn (string $item) => str_rot13($item), $line)); + } + + public function flush(): ResultBucketInterface + { + return new AcceptanceResultBucket([str_rot13('sit amet')]); + } + }, + new NullStepRejection(), + new NullStepState() + ); + + return $pipeline; + }; + yield static function (Pipeline $pipeline) { + $pipeline->load( + StepCode::fromString('loader'), + new class() implements LoaderInterface, + FlushableInterface { + /** @return Generator> */ + public function load(): Generator + { + $line = yield; + $line = yield new AcceptanceResultBucket(array_map(static fn (string $item) => str_rot13($item), $line)); + $line = yield new AcceptanceResultBucket(array_map(static fn (string $item) => str_rot13($item), $line)); + yield new AcceptanceResultBucket(array_map(static fn (string $item) => str_rot13($item), $line)); + } + + public function flush(): ResultBucketInterface + { + return new AcceptanceResultBucket([str_rot13('sit amet')]); + } + }, + new NullStepRejection(), + new NullStepState() + ); + + return $pipeline; + }; + yield static function (Pipeline $pipeline) { + foreach ($pipeline->walk() as $items) { + printf("%s\n", implode(',', $items)); + } + }; + }); + + $flow(new Ip(['lorem', 'ipsum'])); + $flow(new Ip(['dolor'])); + + $flow->await(); + + return Command::SUCCESS; + } +} diff --git a/tools/phpstan/composer.json b/tools/phpstan/composer.json index a460156..a78db70 100644 --- a/tools/phpstan/composer.json +++ b/tools/phpstan/composer.json @@ -44,7 +44,8 @@ "symfony/web-link": "7.1.*", "symfony/yaml": "7.1.*", "twig/extra-bundle": "^2.12|^3.0", - "twig/twig": "^2.12|^3.0" + "twig/twig": "^2.12|^3.0", + "php-etl/pipeline": "^0.6.1" }, "require-dev": { "amphp/amp": "^3.0", diff --git a/tools/phpstan/composer.lock b/tools/phpstan/composer.lock index e728ed7..af79342 100644 --- a/tools/phpstan/composer.lock +++ b/tools/phpstan/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "35f310c6cb0db6d941c87935e15a8dc2", + "content-hash": "1c4df4e773f454cb5235e4e9ed8335d7", "packages": [ { "name": "amphp/amp", @@ -1761,6 +1761,288 @@ ], "time": "2024-06-28T09:40:51+00:00" }, + { + "name": "php-etl/bucket", + "version": "v0.4.0", + "source": { + "type": "git", + "url": "https://github.com/php-etl/bucket.git", + "reference": "8b3f318398ceaca1fb25154ff8a06adf175b1c31" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-etl/bucket/zipball/8b3f318398ceaca1fb25154ff8a06adf175b1c31", + "reference": "8b3f318398ceaca1fb25154ff8a06adf175b1c31", + "shasum": "" + }, + "require": { + "php": "^8.2", + "php-etl/bucket-contracts": "0.3.*" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.38", + "phpspec/phpspec": "^7.3", + "phpstan/phpstan": "^1.10", + "rector/rector": "^0.15" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "0.4.x-dev" + } + }, + "autoload": { + "psr-4": { + "Kiboko\\Component\\Bucket\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Kiboko SAS", + "homepage": "http://kiboko.fr" + }, + { + "name": "Grégory Planchat", + "email": "gregory@kiboko.fr" + } + ], + "description": "This library implements the Extract-Transform-Load pattern asynchronously in PHP with the help of iterators and generators", + "support": { + "issues": "https://github.com/php-etl/bucket/issues", + "source": "https://github.com/php-etl/bucket/tree/v0.4.0" + }, + "time": "2023-11-16T15:32:18+00:00" + }, + { + "name": "php-etl/bucket-contracts", + "version": "v0.3.0", + "source": { + "type": "git", + "url": "https://github.com/php-etl/bucket-contracts.git", + "reference": "64170754cb1f5e556c8e526177be51d905c3b15f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-etl/bucket-contracts/zipball/64170754cb1f5e556c8e526177be51d905c3b15f", + "reference": "64170754cb1f5e556c8e526177be51d905c3b15f", + "shasum": "" + }, + "require": { + "php": "^8.2" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.0", + "phpstan/phpstan": "^1.10", + "rector/rector": "^0.15" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "0.3.x-dev" + } + }, + "autoload": { + "psr-4": { + "Kiboko\\Contract\\Bucket\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Kiboko SAS", + "homepage": "http://kiboko.fr" + }, + { + "name": "Grégory Planchatn rules", + "email": "gregory@kiboko.fr" + } + ], + "description": "This library describes contracts for the data-interchange buckets.", + "support": { + "issues": "https://github.com/php-etl/bucket-contracts/issues", + "source": "https://github.com/php-etl/bucket-contracts/tree/v0.3.0" + }, + "time": "2023-11-14T14:39:36+00:00" + }, + { + "name": "php-etl/pipeline", + "version": "v0.6.1", + "source": { + "type": "git", + "url": "https://github.com/php-etl/pipeline.git", + "reference": "46621f5208cb2b530309c737e846c37cfb64078c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-etl/pipeline/zipball/46621f5208cb2b530309c737e846c37cfb64078c", + "reference": "46621f5208cb2b530309c737e846c37cfb64078c", + "shasum": "" + }, + "require": { + "php": "^8.2", + "php-etl/bucket": "*", + "php-etl/bucket-contracts": "0.3.*", + "php-etl/pipeline-contracts": "0.5.*", + "psr/log": "^3.0" + }, + "provide": { + "php-etl/pipeline-implementation": "0.5.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.38", + "infection/infection": "^0.26.18", + "phpstan/phpstan": "^1.10", + "phpunit/php-invoker": "^4.0", + "phpunit/phpunit": "^10.0", + "rector/rector": "^0.15" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "0.6.x-dev" + } + }, + "autoload": { + "psr-4": { + "Kiboko\\Component\\Pipeline\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Kiboko SAS", + "homepage": "http://kiboko.fr" + }, + { + "name": "Grégory Planchat", + "email": "gregory@kiboko.fr" + } + ], + "description": "This library implements the Extract-Transform-Load pattern asynchronously in PHP with the help of iterators and generators", + "support": { + "issues": "https://github.com/php-etl/pipeline/issues", + "source": "https://github.com/php-etl/pipeline/tree/v0.6.1" + }, + "time": "2023-11-16T15:32:18+00:00" + }, + { + "name": "php-etl/pipeline-contracts", + "version": "v0.5.1", + "source": { + "type": "git", + "url": "https://github.com/php-etl/pipeline-contracts.git", + "reference": "24c71241d16b7deca56d67ea0f1d591c8ae52f09" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-etl/pipeline-contracts/zipball/24c71241d16b7deca56d67ea0f1d591c8ae52f09", + "reference": "24c71241d16b7deca56d67ea0f1d591c8ae52f09", + "shasum": "" + }, + "require": { + "php": "^8.2", + "php-etl/bucket-contracts": "0.3.*", + "php-etl/satellite-contracts": "0.1.*" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.38", + "phpstan/phpstan": "^1.10", + "rector/rector": "^0.15" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "0.5.x-dev" + } + }, + "autoload": { + "psr-4": { + "Kiboko\\Contract\\Pipeline\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Kiboko SAS", + "homepage": "http://kiboko.fr" + }, + { + "name": "Grégory Planchat", + "email": "gregory@kiboko.fr" + } + ], + "description": "This library describes contracts for the Extract-Transform-Load pattern.", + "support": { + "issues": "https://github.com/php-etl/pipeline-contracts/issues", + "source": "https://github.com/php-etl/pipeline-contracts/tree/v0.5.1" + }, + "time": "2023-11-20T10:48:56+00:00" + }, + { + "name": "php-etl/satellite-contracts", + "version": "v0.1.1", + "source": { + "type": "git", + "url": "https://github.com/php-etl/satellite-contracts.git", + "reference": "d2be591800f42e460a59d864888aedf00f111dd7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-etl/satellite-contracts/zipball/d2be591800f42e460a59d864888aedf00f111dd7", + "reference": "d2be591800f42e460a59d864888aedf00f111dd7", + "shasum": "" + }, + "require": { + "php": "^8.2" + }, + "require-dev": { + "rector/rector": "^0.15" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "0.1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Kiboko\\Contract\\Satellite\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Kiboko SAS", + "homepage": "http://kiboko.fr" + }, + { + "name": "Grégory Planchat", + "email": "gregory@kiboko.fr" + } + ], + "description": "This library describes contracts for defining satellite formats", + "support": { + "issues": "https://github.com/php-etl/satellite-contracts/issues", + "source": "https://github.com/php-etl/satellite-contracts/tree/v0.1.1" + }, + "time": "2023-11-20T10:48:56+00:00" + }, { "name": "php-webdriver/webdriver", "version": "1.15.1",