Skip to content

Commit

Permalink
✨ Add scrap
Browse files Browse the repository at this point in the history
  • Loading branch information
matyo91 committed Sep 4, 2024
1 parent 22b4dad commit 7c3d4bd
Show file tree
Hide file tree
Showing 4 changed files with 270 additions and 0 deletions.
80 changes: 80 additions & 0 deletions src/Command/ScrapCommand.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
<?php

declare(strict_types=1);

namespace App\Command;

use App\Job\FlowExamples\ScrapUrlJob;
use App\Job\FlowExamples\ScrapUrlsJob;
use Flow\Driver\FiberDriver;
use Flow\Flow\Flow;
use Flow\Ip;
use Flow\IpStrategy\FlattenIpStrategy;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;

use function sprintf;

#[AsCommand(
name: 'app:scrap',
description: 'This allows scrap pages with flow',
)]
class ScrapCommand extends Command
{
protected function execute(InputInterface $input, OutputInterface $output): int
{
$io = new SymfonyStyle($input, $output);

$driver = new FiberDriver();

$flow = Flow::do(static function () {
// yield new ScrapUrlsJob();
// yield static function ($responses) use ($io) {
// //printf('%s' . "\n", $responses['content']);
// $io->writeln(sprintf('Finished scrapping %d urls', count($responses)));
// };
yield [new ScrapUrlJob(), null, new FlattenIpStrategy()];
yield static function ($data) {
dump($data['url']);
// $io->writeln(sprintf('Finished scrapping %d urls', count($responses)));
};
}, ['driver' => $driver]);

$urls = [
'https://www.google.fr',
'https://www.apple.com',
'https://www.microsoft.com',
'https://www.amazon.com',
'https://www.facebook.com',
'https://www.netflix.com',
'https://www.spotify.com',
'https://www.wikipedia.org',
'https://www.x.com',
'https://www.instagram.com',
'https://www.linkedin.com',
'https://www.reddit.com',
'https://www.ebay.com',
'https://www.cnn.com',
'https://www.bbc.co.uk',
'https://www.yahoo.com',
'https://www.bing.com',
'https://www.pinterest.com',
'https://www.tumblr.com',
'https://www.paypal.com',
'https://www.dropbox.com',
'https://www.adobe.com',
'https://www.salesforce.com',
];

$flow(new Ip($urls));

$flow->await();

$io->success('Scraping is done.');

return Command::SUCCESS;
}
}
71 changes: 71 additions & 0 deletions src/IpStrategy/FlattenIpStrategy.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
<?php

declare(strict_types=1);

namespace Flow\IpStrategy;

use Flow\Event;
use Flow\Event\PoolEvent;
use Flow\Event\PullEvent;
use Flow\Event\PushEvent;
use Flow\Exception\LogicException;
use Flow\Ip;
use Flow\IpPool;
use Flow\IpStrategyInterface;

/**
* @template T
*
* @implements IpStrategyInterface<T>
*/
class FlattenIpStrategy implements IpStrategyInterface
{
/**
* @var IpPool<T>
*/
private IpPool $ipPool;

Check failure on line 26 in src/IpStrategy/FlattenIpStrategy.php

View workflow job for this annotation

GitHub Actions / Execute PHPStan analysis (8.3)

Property Flow\IpStrategy\FlattenIpStrategy::$ipPool has unknown class Flow\IpPool as its type.

Check failure on line 26 in src/IpStrategy/FlattenIpStrategy.php

View workflow job for this annotation

GitHub Actions / Execute PHPStan analysis (8.3)

Property Flow\IpStrategy\FlattenIpStrategy::$ipPool has unknown class Flow\IpPool as its type.

public function __construct()
{
$this->ipPool = new IpPool();

Check failure on line 30 in src/IpStrategy/FlattenIpStrategy.php

View workflow job for this annotation

GitHub Actions / Execute PHPStan analysis (8.3)

Instantiated class Flow\IpPool not found.
}

public static function getSubscribedEvents(): array
{
return [
Event::PUSH => 'push',
Event::PULL => 'pull',
Event::POOL => 'pool',

Check failure on line 38 in src/IpStrategy/FlattenIpStrategy.php

View workflow job for this annotation

GitHub Actions / Execute PHPStan analysis (8.3)

Access to undefined constant Flow\Event::POOL.
];
}

/**
* @param PushEvent<T> $event
*/
public function push(PushEvent $event): void
{
$ip = $event->getIp();
if (!is_iterable($ip->data)) {
throw new LogicException('Ip data must be iterable');
}
foreach ($ip->data as $data) {
$this->ipPool->addIp(new Ip($data));

Check failure on line 52 in src/IpStrategy/FlattenIpStrategy.php

View workflow job for this annotation

GitHub Actions / Execute PHPStan analysis (8.3)

Call to method addIp() on an unknown class Flow\IpPool.
}
}

/**
* @param PullEvent<T> $event
*/
public function pull(PullEvent $event): void
{
$ip = $this->ipPool->shiftIp();

Check failure on line 61 in src/IpStrategy/FlattenIpStrategy.php

View workflow job for this annotation

GitHub Actions / Execute PHPStan analysis (8.3)

Call to method shiftIp() on an unknown class Flow\IpPool.
if ($ip !== null) {
$event->addIp($ip);

Check failure on line 63 in src/IpStrategy/FlattenIpStrategy.php

View workflow job for this annotation

GitHub Actions / Execute PHPStan analysis (8.3)

Call to an undefined method Flow\Event\PullEvent<T>::addIp().
}
}

public function pool(PoolEvent $event): void

Check failure on line 67 in src/IpStrategy/FlattenIpStrategy.php

View workflow job for this annotation

GitHub Actions / Execute PHPStan analysis (8.3)

Parameter $event of method Flow\IpStrategy\FlattenIpStrategy::pool() has invalid type Flow\Event\PoolEvent.
{
$event->addIps($this->ipPool->getIps());

Check failure on line 69 in src/IpStrategy/FlattenIpStrategy.php

View workflow job for this annotation

GitHub Actions / Execute PHPStan analysis (8.3)

Call to method addIps() on an unknown class Flow\Event\PoolEvent.

Check failure on line 69 in src/IpStrategy/FlattenIpStrategy.php

View workflow job for this annotation

GitHub Actions / Execute PHPStan analysis (8.3)

Call to method getIps() on an unknown class Flow\IpPool.
}
}
62 changes: 62 additions & 0 deletions src/Job/ScrapUrlJob.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
<?php

declare(strict_types=1);

namespace App\Job\FlowExamples;

use CurlMultiHandle;
use Fiber;
use Flow\JobInterface;

/**
* @implements JobInterface<string, array>
*/
class ScrapUrlJob implements JobInterface
{
private CurlMultiHandle $mh;

public function __construct()
{
// Initialize a cURL multi handle
$this->mh = curl_multi_init();
}

public function __destruct()
{
curl_multi_close($this->mh);
}

/**
* @param string $url The URL to scrape
*
* @return array<string, mixed> Associative array with URL and its content
*/
public function __invoke($url): array
{
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_multi_add_handle($this->mh, $ch);

do {
$status = curl_multi_exec($this->mh, $active);
curl_multi_exec($this->mh, $active);

Fiber::suspend();

$info = curl_multi_info_read($this->mh);
} while (
$active && $status === CURLM_OK // check curl_multi is active
&& !($info !== false && $info['handle'] === $ch && $info['result'] === CURLE_OK) // check $ch is done
);

$content = curl_multi_getcontent($ch);
curl_multi_remove_handle($this->mh, $ch);
curl_close($ch);

return [
'url' => $url,
'content' => $content,
];
}
}
57 changes: 57 additions & 0 deletions src/Job/ScrapUrlsJob.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
<?php

declare(strict_types=1);

namespace App\Job\FlowExamples;

use Fiber;
use Flow\JobInterface;

/**
* @implements JobInterface<array<string>, array<mixed>>
*/
class ScrapUrlsJob implements JobInterface
{
public function __invoke($urls): array
{
// Initialize a cURL multi handle
$mh = curl_multi_init();

// Array to hold individual cURL handles
$curl_handles = [];

// Initialize individual cURL handles and add them to the multi handle
foreach ($urls as $url) {
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_multi_add_handle($mh, $ch);
$curl_handles[] = [$ch, $url];
}

// Execute the multi handle
$running = null;
do {
curl_multi_exec($mh, $running);

Fiber::suspend();
} while ($running > 0);

// Collect the content from each handle
$responses = [];
foreach ($curl_handles as $curl_handle) {
[$ch, $url] = $curl_handle;
$responses[] = [
'url' => $url,
'content' => curl_multi_getcontent($ch),
];
curl_multi_remove_handle($mh, $ch);
curl_close($ch);
}

// Close the multi handle
curl_multi_close($mh);

return $responses;
}
}

0 comments on commit 7c3d4bd

Please sign in to comment.