crawler.php

<?php

require_once('config.php');
require_once('parse.php');

require_once __DIR__.'/vendor/autoload.php';

use Goutte\Client;
use Doctrine\ORM\Tools\Setup;
use Doctrine\ORM\EntityManager;

$config = Setup::createAnnotationMetadataConfiguration(array(__DIR__), true);
$entityManager = EntityManager::create($dbParams, $config);

$cve_list_base_url = 'https://www.drupal.org/security/contrib?page=';

// Find the total number of pages to crawl.
$client = new Client();
$crawler = $client->request('GET', $cve_list_base_url . '0');
$body = $crawler->html();
if (preg_match('~href="/security/contrib\?page=(\d+)">last~', $body, $matches)) {
  $total_pages = $matches[1];
} else {
  die("Cannot determine total pages to crawl.");
}

// Crawl the page listings and parse the CVE's.
for ($page = 0; $page <= $total_pages; $page++) { 
  $client = new Client();
  $crawler = $client->request('GET', $cve_list_base_url . $page);
  $body = $crawler->html();
  $cve_urls = parse_cve_list($body);
  foreach ($cve_urls as $url) {
    echo "Processing: {$url}\r\n";
    $crawler = $client->request('GET', $url);
    $body = $crawler->html();
    $u = parse_cve($body);
    if (!empty($u)) {
      $u['advisory_url'] = $url; // Keep track of the advisory_url
        $conn = $entityManager->getConnection();
      try {
        $conn->insert('cve', $u);
      } catch (Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) {
        $conn->update('cve', $u, array('advisory_id' => $u['advisory_id']));
      }
    }
  }
}