index

#!/usr/bin/env php
<?php

/**
 * Script for parsing out data from Bags for indexing in ElasticSearch.
 *
 * Run './index --help' for usage.
 */

require 'vendor/autoload.php';
require 'vendor/scholarslab/bagit/lib/bagit.php';
use GuzzleHttp\Client;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Exception\ClientException;
use GuzzleHttp\Psr7;
use \wapmorgan\UnifiedArchive\UnifiedArchive;

$cmd = new Commando\Command();
$cmd->option('i')
  ->aka('input')
  ->require(true)
  ->describedAs('Absolute or relative path to either a directory containing Bags (trailing slash is optional), or to a Bag filename.')
  ->must(function ($dir_path) {
      if (file_exists($dir_path)) {
        return true;
      } else {
        return false;
      }
});
$cmd->option('e')
  ->aka('elasticsearch_url')
  ->describedAs('URL (including port number) of your Elasticsearch endpoint. Default is "http://localhost:9200".')
  ->default('http://localhost:9200');
$cmd->option('c')
  ->aka('content_files')
  ->describedAs('Comma-separated list of plain text or XML file paths relative to the Bag data directory that are to be indexed into the "content" field, e.g., "--content MODS.xml,notes.txt".')
  ->default('');
$cmd->option('x')
  ->aka('elasticsearch_index')
  ->describedAs('Elasticsearch index. Default is "bags".')
  ->default('bags');

if (is_dir($cmd['input'])) {
  $bags = get_bags($cmd['i']);
  $bag_paths = array();
  foreach ($bags as $bag_file) {
    $path = rtrim($cmd['input'], DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $bag_file;
    $bag_paths[] = $path;
  }
}
else {
  $bag_paths = array();
  $bag_paths[] = $cmd['input'];
}

if (!index_exists($cmd['elasticsearch_url'], $cmd['elasticsearch_index'])) {
  create_index($cmd['elasticsearch_url'], $cmd['elasticsearch_index']);
}

$climate = new League\CLImate\CLImate;
$bag_num = 0;
$progress = $climate->progress()->total(count($bag_paths));

foreach ($bag_paths as $bag_path) {
  if (is_file($bag_path)) {
    $bag = new BagIt($bag_path);

    $document['document_timestamp'] = gmdate("Y-m-d\TH:i:s\Z");

    $document['bag_location_exact'] = realpath($bag_path);
    $document['bag_location'] = $document['bag_location_exact'];

    $errors = $bag->validate();
    if (count($errors) === 0) {
      $document['bag_validated']['timestamp'] = gmdate("Y-m-d\TH:i:s\Z");
      $document['bag_validated']['result'] = 'valid';
    } else {
      $document['bag_validated']['timestamp'] = gmdate("Y-m-d\TH:i:s\Z");
      $document['bag_validated']['result'] = 'invalid';
      $document['bag_validated']['errors'][] = $errors;
    }

    $bag_sha1 = sha1_file($bag_path);
    $document['bag_hash']['type'] = 'sha1';
    $document['bag_hash']['value'] = $bag_sha1;

    $document['bagit_version'] = $bag->bagVersion;
    $bag->fetch->fileName = basename($bag->fetch->fileName);
    $document['fetch'] = $bag->fetch;

    $document['serialization'] = pathinfo($bag_path, PATHINFO_EXTENSION);

    $document['content'] = get_content_data(realpath($bag_path), $cmd['content_files']);

    $bag_info = array();
    foreach ($bag->bagInfoData as $tag => &$value) {
      $trimmed_tag = trim($tag); 
      $document['bag-info'][$trimmed_tag] = trim($value);
    }

    $manifest = $bag->manifest;

    unset($manifest->pathPrefix);
    $manifest->fileName = basename($manifest->fileName);

    $data_files = array_keys($manifest->data);
    $document['data_files'] = $data_files;

    $document['manifest'] = $manifest;

    $document['tombstone'] = false;

    $bag_id = $bag_sha1;
    $client = new Client([
      'base_uri' => $cmd['elasticsearch_url'],
      ]);
    try {
      $response = $client->post($cmd['elasticsearch_index'] . '/bag/' . $bag_id, ['json' => $document]);
    }
    catch (ClientException $e) {
      print Psr7\str($e->getRequest());
      print Psr7\str($e->getResponse());
    }
  }

  $bag_num++;
  $progress->current($bag_num);
}

print "Done. " . $bag_num . " Bags added to " . $cmd['elasticsearch_url'] . '/' . $cmd['elasticsearch_index'] . PHP_EOL;

/**
 * Finds Bags to index.
 *
 * @param string $path
 *   A string containing the path to the input directory to scan.
 *
 * @return array
 *   An arrary of all the files in the input directory.
 *   in $paths.
 */
function get_bags($path) {
  $all_files = scandir($path);
  $bags = array_diff($all_files, array('.', '..'));
  return $bags;
}

/**
 * Gets content of files to populate the 'content' field.
 *
 * @param string $paths
 *   A string containing a comma-separate list of file paths relative to the
 *   Bag's 'data' directory.
 *
 * @return string
 *   The concatentated values of the text from each of the files
 *   in $paths.
 */
function get_content_data($bag_path, $paths) {
  if (!strlen($paths)) {
    return '';
  }
  $bag_name = pathinfo($bag_path, PATHINFO_FILENAME);
  if (!$archive = UnifiedArchive::open($bag_path)) {
    return '';
  }
  $file_paths = explode(',', $paths);
  $content_text = '';
  foreach ($file_paths as $path) {
    $text = '';
    $ext = pathinfo($path, PATHINFO_EXTENSION);
    $path_in_archive = $bag_name . '/data/' . $path;
    if ($file_content = $archive->getFileContent($path_in_archive)) {
      // If extension is .xml, parse out the text content of all elements.
      if ($ext == 'xml') {
        $dom = new DOMDocument();
        $dom->loadXML($file_content);
        $text = $dom->documentElement->textContent;
        $text = preg_replace('/\s+/', ' ', $text);
        $text = trim($text) . ' ';
      }
      // If extension is not .xml, read in the file content as is.
      else {
        $text = preg_replace('/\s+/', ' ', $file_content);
        $text = trim($text) . ' ';
      }
    }
    $content_text .= $text;
  }
  return trim($content_text);
}

/**
 * See if the index exists.
 *
 * @param string $base_url
 *   The Elasticsearch server's URL.
 * @param string $index
 *   The Elasticsearch index to apply the mappings to.
 *
 * @return bool
 *   True if the index exists, false if not.
 */
function index_exists($base_url, $index) {
    $client = new Client(['base_uri' => $base_url]);
    try {
      $index_exists_response = $client->request('GET', $index);
    }
    catch (ClientException $e) {
      if ($e->getResponse()->getStatusCode() == 404) {
        return false;
      }
    }

    return true;
}

/**
 * Create the index and apply mappings.
 *
 * @param string $base_url
 *   The Elasticsearch server's URL.
 * @param string $index
 *   The Elasticsearch index to apply the mappings to.
 */
function create_index($base_url, $index) {
    $client = new Client(['base_uri' => $base_url]);

    // Define the 'content' field as 'text' type.
    $mappings['mappings']['bag']['properties']['content']['type'] = 'text';
    $mappings['mappings']['bag']['properties']['tombstone']['type'] = 'boolean';

    // We want to be able to allow exact searches on bag_location_exact, so we
    // need to indicate that field is not to be analyzed. But, we also want to
    // be able to allow standard searches on bag paths, so we create another field
    // (bag_location) for those searches.
    $mappings['mappings']['bag']['properties']['bag_location_exact']['type'] = 'string';
    $mappings['mappings']['bag']['properties']['bag_location_exact']['index'] = 'not_analyzed';

    try {
      $response = $client->put($base_url . '/' . $index, ['json' => $mappings]);
    }
    catch (ClientException $e) {
      if ($e->getResponse()->getStatusCode() != 201) {
        print "Cannot create index, exiting\n";
        exit;
      }
    }
}