fetch_prepare_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ppi.image_metadata_crawler import LibraryOfCongressCrawler\n",
    "from ppi.medium_mapper import MediumMapper\n",
    "from ppi.database import Database\n",
    "from ppi.image_downloader   import ImageDownloader\n",
    "import yaml\n",
    "\n",
    "with open(\"config.yaml\", \"r\") as yamlfile:\n",
    "    config = yaml.load(yamlfile, Loader=yaml.FullLoader)\n",
    "\n",
    "database = Database(db_name=config[\"db_name\"])\n",
    "\n",
    "LIBRARY_OF_CONGRESS_URL_PREFIX = \"https://www.loc.gov/pictures/search/?va=exact&q=Cyanotypes.&fa=displayed%3Aanywhere&fi=format&sg=true&op=EQUAL&sp=\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download images metadata\n",
    "\n",
    "You use the crawler with a URL for retrieving metadata of all images contained in a page. Metadata is stored in a SQLite DB.\n",
    "\n",
    "The following Crawler is used in this example:\n",
    "- [Libary of Congress](https://www.loc.gov/)\n",
    "\n",
    "Note: the following Crawlers are deprecated (due to underlying page format changes):\n",
    "- [Getty Search Gateway](https://search.getty.edu/gateway/landing)\n",
    "- [Cornell University Digital Library](https://digital.library.cornell.edu/)\n",
    "- [Eastman Museum](https://collections.eastman.org/collections)\n",
    "\n",
    "You need to implement them yourself if needed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2023-10-05 23:05:20.411\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m90\u001b[0m - \u001b[1mProcessing page https://www.loc.gov/pictures/search/?va=exact&q=Cyanotypes.&fa=displayed%3Aanywhere&fi=format&sg=true&op=EQUAL&sp=1\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:22.881\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007662712/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:24.216\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664012/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:25.527\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664026/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:26.841\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664018/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:28.182\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664033/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:29.500\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664022/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:30.880\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664028/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:32.200\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007662711/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:33.544\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664031/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:34.873\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664023/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:36.221\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664020/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:38.012\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664016/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:39.338\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664021/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:40.672\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664011/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:42.023\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664006/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:43.356\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664017/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:44.688\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664019/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:46.025\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007662272/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:47.365\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664007/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:48.726\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664032/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:48.727\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m122\u001b[0m - \u001b[1m20 entries added to DB\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:48.727\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m123\u001b[0m - \u001b[1mFailed to insert data for 0 images\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:48.730\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m90\u001b[0m - \u001b[1mProcessing page https://www.loc.gov/pictures/search/?va=exact&q=Cyanotypes.&fa=displayed%3Aanywhere&fi=format&sg=true&op=EQUAL&sp=2\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:50.345\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664059/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:53.185\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664041/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:54.492\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664060/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:55.825\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664044/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:57.156\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664057/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:58.502\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664062/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:05:59.830\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664055/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:01.231\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664037/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:02.622\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664054/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:03.998\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664051/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:05.983\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664042/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:07.375\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664036/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:08.780\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664039/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:10.166\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664035/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:11.554\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664061/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:12.919\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664038/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:14.297\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664056/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:15.667\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664048/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:17.022\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664045/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:18.385\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664043/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:18.386\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m122\u001b[0m - \u001b[1m20 entries added to DB\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:18.387\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m123\u001b[0m - \u001b[1mFailed to insert data for 0 images\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:18.390\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m90\u001b[0m - \u001b[1mProcessing page https://www.loc.gov/pictures/search/?va=exact&q=Cyanotypes.&fa=displayed%3Aanywhere&fi=format&sg=true&op=EQUAL&sp=3\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:20.488\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664066/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:21.882\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664030/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:23.232\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007662258/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:24.600\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664029/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:26.309\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664024/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:28.249\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664046/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:30.483\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007663550/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:31.816\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664010/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:33.554\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2002711153/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:34.872\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664050/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:36.230\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664064/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:37.525\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664027/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:38.853\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664040/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:40.213\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664049/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:41.587\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664014/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:42.906\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664025/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:44.226\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664034/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:45.600\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664013/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:46.998\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664052/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:48.394\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mAdded image metadata: https://www.loc.gov/pictures/item/2007664065/\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:48.395\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m122\u001b[0m - \u001b[1m20 entries added to DB\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:48.395\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_metadata_crawler\u001b[0m:\u001b[36msave_pages_img_url_metadata\u001b[0m:\u001b[36m123\u001b[0m - \u001b[1mFailed to insert data for 0 images\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "congress_crawler = LibraryOfCongressCrawler(config=config, database=database)\n",
    "congress_crawler.save_pages_img_url_metadata(\n",
    "    prefix_url_search=LIBRARY_OF_CONGRESS_URL_PREFIX, first_page=1, last_page=3\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Standardize Photographic Processes descriptions\n",
    "\n",
    "There is code in ``medium_maper.py`` to map the source descriptions to the predefined descriptions in ``config.yaml`` ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2023-10-05 23:06:48.415\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.medium_mapper\u001b[0m:\u001b[36mshow_stats\u001b[0m:\u001b[36m128\u001b[0m - \u001b[1mCurrent stats: \n",
      "   new_medium  count\n",
      "0  CYANOTYPE     60\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "mapper = MediumMapper(config=config, database=database)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run the cell below. If results are ok, move to next cell. Otherwise, adjust ```propose_mapping``` method in ```medium_mapper.py```."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2023-10-05 23:06:48.419\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.medium_mapper\u001b[0m:\u001b[36mshow_undefined_mappings\u001b[0m:\u001b[36m150\u001b[0m - \u001b[1mNo undefined medium descriptions.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "mapper.show_undefined_mappings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper.update_mediums()  # Updates mediums in DB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download images to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2023-10-05 23:06:48.450\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m46\u001b[0m - \u001b[1mDownloading images for medium ALBUMEN_PRINT\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:48.452\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m72\u001b[0m - \u001b[1mNo more images to download\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:48.452\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m46\u001b[0m - \u001b[1mDownloading images for medium AMBROTYPE_TINTYPE_FERROTYPE\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:48.453\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m72\u001b[0m - \u001b[1mNo more images to download\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:48.454\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m46\u001b[0m - \u001b[1mDownloading images for medium CARBON_PRINT\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:48.455\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m72\u001b[0m - \u001b[1mNo more images to download\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:48.455\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m46\u001b[0m - \u001b[1mDownloading images for medium CYANOTYPE\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:48.456\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m55\u001b[0m - \u001b[1mDownloading: https://cdn.loc.gov/master/pnp/pan/6a08000/6a08700/6a08762u.tif\u001b[0m\n",
      "\u001b[32m2023-10-05 23:06:57.299\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m55\u001b[0m - \u001b[1mDownloading: https://cdn.loc.gov/master/pnp/pan/6a36000/6a36100/6a36134u.tif\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:01.820\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m55\u001b[0m - \u001b[1mDownloading: https://cdn.loc.gov/master/pnp/pan/6a36000/6a36000/6a36021u.tif\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:02.521\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m55\u001b[0m - \u001b[1mDownloading: https://cdn.loc.gov/master/pnp/pan/6a35000/6a35900/6a35974u.tif\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:02.920\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m55\u001b[0m - \u001b[1mDownloading: https://cdn.loc.gov/master/pnp/pan/6a36000/6a36000/6a36059u.tif\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:03.330\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m55\u001b[0m - \u001b[1mDownloading: https://cdn.loc.gov/master/pnp/pan/6a35000/6a35900/6a35998u.tif\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:03.705\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m55\u001b[0m - \u001b[1mDownloading: https://cdn.loc.gov/master/pnp/pan/6a36000/6a36000/6a36031u.tif\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:04.095\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m55\u001b[0m - \u001b[1mDownloading: https://cdn.loc.gov/master/pnp/pan/6a08000/6a08700/6a08753u.tif\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:04.526\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m55\u001b[0m - \u001b[1mDownloading: https://cdn.loc.gov/master/pnp/pan/6a36000/6a36000/6a36047u.tif\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:04.931\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m55\u001b[0m - \u001b[1mDownloading: https://cdn.loc.gov/master/pnp/pan/6a36000/6a36000/6a36004u.tif\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.055\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m55\u001b[0m - \u001b[1mDownloading: https://cdn.loc.gov/master/pnp/pan/6a35000/6a35900/6a35986u.tif\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.188\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m72\u001b[0m - \u001b[1mMaximum number of images downloaded\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.189\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m46\u001b[0m - \u001b[1mDownloading images for medium DAGUERREOTYPE\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.190\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m72\u001b[0m - \u001b[1mNo more images to download\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.190\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m46\u001b[0m - \u001b[1mDownloading images for medium DOP\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.191\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m72\u001b[0m - \u001b[1mNo more images to download\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.192\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m46\u001b[0m - \u001b[1mDownloading images for medium POP\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.193\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m72\u001b[0m - \u001b[1mNo more images to download\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.193\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m46\u001b[0m - \u001b[1mDownloading images for medium PLATINOTYPE_PALLADIOTYPE\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.194\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m72\u001b[0m - \u001b[1mNo more images to download\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.194\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m46\u001b[0m - \u001b[1mDownloading images for medium SALTED_PAPER_PRINT\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.196\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m72\u001b[0m - \u001b[1mNo more images to download\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.196\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m77\u001b[0m - \u001b[1mBacking up images\u001b[0m\n",
      "\u001b[32m2023-10-05 23:07:05.265\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mppi.image_downloader\u001b[0m:\u001b[36mdownload_images\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mBack up completed\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "image_download = ImageDownloader(config=config, database=database)\n",
    "image_download.download_images(max_number_downloads=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare images for Deep Learning\n",
    "\n",
    "Images must be MANUALLY cropped, as exemplified below (we only want to keep \"relevant\" information):\n",
    "\n",
    "<img src = \"ppi/images/GettyCrawler_49753.jpg\" width=\"180\" height=\"180\">\n",
    "<img src = \"ppi/images/GettyCrawler_49753_crop.jpg\" width=\"180\" height=\"180\">\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "443555451089c934f8af11b20a891a7298bfb9795a8b0a748ec7e65dc6a0f51c"
  },
  "kernelspec": {
   "display_name": "Python 3.9.1 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}