Merge pull request #2 from conveyal/update-batch-download

Update batch regional download example
conveyal · Apr 18, 2024 · 01bfd61 · 01bfd61
2 parents e44c340 + edcdd83
commit 01bfd61
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 100 deletions.
diff --git a/README.md b/README.md
@@ -1,3 +1,3 @@
-Examples of how to download travel time surfaces (`.tiff` format) from [Conveyal Analysis](http://conveyal.com/analysis), extract travel time contours (isochrones), and save them as `.geojson`, or create a `.csv` of accessibility results, for multiple origins
-
-Details are in the jupyter lab notebook `batch-isos.ipynb` (viewable directly on GitHub)
+Examples of batch downloads from [Conveyal](http://conveyal.com/analysis):
+- Download travel time surfaces (in the `.tif` GeoTiff format) for multiple origins, extract travel time contours (isochrones), and save them as `.geojson` or create a `.csv` of accessibility results. See the `batch-isos.ipynb` jupyter lab notebook  (viewable directly on GitHub).
+- Download a .zip file of regional analysis results including all combinations of time cutoffs, percentiles, and destination layers (in the `.tif` GeoTiff format). See the `batch-result-download.ipynb` jupyter lab notebook (viewable directly on GitHub).
diff --git a/batch-result-download.ipynb b/batch-result-download.ipynb
@@ -5,17 +5,15 @@
    "metadata": {},
    "source": [
     "# Batch Result Download\n",
-    "Download a batch of Conveyal regional analysis results\n",
+    "Download a batch of Conveyal regional analysis results. This script was streamlined in April 2024 with the v7.2 Conveyal / R5 release.\n",
     "\n",
     "Setup requires:\n",
     " - Completed regional analysis results, viewable in the Conveyal user interface\n",
-    " - JSON with array values for the keys `analysisIds`, `cutoffs`, `pointSetId`, and `percentile`, plus the `regionId` saved to `config/regionalAnalysisParameters.json`. The values in the arrays can be derived from the URL of a regional analysis results page in the Conveyal user interface: `https://analysis.conveyal.com/regions/[regionId]/regional/[analysisId]?cutoff=[cutoff]&pointSetId=[pointSetId]&percentile=[percentile]`\n",
+    " - JSON with array values for the key `analysisIds` saved to `config/regionalAnalysisParameters.json`. The values in the array can be derived from the URL of regional analysis results pages in the Conveyal user interface: `https://analysis.conveyal.com/regions/[regionId]/regional/[analysisId]`\n",
     " - A current Conveyal token (e.g. 'bearer 1234abcd...') saved at `config/.auth`, based on the `idToken` shown at https://analysis.conveyal.com/session (for logged in users).\n",
-    " - Optionally, a session cookie copied from browser DevTools saved at `config/.cookie`\n",
     "\n",
-    "After the setup cell in this notebook, there are cells to: \n",
-    " - Optionally, fetch the names of regional analyses and destination pointset layers\n",
-    " - Download all combinations of analyses, cutoffs, destination pointsets, and percentiles"
+    "After the setup cell in this notebook, the remaining cell: \n",
+    " - Downloads a .zip file containing geotiff files for all combinations of analyses, cutoffs, destination pointsets, and percentiles"
    ]
   },
   {
@@ -27,60 +25,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "import requests\n",
     "import json\n",
     "import urllib\n",
     "import os\n",
+    "import time\n",
     "\n",
     "config = json.load(open('config/regionalAnalysisParameters.json'))\n",
     "\n",
-    "# Authorization header copied from DevTools Network request or https://analysis.conveyal.com/session\n",
+    "# Authorization header with idToken copied from https://analysis.conveyal.com/session\n",
     "token = open('config/.auth').readline().strip()\n",
     "headers = {\n",
     "    'Authorization': token\n",
     "}\n",
-    "resultUrl = 'https://api.conveyal.com/api/regional/'\n",
-    "\n",
-    "namesFetched = False;\n",
-    "\n",
-    "# Cookie copied from browser Devtools Network request header\n",
-    "cookie = open('config/.cookie').readline().strip()\n",
-    "dbHeaders = {\n",
-    "    'Cookie': cookie\n",
-    "}\n",
-    "dbUrl = 'https://analysis.conveyal.com/api/db/'"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Fetch names\n",
-    "Optional, requires saved cookie"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "analysesRequest = requests.get(dbUrl + 'regional-analyses?options=' + urllib.parse.urlencode({\"projection\":{\"request.scenario.modifications\":0}}), headers = dbHeaders)\n",
-    "analyses = analysesRequest.json()\n",
-    "\n",
-    "opportunitiesRequest = requests.get(dbUrl + 'opportunityDatasets?', headers = dbHeaders)\n",
-    "opportunities = opportunitiesRequest.json()\n",
-    "destinations = {}\n",
-    "\n",
-    "for destinationPointSetId in config['pointSetIds']:\n",
-    "    destinationPointSetName = list(filter(lambda x:x[\"_id\"] == destinationPointSetId, opportunities))[0]['name']\n",
-    "    destinations[destinationPointSetId] = destinationPointSetName\n",
-    "\n",
-    "namesFetched = True;"
+    "baseUrl = 'https://api.conveyal.com/api/regional/'"
    ]
   },
   {
@@ -100,43 +62,36 @@
     "# Loop over analysis ids\n",
     "for analysisId in config['analysisIds']:\n",
     "    \n",
-    "    analysisName = analysisId\n",
-    "    if namesFetched:\n",
-    "        analysisProperties = list(filter(lambda x:x['_id']==analysisId, analyses))[0]\n",
-    "        with open('results/' + analysisId + '.json', 'w') as f:\n",
-    "            json.dump(analysisProperties, f)\n",
-    "        analysisName = analysisProperties['name']\n",
-    "    print('Processing ' + analysisName)\n",
-    "    \n",
-    "    # Loop over cutoffs, percentiles, and destination opportunity pointsets\n",
-    "    for cutoff in config['cutoffs']:\n",
-    "        for percentile in config['percentiles']:\n",
-    "            for destinationPointSetId in config['pointSetIds']:\n",
-    "                \n",
-    "                destinationPointSetName = destinationPointSetId\n",
-    "                if namesFetched:\n",
-    "                    destinationPointSetName = destinations[destinationPointSetId]\n",
-    "                \n",
-    "                gridQuery = urllib.parse.urlencode({'cutoff': cutoff, 'percentile': percentile, 'destinationPointSetId': destinationPointSetId})\n",
-    "                gridUrl = resultUrl + analysisId + '/grid/geotiff?' + gridQuery\n",
-    "                \n",
-    "                # Request a signed S3 url from the Conveyal API\n",
-    "                r = requests.get(gridUrl, headers = headers, verify = False)\n",
+    "    print('Processing ' + analysisId)\n",
+    "    status = 202;\n",
     "    \n",
-    "                if r.status_code == 403:\n",
-    "                    print('Unauthorized access. Your authorization token may be invalid or expired.')\n",
+    "    while (status == 202):\n",
+    "        resultUrl = baseUrl + analysisId + '/all'\n",
+    "        # Request a signed S3 url from the Conveyal API\n",
+    "        r = requests.get(resultUrl, headers = headers, verify = False)\n",
+    "        if r.status_code == 403:\n",
+    "            print('Unauthorized access. Your authorization token may be invalid or expired.')\n",
+    "            \n",
+    "        elif r.status_code == 404:\n",
+    "            print('Results not found. Check your analysisId values')\n",
     "        \n",
-    "                elif r.status_code != 200:\n",
-    "                    print('Error: ' + r.text)\n",
+    "        elif r.status_code == 202:\n",
+    "            print(r.text + ' Retrying in 15 seconds.')\n",
+    "            time.sleep(15)\n",
     "        \n",
-    "                else:\n",
-    "                    # From the signed S3 url, fetch the grid as a geotiff\n",
-    "                    grid = requests.get(r.json()['url'], verify = False)\n",
-    "                    # Save response from Conveyal Analysis to a local .geotiff file\n",
-    "                    with open('results/'  + ('-').join((analysisId, destinationPointSetName, str(cutoff) + 'min', str(percentile) + 'pct')) + '.geotiff', 'wb') as f:\n",
-    "                        for chunk in grid.iter_content(chunk_size=128):\n",
-    "                            f.write(chunk)\n",
-    "                        f.close()"
+    "        elif (r.status_code != 200):\n",
+    "            print('Error: ' + r.text)\n",
+    "        \n",
+    "        status = r.status_code\n",
+    "            \n",
+    "    if (status == 200):\n",
+    "        zipRequest = requests.get(r.json()['url'], verify = False)\n",
+    "\n",
+    "        # Save result to a local .zip file\n",
+    "        with open('results/'  + analysisId + '.zip', 'wb') as f:\n",
+    "            for chunk in zipRequest.iter_content(chunk_size=128):\n",
+    "                f.write(chunk)\n",
+    "            f.close()"
    ]
   }
  ],

diff --git a/config/regionalAnalysisParameters.json b/config/regionalAnalysisParameters.json
@@ -1,21 +1,6 @@
 {
-    "regionId": "abc123",
     "analysisIds": [
-        "def456",
-        "ghi789"
-    ],
-    "cutoffs": [
-        30,
-        45,
-        60
-    ],
-    "pointSetIds": [
-        "uvw123",
-        "xyz456"
-    ],
-    "percentiles": [
-        25,
-        50,
-        75
+        "abc123",
+        "def456"
     ]
 }