diff --git a/README.md b/README.md index 7b0f8b6..5189838 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ -Examples of how to download travel time surfaces (`.tiff` format) from [Conveyal Analysis](http://conveyal.com/analysis), extract travel time contours (isochrones), and save them as `.geojson`, or create a `.csv` of accessibility results, for multiple origins - -Details are in the jupyter lab notebook `batch-isos.ipynb` (viewable directly on GitHub) +Examples of batch downloads from [Conveyal](http://conveyal.com/analysis): +- Download travel time surfaces (in the `.tif` GeoTiff format) for multiple origins, extract travel time contours (isochrones), and save them as `.geojson` or create a `.csv` of accessibility results. See the `batch-isos.ipynb` jupyter lab notebook (viewable directly on GitHub). +- Download a .zip file of regional analysis results including all combinations of time cutoffs, percentiles, and destination layers (in the `.tif` GeoTiff format). See the `batch-result-download.ipynb` jupyter lab notebook (viewable directly on GitHub). diff --git a/batch-result-download.ipynb b/batch-result-download.ipynb index 632487c..25e2cdd 100644 --- a/batch-result-download.ipynb +++ b/batch-result-download.ipynb @@ -5,17 +5,15 @@ "metadata": {}, "source": [ "# Batch Result Download\n", - "Download a batch of Conveyal regional analysis results\n", + "Download a batch of Conveyal regional analysis results. This script was streamlined in April 2024 with the v7.2 Conveyal / R5 release.\n", "\n", "Setup requires:\n", " - Completed regional analysis results, viewable in the Conveyal user interface\n", - " - JSON with array values for the keys `analysisIds`, `cutoffs`, `pointSetId`, and `percentile`, plus the `regionId` saved to `config/regionalAnalysisParameters.json`. The values in the arrays can be derived from the URL of a regional analysis results page in the Conveyal user interface: `https://analysis.conveyal.com/regions/[regionId]/regional/[analysisId]?cutoff=[cutoff]&pointSetId=[pointSetId]&percentile=[percentile]`\n", + " - JSON with array values for the key `analysisIds` saved to `config/regionalAnalysisParameters.json`. The values in the array can be derived from the URL of regional analysis results pages in the Conveyal user interface: `https://analysis.conveyal.com/regions/[regionId]/regional/[analysisId]`\n", " - A current Conveyal token (e.g. 'bearer 1234abcd...') saved at `config/.auth`, based on the `idToken` shown at https://analysis.conveyal.com/session (for logged in users).\n", - " - Optionally, a session cookie copied from browser DevTools saved at `config/.cookie`\n", "\n", - "After the setup cell in this notebook, there are cells to: \n", - " - Optionally, fetch the names of regional analyses and destination pointset layers\n", - " - Download all combinations of analyses, cutoffs, destination pointsets, and percentiles" + "After the setup cell in this notebook, the remaining cell: \n", + " - Downloads a .zip file containing geotiff files for all combinations of analyses, cutoffs, destination pointsets, and percentiles" ] }, { @@ -27,7 +25,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -35,52 +33,16 @@ "import json\n", "import urllib\n", "import os\n", + "import time\n", "\n", "config = json.load(open('config/regionalAnalysisParameters.json'))\n", "\n", - "# Authorization header copied from DevTools Network request or https://analysis.conveyal.com/session\n", + "# Authorization header with idToken copied from https://analysis.conveyal.com/session\n", "token = open('config/.auth').readline().strip()\n", "headers = {\n", " 'Authorization': token\n", "}\n", - "resultUrl = 'https://api.conveyal.com/api/regional/'\n", - "\n", - "namesFetched = False;\n", - "\n", - "# Cookie copied from browser Devtools Network request header\n", - "cookie = open('config/.cookie').readline().strip()\n", - "dbHeaders = {\n", - " 'Cookie': cookie\n", - "}\n", - "dbUrl = 'https://analysis.conveyal.com/api/db/'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Fetch names\n", - "Optional, requires saved cookie" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "analysesRequest = requests.get(dbUrl + 'regional-analyses?options=' + urllib.parse.urlencode({\"projection\":{\"request.scenario.modifications\":0}}), headers = dbHeaders)\n", - "analyses = analysesRequest.json()\n", - "\n", - "opportunitiesRequest = requests.get(dbUrl + 'opportunityDatasets?', headers = dbHeaders)\n", - "opportunities = opportunitiesRequest.json()\n", - "destinations = {}\n", - "\n", - "for destinationPointSetId in config['pointSetIds']:\n", - " destinationPointSetName = list(filter(lambda x:x[\"_id\"] == destinationPointSetId, opportunities))[0]['name']\n", - " destinations[destinationPointSetId] = destinationPointSetName\n", - "\n", - "namesFetched = True;" + "baseUrl = 'https://api.conveyal.com/api/regional/'" ] }, { @@ -100,43 +62,36 @@ "# Loop over analysis ids\n", "for analysisId in config['analysisIds']:\n", " \n", - " analysisName = analysisId\n", - " if namesFetched:\n", - " analysisProperties = list(filter(lambda x:x['_id']==analysisId, analyses))[0]\n", - " with open('results/' + analysisId + '.json', 'w') as f:\n", - " json.dump(analysisProperties, f)\n", - " analysisName = analysisProperties['name']\n", - " print('Processing ' + analysisName)\n", - " \n", - " # Loop over cutoffs, percentiles, and destination opportunity pointsets\n", - " for cutoff in config['cutoffs']:\n", - " for percentile in config['percentiles']:\n", - " for destinationPointSetId in config['pointSetIds']:\n", - " \n", - " destinationPointSetName = destinationPointSetId\n", - " if namesFetched:\n", - " destinationPointSetName = destinations[destinationPointSetId]\n", - " \n", - " gridQuery = urllib.parse.urlencode({'cutoff': cutoff, 'percentile': percentile, 'destinationPointSetId': destinationPointSetId})\n", - " gridUrl = resultUrl + analysisId + '/grid/geotiff?' + gridQuery\n", - " \n", - " # Request a signed S3 url from the Conveyal API\n", - " r = requests.get(gridUrl, headers = headers, verify = False)\n", + " print('Processing ' + analysisId)\n", + " status = 202;\n", " \n", - " if r.status_code == 403:\n", - " print('Unauthorized access. Your authorization token may be invalid or expired.')\n", + " while (status == 202):\n", + " resultUrl = baseUrl + analysisId + '/all'\n", + " # Request a signed S3 url from the Conveyal API\n", + " r = requests.get(resultUrl, headers = headers, verify = False)\n", + " if r.status_code == 403:\n", + " print('Unauthorized access. Your authorization token may be invalid or expired.')\n", + " \n", + " elif r.status_code == 404:\n", + " print('Results not found. Check your analysisId values')\n", " \n", - " elif r.status_code != 200:\n", - " print('Error: ' + r.text)\n", + " elif r.status_code == 202:\n", + " print(r.text + ' Retrying in 15 seconds.')\n", + " time.sleep(15)\n", " \n", - " else:\n", - " # From the signed S3 url, fetch the grid as a geotiff\n", - " grid = requests.get(r.json()['url'], verify = False)\n", - " # Save response from Conveyal Analysis to a local .geotiff file\n", - " with open('results/' + ('-').join((analysisId, destinationPointSetName, str(cutoff) + 'min', str(percentile) + 'pct')) + '.geotiff', 'wb') as f:\n", - " for chunk in grid.iter_content(chunk_size=128):\n", - " f.write(chunk)\n", - " f.close()" + " elif (r.status_code != 200):\n", + " print('Error: ' + r.text)\n", + " \n", + " status = r.status_code\n", + " \n", + " if (status == 200):\n", + " zipRequest = requests.get(r.json()['url'], verify = False)\n", + "\n", + " # Save result to a local .zip file\n", + " with open('results/' + analysisId + '.zip', 'wb') as f:\n", + " for chunk in zipRequest.iter_content(chunk_size=128):\n", + " f.write(chunk)\n", + " f.close()" ] } ], diff --git a/config/regionalAnalysisParameters.json b/config/regionalAnalysisParameters.json index 8428395..90f5d55 100644 --- a/config/regionalAnalysisParameters.json +++ b/config/regionalAnalysisParameters.json @@ -1,21 +1,6 @@ { - "regionId": "abc123", "analysisIds": [ - "def456", - "ghi789" - ], - "cutoffs": [ - 30, - 45, - 60 - ], - "pointSetIds": [ - "uvw123", - "xyz456" - ], - "percentiles": [ - 25, - 50, - 75 + "abc123", + "def456" ] }