From 329f3601efd6ec92b1102407defad86ef818c528 Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Thu, 29 Oct 2020 00:03:53 +0100 Subject: [PATCH 01/21] Intial release. Note, to test it, you need to manually create the index and source files for srtm, as the srtm script fails. You also need to install PDAL with either LASzip or LAZperf --- config.download.slovenia.yaml | 36 ++++ config.download.slovenia_test.yaml | 36 ++++ config.render.slovenia.yaml | 39 +++++ config.render.slovenia_test.yaml | 39 +++++ joerd/source/d96tm.py | 260 +++++++++++++++++++++++++++++ joerd/srs.py | 14 ++ 6 files changed, 424 insertions(+) create mode 100644 config.download.slovenia.yaml create mode 100644 config.download.slovenia_test.yaml create mode 100644 config.render.slovenia.yaml create mode 100644 config.render.slovenia_test.yaml create mode 100644 joerd/source/d96tm.py diff --git a/config.download.slovenia.yaml b/config.download.slovenia.yaml new file mode 100644 index 0000000..1eb2e1d --- /dev/null +++ b/config.download.slovenia.yaml @@ -0,0 +1,36 @@ +--- +regions: + # san-francisco-bay_california: + # bbox: + # top: 38.719 + # left: -123.640 + # bottom: 36.791 + # right: -121.025 + # zoom_range: [0, 16] + slovenia: + bbox: + top: 46.88333333 + left: 13.39027778 + bottom: 45.40750000 + right: 16.62694444 + zoom_range: [0, 15] +outputs: + - type: skadi + - type: tiff + - type: terrarium + - type: normal +sources: + - type: d96tm + uri: http://gis.arso.gov.si/lidar/otr/laz + fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip +logging: + config: logging.example.config +cluster: + queue: + type: fake +store: + type: file + base_dir: 'render' +source_store: + type: file + base_dir: 'source' \ No newline at end of file diff --git a/config.download.slovenia_test.yaml b/config.download.slovenia_test.yaml new file mode 100644 index 0000000..55744e1 --- /dev/null +++ b/config.download.slovenia_test.yaml @@ -0,0 +1,36 @@ +--- +regions: + # san-francisco-bay_california: + # bbox: + # top: 38.719 + # left: -123.640 + # bottom: 36.791 + # right: -121.025 + # zoom_range: [0, 16] + slovenia-test: + bbox: + top: 46.43305556 + left: 15.67583333 + bottom: 46.38861111 + right: 15.74166667 + zoom_range: [14, 15] +outputs: + - type: skadi + - type: tiff + - type: terrarium + - type: normal +sources: + - type: 'd96tm' + uri: 'http://gis.arso.gov.si/lidar/otr/laz' + fishnet_url: 'https://github.com/DavixDevelop/TerraLidar/raw/master/mapzen_data/lidar_fishnet_D96TM.zip' +logging: + config: logging.example.config +cluster: + queue: + type: fake +store: + type: file + base_dir: 'render' +source_store: + type: file + base_dir: 'source' \ No newline at end of file diff --git a/config.render.slovenia.yaml b/config.render.slovenia.yaml new file mode 100644 index 0000000..6cfe559 --- /dev/null +++ b/config.render.slovenia.yaml @@ -0,0 +1,39 @@ +--- +regions: + # san-francisco-bay_california: + # bbox: + # top: 38.719 + # left: -123.640 + # bottom: 36.791 + # right: -121.025 + # zoom_range: [0, 16] + slovenia: + bbox: + top: 46.88333333 + left: 13.39027778 + bottom: 45.40750000 + right: 16.62694444 + zoom_range: [0, 15] +outputs: + - type: skadi + - type: tiff + - type: terrarium + - type: normal +sources: + - type: srtm + url: https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 + mask-url: https://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 + - type: d96tm + uri: http://gis.arso.gov.si/lidar/otr/laz + fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip +logging: + config: logging.example.config +cluster: + queue: + type: fake +store: + type: file + base_dir: 'render' +source_store: + type: file + base_dir: 'source' \ No newline at end of file diff --git a/config.render.slovenia_test.yaml b/config.render.slovenia_test.yaml new file mode 100644 index 0000000..3dfe089 --- /dev/null +++ b/config.render.slovenia_test.yaml @@ -0,0 +1,39 @@ +--- +regions: + # san-francisco-bay_california: + # bbox: + # top: 38.719 + # left: -123.640 + # bottom: 36.791 + # right: -121.025 + # zoom_range: [0, 16] + slovenia-test: + bbox: + top: 46.43305556 + left: 15.67583333 + bottom: 46.38861111 + right: 15.74166667 + zoom_range: [14, 15] +outputs: + - type: skadi + - type: tiff + - type: terrarium + - type: normal +sources: + - type: srtm + url: https://github.com/DavixDevelop/TerraLidar/raw/master/mapzen_data/tile + mask-url: https://github.com/DavixDevelop/TerraLidar/raw/master/mapzen_data/mask + - type: 'd96tm' + uri: 'http://gis.arso.gov.si/lidar/otr/laz' + fishnet_url: 'https://github.com/DavixDevelop/TerraLidar/raw/master/mapzen_data/lidar_fishnet_D96TM.zip' +logging: + config: logging.example.config +cluster: + queue: + type: fake +store: + type: file + base_dir: 'render' +source_store: + type: file + base_dir: 'source' \ No newline at end of file diff --git a/joerd/source/d96tm.py b/joerd/source/d96tm.py new file mode 100644 index 0000000..4b5c049 --- /dev/null +++ b/joerd/source/d96tm.py @@ -0,0 +1,260 @@ +from bs4 import BeautifulSoup +from joerd.util import BoundingBox +import joerd.download as download +import joerd.check as check +import joerd.srs as srs +import joerd.index as index +import joerd.mask as mask +import joerd.tmpdir as tmpdir +from joerd.mkdir_p import mkdir_p +from contextlib2 import closing, ExitStack +from shutil import copyfile, move +import os.path +import os +import io +import requests +import logging +import re +import tempfile +import sys +import zipfile +import traceback +import subprocess +import glob +from osgeo import gdal +from osgeo import ogr +from osgeo import osr +import pdal +import yaml +import time +import json +import subprocess + +class D96TMTile(object): + def __init__(self, parent, link, name, block, bbox): + self.uri = parent.uri + self.download_options = parent.download_options + self.base_dir = parent.base_dir + self.name = name + self.block = block + self.link = link + self.bbox = bbox + + def __key(self): + return self.name + + def __eq__(a, b): + return isinstance(b, type(a)) and \ + a.__key() == b.__key() + + def __hash__(self): + return hash(self.__key()) + + def urls(self): + uri_list = [self.uri + "/" + self.link] + return uri_list + + + def verifier(self): + return is_las + + def options(self): + return self.download_options + + def output_file(self): + file_name = "TMR_" + self.name + ".tif" + return os.path.join(self.base_dir, file_name) + + def _tif_file(self): + # returns the name of the geotiff file within the distributed archive. + return "TMR_%(name)s.tif" % dict(name=self.name) + + def unpack(self, store, las_file): + tif_file = self._tif_file; + + with store.upload_dir() as target: + target_dir = os.path.join(target, self.base_dir) + mkdir_p(target_dir) + with tmpdir.tmpdir() as temp_dir: + + arr = [] + inp = { + "type": "readers.las", + "filename": "%(fname)s" % dict(fname=las_file.name), + "spatialreference":"EPSG:3794" + } + arr.append(inp) + par = { + "type": "writers.gdal", + "resolution": 1, + "radius": 7, + "filename": os.path.join(temp_dir, "TMR_%(name)s.tif" % dict(name=self.name)) + } + arr.append(par) + + astage = { + "pipeline" : arr + } + + + #j = json.dumps(astage) + #u = unicode(j, "utf-8") + + json_path = os.path.join(temp_dir, "TMR_%(name)s.json" % dict(name=self.name)) + + with open(json_path, 'w') as json_file: + json.dump(astage, json_file) + + #covert lidar to geotiff + #pipeline = pdal.Pipeline(u) + #count = pipeline.execute() + + subprocess.check_output('pdal pipeline {jfile}'.format(jfile=json_path), cwd=temp_dir, shell=True) + + output_file = os.path.join(target, self.output_file()) + mask.negative(os.path.join(temp_dir, "TMR_%(name)s.tif" % dict(name=self.name)), "GTiff", output_file) + + + def freeze_dry(self): + return dict(type='d96tm', link=self.link) + +IS_D96TM_FILE = re.compile( + '^b_([0-9]{2})/D96TM/TMR_([0-9]{2,3})_([0-9]{2,3}).laz') + +def is_las(self): + def func(tmp): + if tmp.name.endswith(".laz"): + return True + else: + return False + return func + + +def _parse_d96tm_tile(link, parent): + m = IS_D96TM_FILE.match(link) + + if not m: + return None + + d96_bottom = int(m.group(3)) * 1000 + d96_left = int(m.group(2)) * 1000 + d96_top = (int(m.group(3)) + 1) * 1000 + d96_right = (int(m.group(2)) + 1) * 1000 + + block = int(m.group(1)) + name = m.group(2) + "_" + m.group(3) + + #D96 (EPSG::3794) to WGS84 (EPSG::4326) + src = osr.SpatialReference() + tgt = osr.SpatialReference() + src.ImportFromEPSG(3794) + tgt.ImportFromEPSG(4326) + + transform = osr.CoordinateTransformation(src, tgt) + coords = transform.TransformPoint(d96_left, d96_bottom) + left,bottom = coords[0:2] + coords = transform.TransformPoint(d96_right, d96_top) + right,top= coords[0:2] + + bbox = BoundingBox(left, bottom, right, top) + + return D96TMTile(parent, link, name, block, bbox) + +class D96TM(object): + def __init__(self, options={}): + self.base_dir = options.get('base_dir', 'd96tm') + self.uri = options['uri'] + self.fishnet_url = options['fishnet_url'] + self.download_options = options + self.tile_index = None + + def get_index(self): + if not os.path.isdir(self.base_dir): + os.makedirs(self.base_dir) + index_name = 'index.yaml' + index_file = os.path.join(self.base_dir, index_name) + # if index doesn't exist, or is more than 24h old + if not os.path.isfile(index_file) or \ + time.time() > os.path.getmtime(index_file) + 86400: + self.download_index(index_file) + + def download_index(self, index_file): + logger = logging.getLogger('d96tm') + logger.info('Fetcthing D96TM index...') + links = [] + + fishnet = 'LIDAR_FISHNET_D96.shp' + + req = requests.get(self.fishnet_url, stream=True) + with tmpdir.tmpdir() as d: + with zipfile.ZipFile(io.BytesIO(req.content)) as zip_file: + zip_file.extractall(d) + + fishnet_file = os.path.join(d, fishnet) + driver = ogr.GetDriverByName("ESRI Shapefile") + dataSource = driver.Open(fishnet_file, 1) + layer = dataSource.GetLayer() + + for feature in layer: + links.append(feature.GetField("BLOK") + "/D96TM/TMR_" + feature.GetField("NAME") + ".laz") + layer.ResetReading() + + with open(index_file, 'w') as file: + file.write(yaml.dump(links)) + + def _ensure_tile_index(self): + if self.tile_index is None: + index_file = os.path.join(self.base_dir, 'index.yaml') + bbox = (15.67583333,46.38861111,15.74166667,46.43305556) + self.tile_index = index.create(index_file, bbox, _parse_d96tm_tile, self) + + return self.tile_index + + def existing_files(self): + for base, dirs, files in os.walk(self.base_dir): + for f in files: + if f.endswith('tif'): + yield os.path.join(base, f) + + def rehydrate(self, data): + assert data.get('type') == 'd96tm', \ + "Unable to rehydrate %r from Slovenia." %data + return _parse_d96tm_tile(data['link'], self) + + def downloads_for(self, tile): + tiles = set() + # if the tile scale is greater than 20x the D96TM scale, then there's no + # point in including D96TM, it'll be far too fine to make a difference. + # D96TM is 1m. + + if tile.max_resolution() > 20: + return tiles + + # buffer by 0.0075 degrees (81px) to grab neighbouring tiles and ensure + # some overlap to take care of boundary issues. + tile_bbox = tile.latlon_bbox().buffer(0.0075) + + tile_index = self._ensure_tile_index() + + for t in index.intersections(tile_index, tile_bbox): + tiles.add(t) + + return tiles + + def filter_type(self, src_res, dst_res): + return gdal.GRA_Lanczos if src_res > dst_res else gdal.GRA_Cubic + + def vrts_for(self, tile): + """ + Returns a list of sets of tiles, with each list element intended as a + separate VRT for use in GDAL. + + D96TM is non-overlapping. + """ + return [self.downloads_for(tile)] + + def srs(self): + return srs.d96() + +def create(options): + return D96TM(options) \ No newline at end of file diff --git a/joerd/srs.py b/joerd/srs.py index 45923ad..d3a3476 100644 --- a/joerd/srs.py +++ b/joerd/srs.py @@ -12,6 +12,15 @@ 'AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0],UNIT["degree",' \ '0.0174532925199433],AUTHORITY["EPSG","4269"]]' +D96_WKT = 'PROJCS["Slovenia 1996 / Slovene National Grid",GEOGCS["Slovenia 1996",' \ +'DATUM["Slovenia_Geodetic_Datum_1996",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],' \ +'TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6765"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],' \ +'UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4765"]],PROJECTION["Transverse_Mercator"],' \ +'PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",15],PARAMETER["scale_factor",0.9999],' \ +'PARAMETER["false_easting",500000],PARAMETER["false_northing",-5000000],UNIT["metre",1,AUTHORITY["EPSG","9001"]],' \ +'AXIS["Easting",EAST],AXIS["Northing",NORTH],AUTHORITY["EPSG","3794"]]' + + def wgs84(): sr = osr.SpatialReference() @@ -23,3 +32,8 @@ def nad83(): sr = osr.SpatialReference() sr.ImportFromWkt(NAD83_WKT) return sr + +def d96(): + sr = osr.SpatialReference() + sr.ImportFromWkt(D96_WKT) + return sr \ No newline at end of file From b1e68dd42c7a7779a5891fc390418f3dfee4b002 Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Thu, 29 Oct 2020 20:14:45 +0100 Subject: [PATCH 02/21] Fixed srtm and example config. Downloading now supports basic auth --- README.md | 2 ++ config.example.yaml | 10 +++++++-- config.slovenia.yaml | 42 ++++++++++++++++++++++++++++++++++++++ config.slovenia_test.yaml | 42 ++++++++++++++++++++++++++++++++++++++ joerd/download.py | 33 +++++++++++++++++++++++++++++- joerd/source/d96tm.py | 1 + joerd/source/srtm.py | 43 ++++++++++++++++++++++++++------------- 7 files changed, 156 insertions(+), 17 deletions(-) create mode 100644 config.slovenia.yaml create mode 100644 config.slovenia_test.yaml diff --git a/README.md b/README.md index 6940eb2..6875f49 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ sudo apt-get install python-gdal python-bs4 python-numpy gdal-bin python-setupto (NOTE: not sure if this works: I installed GDAL-2.0.1 manually here, but I don't think it really needs it.) +(NOTE: if you want to use d96tm, you must install pdal, python-pdal, and LASzip or LAZperf) + You can then install it (recommended in a `virtualenv`) by running: ```sh diff --git a/config.example.yaml b/config.example.yaml index 0066cea..e8a6188 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -29,8 +29,11 @@ sources: tries: 100 - type: greatlakes - type: srtm - url: http://e4ftl01.cr.usgs.gov/SRTM/SRTMGL1.003/2000.02.11/ - mask-url: http://e4ftl01.cr.usgs.gov/SRTM/SRTMSWBD.003/2000.02.11/ + url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 + mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 + username: 'earthdata username' + password: 'earthdata password' + auth-url: https://urs.earthdata.nasa.gov - type: ned13 ftp_server: rockyftp.cr.usgs.gov base_path: vdelivery/Datasets/Staged/NED/13/IMG @@ -40,5 +43,8 @@ sources: - type: ned_topobathy ftp_server: rockyftp.cr.usgs.gov base_path: vdelivery/Datasets/Staged/NED/19/IMG + - type: d96tm + uri: http://gis.arso.gov.si/lidar/otr/laz + fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip logging: config: logging.example.config diff --git a/config.slovenia.yaml b/config.slovenia.yaml new file mode 100644 index 0000000..3536f05 --- /dev/null +++ b/config.slovenia.yaml @@ -0,0 +1,42 @@ +--- +regions: + # san-francisco-bay_california: + # bbox: + # top: 38.719 + # left: -123.640 + # bottom: 36.791 + # right: -121.025 + # zoom_range: [0, 16] + slovenia: + bbox: + top: 46.88333333 + left: 13.39027778 + bottom: 45.40750000 + right: 16.62694444 + zoom_range: [0, 15] +outputs: + - type: skadi + - type: tiff + - type: terrarium + - type: normal +sources: + - type: srtm + url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 + mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 + username: 'earthdata username' + password: 'earthdata password' + auth-url: https://urs.earthdata.nasa.gov + - type: d96tm + uri: http://gis.arso.gov.si/lidar/otr/laz + fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip +logging: + config: logging.example.config +cluster: + queue: + type: fake +store: + type: file + base_dir: 'render' +source_store: + type: file + base_dir: 'source' \ No newline at end of file diff --git a/config.slovenia_test.yaml b/config.slovenia_test.yaml new file mode 100644 index 0000000..fa703c3 --- /dev/null +++ b/config.slovenia_test.yaml @@ -0,0 +1,42 @@ +--- +regions: + # san-francisco-bay_california: + # bbox: + # top: 38.719 + # left: -123.640 + # bottom: 36.791 + # right: -121.025 + # zoom_range: [0, 16] + slovenia-test: + bbox: + top: 46.43305556 + left: 15.67583333 + bottom: 46.38861111 + right: 15.74166667 + zoom_range: [14, 15] +outputs: + - type: skadi + - type: tiff + - type: terrarium + - type: normal +sources: + - type: srtm + url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 + mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 + username: 'earthdata username' + password: 'earthdata password' + auth-url: https://urs.earthdata.nasa.gov + - type: 'd96tm' + uri: 'http://gis.arso.gov.si/lidar/otr/laz' + fishnet_url: 'https://github.com/DavixDevelop/TerraLidar/raw/master/mapzen_data/lidar_fishnet_D96TM.zip' +logging: + config: logging.example.config +cluster: + queue: + type: fake +store: + type: file + base_dir: 'render' +source_store: + type: file + base_dir: 'source' \ No newline at end of file diff --git a/joerd/download.py b/joerd/download.py index 358c780..7b39762 100644 --- a/joerd/download.py +++ b/joerd/download.py @@ -62,6 +62,15 @@ def get(url, options={}): # to restart from the beginning every time). accept_range = False + #auth username + username = options.get('username') + + #auth pass + password = options.get('password') + + #auth top level url + auth_url = options.get('auth-url') + # we need to download _something_ if the file position is less than the # known size, or the size is unknown. while filesize is None or filepos < filesize: @@ -77,6 +86,17 @@ def get(url, options={}): req = urllib2.Request(url) + # if the user provided a username and password, add Authorization + if username is not None: + redirectHandler = urllib2.HTTPRedirectHandler() + cookieProcessor = urllib2.HTTPCookieProcessor() + passwordManager = urllib2.HTTPPasswordMgrWithDefaultRealm() + passwordManager.add_password(None, auth_url, username, password) + authHandler = urllib2.HTTPBasicAuthHandler(passwordManager) + opener = urllib2.build_opener(redirectHandler,cookieProcessor,authHandler) + urllib2.install_opener(opener) + #req.add_header("Authorization", "Basic %s" % base64string) + # if the server supports accept range, and we have a partial # download then attemp to resume it. if accept_range and filepos > 0: @@ -93,7 +113,9 @@ def get(url, options={}): tmp.seek(0, os.SEEK_SET) tmp.truncate(0) + try: + f = urllib2.urlopen(req, timeout=timeout) # try to get the filesize, if the server reports it. @@ -183,4 +205,13 @@ def options(in_opts={}): tries = in_opts.get('tries', 10) out_opts['tries'] = int(tries) - return out_opts + username = in_opts.get('username', None) + out_opts['username'] = username + + password = in_opts.get('password', None) + out_opts['password'] = password + + auth_url = in_opts.get('auth-url', None) + out_opts['auth-url'] = auth_url + + return out_opts \ No newline at end of file diff --git a/joerd/source/d96tm.py b/joerd/source/d96tm.py index 4b5c049..960c542 100644 --- a/joerd/source/d96tm.py +++ b/joerd/source/d96tm.py @@ -205,6 +205,7 @@ def download_index(self, index_file): def _ensure_tile_index(self): if self.tile_index is None: index_file = os.path.join(self.base_dir, 'index.yaml') + #this cords are only for the test data and need to be changed for which region you want to download bbox = (15.67583333,46.38861111,15.74166667,46.43305556) self.tile_index = index.create(index_file, bbox, _parse_d96tm_tile, self) diff --git a/joerd/source/srtm.py b/joerd/source/srtm.py index 7af8386..a15859b 100644 --- a/joerd/source/srtm.py +++ b/joerd/source/srtm.py @@ -9,6 +9,7 @@ from joerd.mkdir_p import mkdir_p from contextlib2 import closing, ExitStack from shutil import copyfile, move +from ftplib import FTP import os.path import os import requests @@ -82,12 +83,12 @@ def _unpack_hgt(self, zip_name, target_dir): zfile.extract(n, target_dir) if n != self.fname: move(os.path.join(target_dir, n), - os.path.join(target_dir, self.fname)) + os.path.join(target_dir, self.fname)) return raise LookupError("None of the alternative names %r were found " - "in the SRTM zipfile %r. Contents are: %r" % - (names, zip_name, exists)) + "in the SRTM zipfile %r. Contents are: %r" % + (names, zip_name, exists)) def unpack(self, store, data_zip, mask_zip=None): with store.upload_dir() as target: @@ -112,13 +113,18 @@ def unpack(self, store, data_zip, mask_zip=None): # mask off the water using the mask raster raw file output_file = os.path.join(target, self.output_file()) mask.raw(os.path.join(d, self.fname), mask_file, 255, - "SRTMHGT", output_file) + "SRTMHGT", output_file) def freeze_dry(self): return dict(type='srtm', link=self.link, is_masked=self.is_masked) def _parse_srtm_tile(link, parent, is_masked=None): + i = IS_SRTM_FILE.match(link) + + if not i: + return None + fname = link.replace(".SRTMGL1.hgt.zip", ".hgt") bbox = parent._parse_bbox(link) if is_masked is None: @@ -128,7 +134,6 @@ def _parse_srtm_tile(link, parent, is_masked=None): class SRTM(object): - def __init__(self, options={}): self.base_dir = options.get('base_dir', 'srtm') self.url = options['url'] @@ -155,7 +160,7 @@ def get_one_index(self, name): index_file = os.path.join(self.base_dir, fname) # if index doesn't exist, or is more than 24h old if not os.path.isfile(index_file) or \ - time.time() > os.path.getmtime(index_file) + 86400: + time.time() > os.path.getmtime(index_file) + 86400: self.download_index(index_file, name) def download_index(self, index_file, name): @@ -167,30 +172,40 @@ def download_index(self, index_file, name): url = None if name == 'tile': - url = self.url + url = self.url if name == 'mask': url = self.mask_url - r = requests.get(url) + url_r = url + "/" + + r = requests.get(url.replace('http','https')) soup = BeautifulSoup(r.text, 'html.parser') links = [] for a in soup.find_all('a'): link = a.get('href') if link is not None: - bbox = self._parse_bbox(link) - if bbox: - links.append(link) + if link.find(url.replace('http','https')) != -1: + req = requests.get(link) + isoup = BeautifulSoup(req.text, 'html.parser') + for aa in isoup.find_all('a'): + ilink = aa.get('href') + if ilink is not None: + fname = ilink.replace(url_r, '') + bbox = self._parse_bbox(fname) + if bbox: + links.append(fname) with open(index_file, 'w') as f: f.write(yaml.dump(links)) + + def _ensure_tile_index(self): if self.tile_index is None: index_file = os.path.join(self.base_dir, 'index_tile.yaml') bbox = (-180, -90, 180, 90) - self.tile_index = index.create(index_file, bbox, _parse_srtm_tile, - self) + self.tile_index = index.create(index_file, bbox, _parse_srtm_tile,self) return self.tile_index @@ -271,4 +286,4 @@ def _parse_bbox(self, link): def create(options): - return SRTM(options) + return SRTM(options) \ No newline at end of file From 5788ffe5f8847446d5c889ffce444b5be119c0e3 Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Thu, 29 Oct 2020 21:23:26 +0100 Subject: [PATCH 03/21] Updated docs --- config.slovenia_test.yaml | 6 +++--- docs/data-sources.md | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config.slovenia_test.yaml b/config.slovenia_test.yaml index fa703c3..70f3bf1 100644 --- a/config.slovenia_test.yaml +++ b/config.slovenia_test.yaml @@ -26,9 +26,9 @@ sources: username: 'earthdata username' password: 'earthdata password' auth-url: https://urs.earthdata.nasa.gov - - type: 'd96tm' - uri: 'http://gis.arso.gov.si/lidar/otr/laz' - fishnet_url: 'https://github.com/DavixDevelop/TerraLidar/raw/master/mapzen_data/lidar_fishnet_D96TM.zip' + - type: d96tm + uri: http://gis.arso.gov.si/lidar/otr/laz + fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip logging: config: logging.example.config cluster: diff --git a/docs/data-sources.md b/docs/data-sources.md index bfecb46..3386631 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -21,6 +21,7 @@ The underlying data sources are a mix of: - [Kartverket](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33)'s Digital Terrain Model, 10 meters over Norway - [LINZ](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/), 8 meters over New Zealand - [SRTM](https://lta.cr.usgs.gov/SRTM) globally except high latitudes, 30 meters (90 meters nominal quality) in land areas +- [D96TM](http://gis.arso.gov.si/evode/profile.aspx?id=atlas_voda_Lidar%40Arso&initialExtent=499500.5%2C109841.5%2C264.58333) derived from Lidar, 1200m-1400m over Slovenia ### Footprints database From bf70ec74b370dabdbbb05cb247b6d5e9549e3c5a Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Thu, 29 Oct 2020 22:58:56 +0100 Subject: [PATCH 04/21] Updated docs --- docs/attribution.md | 9 +++++++++ docs/data-sources.md | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/attribution.md b/docs/attribution.md index 39d7c8c..f068c3c 100644 --- a/docs/attribution.md +++ b/docs/attribution.md @@ -200,3 +200,12 @@ Attribution statement: Attribution statement: > © Commonwealth of Australia (Geoscience Australia) 2017. + +### Digital Elevation Model (DEM) of Slovenia derived from LiDAR 1 meter grid + +[Released](https://gis.arso.gov.si/related/lidar_porocila/b_25_izdelava_izdelkov.pdf) by Slovenia's [GURS](https://www.e-prostor.gov.si/), licensed under [Creative Commons 4.0](https://creativecommons.org/licenses/by/4.0/deed.sl) + +Attribution statement: + +> Geodetska uprava Republike Slovenije, Lidar 2016 +> © 2017 MOP - Geodetska uprava Republike Slovenije - Vse pravice pridržane. diff --git a/docs/data-sources.md b/docs/data-sources.md index 3386631..b03ffef 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -21,7 +21,7 @@ The underlying data sources are a mix of: - [Kartverket](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33)'s Digital Terrain Model, 10 meters over Norway - [LINZ](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/), 8 meters over New Zealand - [SRTM](https://lta.cr.usgs.gov/SRTM) globally except high latitudes, 30 meters (90 meters nominal quality) in land areas -- [D96TM](http://gis.arso.gov.si/evode/profile.aspx?id=atlas_voda_Lidar%40Arso&initialExtent=499500.5%2C109841.5%2C264.58333) derived from Lidar, 1200m-1400m over Slovenia +- [D96TM](http://gis.arso.gov.si/evode/profile.aspx?id=atlas_voda_Lidar%40Arso&initialExtent=499500.5%2C109841.5%2C264.58333) derived from pre-processed Lidar, which contains only the terrain at 1m resolution, 1200m-1400m over Slovenia ### Footprints database From d5630c7a9f5d103a80b0ee819db0cae0e273fce3 Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Fri, 30 Oct 2020 22:26:01 +0100 Subject: [PATCH 05/21] Renamed d96tm to otr and updated docs --- .gitignore | 57 ++--- config.example.yaml | 100 ++++----- config.slovenia.yaml | 82 ++++---- config.slovenia_test.yaml | 82 ++++---- docs/attribution.md | 422 +++++++++++++++++++------------------- docs/data-sources.md | 292 +++++++++++++------------- joerd/source/otr.py | 260 +++++++++++++++++++++++ 7 files changed, 778 insertions(+), 517 deletions(-) create mode 100644 joerd/source/otr.py diff --git a/.gitignore b/.gitignore index 1ff6a80..77440f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,28 +1,29 @@ -# editor temporaries -*~ - -# python byte code -*.pyc - -# setup.py stuff -build/ -dist/ -joerd.egg-info/ -/*.egg/ -/*.egg -/.eggs - -# data directories -srtm/ -gmted/ -etopo1/ -ned/ -ned_topobathy/ - -# tile generation directories -tiles/ -terrarium_tiles/ -normal_tiles/ - -# macOS sillyness -.DS_Store +# editor temporaries +*~ + +# python byte code +*.pyc + +# setup.py stuff +build/ +dist/ +joerd.egg-info/ +/*.egg/ +/*.egg +/.eggs + +# data directories +srtm/ +gmted/ +etopo1/ +ned/ +ned_topobathy/ +otr/ + +# tile generation directories +tiles/ +terrarium_tiles/ +normal_tiles/ + +# macOS sillyness +.DS_Store diff --git a/config.example.yaml b/config.example.yaml index e8a6188..9f08f84 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1,50 +1,50 @@ ---- -regions: - # san-francisco-bay_california: - # bbox: - # top: 38.719 - # left: -123.640 - # bottom: 36.791 - # right: -121.025 - # zoom_range: [0, 16] - san-francisco-downtown: - bbox: - top: 37.7997 - left: -122.5109 - bottom: 37.7127 - right: -122.3636 - zoom_range: [0, 16] -outputs: - - type: skadi - - type: tiff - - type: terrarium - - type: normal -sources: - - type: etopo1 - url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip - - type: gmted - url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED - ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] - xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] - tries: 100 - - type: greatlakes - - type: srtm - url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 - mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 - username: 'earthdata username' - password: 'earthdata password' - auth-url: https://urs.earthdata.nasa.gov - - type: ned13 - ftp_server: rockyftp.cr.usgs.gov - base_path: vdelivery/Datasets/Staged/NED/13/IMG - - type: ned - ftp_server: rockyftp.cr.usgs.gov - base_path: vdelivery/Datasets/Staged/NED/19/IMG - - type: ned_topobathy - ftp_server: rockyftp.cr.usgs.gov - base_path: vdelivery/Datasets/Staged/NED/19/IMG - - type: d96tm - uri: http://gis.arso.gov.si/lidar/otr/laz - fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip -logging: - config: logging.example.config +--- +regions: + # san-francisco-bay_california: + # bbox: + # top: 38.719 + # left: -123.640 + # bottom: 36.791 + # right: -121.025 + # zoom_range: [0, 16] + san-francisco-downtown: + bbox: + top: 37.7997 + left: -122.5109 + bottom: 37.7127 + right: -122.3636 + zoom_range: [0, 16] +outputs: + - type: skadi + - type: tiff + - type: terrarium + - type: normal +sources: + - type: etopo1 + url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip + - type: gmted + url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED + ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] + xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] + tries: 100 + - type: greatlakes + - type: srtm + url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 + mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 + username: 'earthdata username' + password: 'earthdata password' + auth-url: https://urs.earthdata.nasa.gov + - type: ned13 + ftp_server: rockyftp.cr.usgs.gov + base_path: vdelivery/Datasets/Staged/NED/13/IMG + - type: ned + ftp_server: rockyftp.cr.usgs.gov + base_path: vdelivery/Datasets/Staged/NED/19/IMG + - type: ned_topobathy + ftp_server: rockyftp.cr.usgs.gov + base_path: vdelivery/Datasets/Staged/NED/19/IMG + - type: otr + uri: http://gis.arso.gov.si/lidar/otr/laz + fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip +logging: + config: logging.example.config diff --git a/config.slovenia.yaml b/config.slovenia.yaml index 3536f05..ca89421 100644 --- a/config.slovenia.yaml +++ b/config.slovenia.yaml @@ -1,42 +1,42 @@ ---- -regions: - # san-francisco-bay_california: - # bbox: - # top: 38.719 - # left: -123.640 - # bottom: 36.791 - # right: -121.025 - # zoom_range: [0, 16] - slovenia: - bbox: - top: 46.88333333 - left: 13.39027778 - bottom: 45.40750000 - right: 16.62694444 - zoom_range: [0, 15] -outputs: - - type: skadi - - type: tiff - - type: terrarium - - type: normal -sources: - - type: srtm - url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 - mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 - username: 'earthdata username' - password: 'earthdata password' - auth-url: https://urs.earthdata.nasa.gov - - type: d96tm - uri: http://gis.arso.gov.si/lidar/otr/laz - fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip -logging: - config: logging.example.config -cluster: - queue: - type: fake -store: - type: file - base_dir: 'render' -source_store: - type: file +--- +regions: + # san-francisco-bay_california: + # bbox: + # top: 38.719 + # left: -123.640 + # bottom: 36.791 + # right: -121.025 + # zoom_range: [0, 16] + slovenia: + bbox: + top: 46.88333333 + left: 13.39027778 + bottom: 45.40750000 + right: 16.62694444 + zoom_range: [0, 15] +outputs: + - type: skadi + - type: tiff + - type: terrarium + - type: normal +sources: + - type: srtm + url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 + mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 + username: 'earthdata username' + password: 'earthdata password' + auth-url: https://urs.earthdata.nasa.gov + - type: otr + uri: http://gis.arso.gov.si/lidar/otr/laz + fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip +logging: + config: logging.example.config +cluster: + queue: + type: fake +store: + type: file + base_dir: 'render' +source_store: + type: file base_dir: 'source' \ No newline at end of file diff --git a/config.slovenia_test.yaml b/config.slovenia_test.yaml index 70f3bf1..539efd6 100644 --- a/config.slovenia_test.yaml +++ b/config.slovenia_test.yaml @@ -1,42 +1,42 @@ ---- -regions: - # san-francisco-bay_california: - # bbox: - # top: 38.719 - # left: -123.640 - # bottom: 36.791 - # right: -121.025 - # zoom_range: [0, 16] - slovenia-test: - bbox: - top: 46.43305556 - left: 15.67583333 - bottom: 46.38861111 - right: 15.74166667 - zoom_range: [14, 15] -outputs: - - type: skadi - - type: tiff - - type: terrarium - - type: normal -sources: - - type: srtm - url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 - mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 - username: 'earthdata username' - password: 'earthdata password' - auth-url: https://urs.earthdata.nasa.gov - - type: d96tm - uri: http://gis.arso.gov.si/lidar/otr/laz - fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip -logging: - config: logging.example.config -cluster: - queue: - type: fake -store: - type: file - base_dir: 'render' -source_store: - type: file +--- +regions: + # san-francisco-bay_california: + # bbox: + # top: 38.719 + # left: -123.640 + # bottom: 36.791 + # right: -121.025 + # zoom_range: [0, 16] + slovenia-test: + bbox: + top: 46.43305556 + left: 15.67583333 + bottom: 46.38861111 + right: 15.74166667 + zoom_range: [14, 15] +outputs: + - type: skadi + - type: tiff + - type: terrarium + - type: normal +sources: + - type: srtm + url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 + mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 + username: 'earthdata username' + password: 'earthdata password' + auth-url: https://urs.earthdata.nasa.gov + - type: otr + uri: http://gis.arso.gov.si/lidar/otr/laz + fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip +logging: + config: logging.example.config +cluster: + queue: + type: fake +store: + type: file + base_dir: 'render' +source_store: + type: file base_dir: 'source' \ No newline at end of file diff --git a/docs/attribution.md b/docs/attribution.md index f068c3c..f00ebf7 100644 --- a/docs/attribution.md +++ b/docs/attribution.md @@ -1,211 +1,211 @@ -# Attribution - -Attribution is required for many terrain tile data providers. Example language is provided below, but you are responsible for researching each project to follow their license terms. More details are available on the [Data Sources](data-sources.md) page at [Mapzen rights](https://mapzen.com/rights) for Mapzen's hosted service. - -***Required attribution:*** - -``` -* ArcticDEM terrain data DEM(s) were created from DigitalGlobe, Inc., imagery and - funded under National Science Foundation awards 1043681, 1559691, and 1542736; -* Australia terrain data © Commonwealth of Australia (Geoscience Australia) 2017; -* Austria terrain data © offene Daten Österreichs – Digitales Geländemodell (DGM) - Österreich; -* Canada terrain data contains information licensed under the Open Government - Licence – Canada; -* Europe terrain data produced using Copernicus data and information funded by the - European Union - EU-DEM layers; -* Global ETOPO1 terrain data U.S. National Oceanic and Atmospheric Administration -* Mexico terrain data source: INEGI, Continental relief, 2016; -* New Zealand terrain data Copyright 2011 Crown copyright (c) Land Information New - Zealand and the New Zealand Government (All rights reserved); -* Norway terrain data © Kartverket; -* United Kingdom terrain data © Environment Agency copyright and/or database right - 2015. All rights reserved; -* United States 3DEP (formerly NED) and global GMTED2010 and SRTM terrain data - courtesy of the U.S. Geological Survey. -``` - -***Required attribution when using Mapzen's hosted service:*** - -``` -* Mapzen -* ArcticDEM terrain data DEM(s) were created from DigitalGlobe, Inc., imagery and - funded under National Science Foundation awards 1043681, 1559691, and 1542736; -* Australia terrain data © Commonwealth of Australia (Geoscience Australia) 2017; -* Austria terrain data © offene Daten Österreichs – Digitales Geländemodell (DGM) - Österreich; -* Canada terrain data contains information licensed under the Open Government - Licence – Canada; -* Europe terrain data produced using Copernicus data and information funded by the - European Union - EU-DEM layers; -* Global ETOPO1 terrain data U.S. National Oceanic and Atmospheric Administration -* Mexico terrain data source: INEGI, Continental relief, 2016; -* New Zealand terrain data Copyright 2011 Crown copyright (c) Land Information New - Zealand and the New Zealand Government (All rights reserved); -* Norway terrain data © Kartverket; -* United Kingdom terrain data © Environment Agency copyright and/or database right - 2015. All rights reserved; -* United States 3DEP (formerly NED) and global GMTED2010 and SRTM terrain data - courtesy of the U.S. Geological Survey. -``` - -### Where to attribute - -Attribution needs to "appear in a place that is reasonable to the medium or means you are utilising." [Specific examples](http://wiki.osmfoundation.org/wiki/License#Where_to_put_it.3F) are given by the OSM Foundation and are generally best practices for giving credit to any source. More information on attribution is on our [rights](https://mapzen.com/rights) page. - -## The fine print - -### 3DEP - -3DEP (formerly NED) are several public domain datasets released by USGS: - -> All 3DEP products available through The National Map are in the public domain and may be used without restriction. - -However, the USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. - -`3DEP data courtesy of the U.S. Geological Survey` - -More details can be found on the NED/3DEP [FAQ](https://www2.usgs.gov/faq/categories/9865/5041) page, [info](http://nationalmap.gov/3DEP/3dep_prodserv.html) page, and general [acknowledging](https://www2.usgs.gov/visual-id/credit_usgs.html) page. - - -### SRTM - -SRTM is a public domain dataset released as follows, circa 2000 by NASA/NGA with several updates including in 2014, now distributed by USGS: - -> 3.15 -> -> a. Copyright protection is asserted for all products generated by these specifications which are distributed outside of the United States. No domestic copyright will be asserted. -> -> b. The copyright notice (with year of production inserted) states: -> -> © COPYRIGHT (year of production) BY THE UNITED STATES GOVERNMENT. -> NO COPYRIGHT CLAIMED UNDER TITLE 17 U.S.C. - -However, the USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. - -`SRTM data courtesy of the U.S. Geological Survey` - -More details can be found on the SRTM [about](https://lta.cr.usgs.gov/SRTM) from USGS and [about](http://www2.jpl.nasa.gov/srtm/) page from NASA, [data distribution policy](https://lta.cr.usgs.gov/srtm/data_distribution_policy) page, the [metadata](http://dds.cr.usgs.gov/srtm/version2_1/Documentation/MIL-PDF-89020B.pdf), and NASA/NGA original [memorandum of understanding](http://www2.jpl.nasa.gov/srtm/mou.html) and 2014 [press release](http://www.jpl.nasa.gov/news/news.php?release=2014-321) declassifying the global 30 meter data. - -### GMTED2010 - -GMTED2010 is a public domain dataset released, circa 2010 by USGS: - -> Access constraints: -> -> No restrictions, All GMTED2010 data products are publically available. Any acquisition or use of these data signifies a user's agreement to comprehension and compliance of the USGS Standard Disclaimer. Ensure all portions of metadata are read and clearly understood before using these data in order to protect both user and USGS interests. Please refer to http://www.usgs.gov/privacy.html for the USGS disclaimer. -> -> Use constraints: -> -> Although the USGS is making these data available to others who may find the data of value, USGS does not warrant, endorse, or recommend the use of these data for any given purpose. The user assumes the entire risk related to the use of these data. USGS is providing these data "as is", and USGS disclaims any and all warranties, whether expressed or implied, including (without limitation) any implied warranties of merchantability or fitness for a particular purpose. In no event will USGS be liable to you or to any third party for any direct, indirect, incidental, consequential, special, or exemplary damages or lost profits resulting from any use or misuse of these data. Acknowledgement of the U.S. Geological Survey would be appreciated in products derived from these data. Any user who modifies the data is obligated to describe the types of modifications they perform. User specifically agrees not to misrepresent the data, nor to imply that changes made were approved or endorsed by the USGS. - -The USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. - -`GMTED2010 data courtesy of the U.S. Geological Survey` - -More details can be found on the GMTED2010 [about](http://topotools.cr.usgs.gov/gmted_viewer/) page and [Open File Report](http://pubs.usgs.gov/of/2011/1073/pdf/of2011-1073.pdf) PDF, and updated [home](https://lta.cr.usgs.gov/GMTED2010) page. - -### ETOPO1 - -ETOPO1 is a public domain dataset released as follows, circa 2009 by NOAA: - -> Use Limitations -> -> Not to be used for navigation. Although these data are of high quality and useful for planning and modeling purposes, they are not suitable for navigation. For navigation, please refer to the NOS nautical chart series. -> -> Produced by the NOAA National Geophysical Data Center. Not subject to copyright protection within the United States. -> -> While every effort has been made to ensure that these data are accurate and reliable within the limits of the current state of the art, NOAA cannot assume liability for any damages caused by any errors or omissions in the data, nor as a result of the failure of the data to function on a particular system. NOAA makes no warranty, expressed or implied, nor does the fact of distribution constitute such a warranty. - -and - -> Copyright Notice -> -> As required by 17 U.S.C. 403, third parties producing copyrighted works consisting predominantly of the material produced by U.S. government agencies must provide notice with such work(s) identifying the U.S. Government material incorporated and stating that such material is not subject to copyright protection within the United States. - -Formal credit is specified as: - -> DOC/NOAA/NESDIS/NCEI > National Centers for Environmental Information, NESDIS, NOAA, U.S. Department of Commerce - -More details can be found on the ETOPO1 [about](https://www.ngdc.noaa.gov/mgg/global/global.html) page and [data sources](https://www.ngdc.noaa.gov/mgg/global/etopo1sources.html) page, and [description](https://www.ngdc.noaa.gov/docucomp/page?xml=NOAA/NESDIS/NGDC/MGG/DEM/iso/xml/316.xml&view=getDataView&header=none) page and [disclaimer](https://www.ngdc.noaa.gov/ngdcinfo/privacy.html#disclaimer) page. - -### Land Information New Zealand (LINZ) - -[Released](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/) by [LINZ](https://data.linz.govt.nz/), licensed under [Creative Commons Attribution 3.0 New Zealand](https://data.linz.govt.nz/license/attribution-3-0-new-zealand/). - -Attribution statement: - -> Copyright 2011 Crown copyright (c) Land Information New Zealand and the New Zealand Government. All rights reserved - -### data.gov.uk LIDAR Composite Digital Terrain Model - -[Released](https://data.gov.uk/dataset/lidar-composite-dtm-2m1) by [data.gov.uk](https://data.gov.uk/), licensed under the United Kingdom [Open Government License version 3](http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) - -Attribution statement: - -> © Environment Agency copyright and/or database right 2015. All rights reserved. - -### data.gv.at Digitales Geländemodell (DGM) Österreich - -[Released](https://www.data.gv.at/katalog/dataset/b5de6975-417b-4320-afdb-eb2a9e2a1dbf) by Austria's [data.gv.at](https://www.data.gv.at/), licensed under [Creative Commons Namensnennung 3.0 Österreich](https://creativecommons.org/licenses/by/3.0/at/deed.de) - -Attribution statement: - -> © offene Daten Österreichs – Digitales Geländemodell (DGM) Österreich. - -### data.kartverket.no Digital terrengmodell - -[Released](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33) by Austria's Kartverket, [licensed](http://www.kartverket.no/data/lisens/) under [Creative Commons Attribution 4.0 International (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/deed.no). - -Attribution statement: - -> © Kartverket - -### Arctic Digital Elevation Model (ArcticDEM) - -[Released](http://nga.maps.arcgis.com/apps/MapSeries/index.html?appid=cf2fba21df7540fb981f8836f2a97e25) without limitation. "ArcticDEM data is an unlicensed product and may be used, distributed, and modified without permission". - -Requested citation: - -> DEM(s) were created from DigitalGlobe, Inc., imagery and funded under National Science Foundation awards 1043681, 1559691, and 1542736. - -### Digital Terrain Model over Europe (EU-DEM) - -[Released](https://www.eea.europa.eu/data-and-maps/data/eu-dem#tab-metadata) by [European Environmental Agency](https://www.eea.europa.eu/). - -Attribution statement: - -> Produced using Copernicus data and information funded by the European Union - EU-DEM layers. - -### Canadian Digital Elevation Model (CDEM) - -[Released](http://ftp.geogratis.gc.ca/pub/nrcan_rncan/elevation/cdem_mnec/doc/CDEM_product_specs.pdf) under the [Open Government Licence Agreement for Unrestricted Use of Digital Data](http://open.canada.ca/en/open-government-licence-canada). - -Attribution statement: - -> Contains information licensed under the Open Government Licence – Canada. - -### National Institute of Statistics and Geography (INEGI) - -[Released](http://en.www.inegi.org.mx/temas/mapas/relieve/continental/) under Mexico's ["free use of information" license](http://en.www.inegi.org.mx/inegi/terminos.html). - -Attribution statement: - -> Source: INEGI, Continental relief, 2016 - -### Digital Elevation Model (DEM) of Australia derived from LiDAR 5 Metre Grid - -[Released](https://ecat.ga.gov.au/geonetwork/srv/eng/search#!22be4b55-2465-4320-e053-10a3070a5236) by Australia's [Geoscience Australia](http://www.ga.gov.au/), licensed under [Creative Commons Attribution 4.0 International Licence](https://creativecommons.org/licenses/by/4.0/) - -Attribution statement: - -> © Commonwealth of Australia (Geoscience Australia) 2017. - -### Digital Elevation Model (DEM) of Slovenia derived from LiDAR 1 meter grid - -[Released](https://gis.arso.gov.si/related/lidar_porocila/b_25_izdelava_izdelkov.pdf) by Slovenia's [GURS](https://www.e-prostor.gov.si/), licensed under [Creative Commons 4.0](https://creativecommons.org/licenses/by/4.0/deed.sl) - -Attribution statement: - -> Geodetska uprava Republike Slovenije, Lidar 2016 -> © 2017 MOP - Geodetska uprava Republike Slovenije - Vse pravice pridržane. +# Attribution + +Attribution is required for many terrain tile data providers. Example language is provided below, but you are responsible for researching each project to follow their license terms. More details are available on the [Data Sources](data-sources.md) page at [Mapzen rights](https://mapzen.com/rights) for Mapzen's hosted service. + +***Required attribution:*** + +``` +* ArcticDEM terrain data DEM(s) were created from DigitalGlobe, Inc., imagery and + funded under National Science Foundation awards 1043681, 1559691, and 1542736; +* Australia terrain data © Commonwealth of Australia (Geoscience Australia) 2017; +* Austria terrain data © offene Daten Österreichs – Digitales Geländemodell (DGM) + Österreich; +* Canada terrain data contains information licensed under the Open Government + Licence – Canada; +* Europe terrain data produced using Copernicus data and information funded by the + European Union - EU-DEM layers; +* Global ETOPO1 terrain data U.S. National Oceanic and Atmospheric Administration +* Mexico terrain data source: INEGI, Continental relief, 2016; +* New Zealand terrain data Copyright 2011 Crown copyright (c) Land Information New + Zealand and the New Zealand Government (All rights reserved); +* Norway terrain data © Kartverket; +* United Kingdom terrain data © Environment Agency copyright and/or database right + 2015. All rights reserved; +* United States 3DEP (formerly NED) and global GMTED2010 and SRTM terrain data + courtesy of the U.S. Geological Survey. +``` + +***Required attribution when using Mapzen's hosted service:*** + +``` +* Mapzen +* ArcticDEM terrain data DEM(s) were created from DigitalGlobe, Inc., imagery and + funded under National Science Foundation awards 1043681, 1559691, and 1542736; +* Australia terrain data © Commonwealth of Australia (Geoscience Australia) 2017; +* Austria terrain data © offene Daten Österreichs – Digitales Geländemodell (DGM) + Österreich; +* Canada terrain data contains information licensed under the Open Government + Licence – Canada; +* Europe terrain data produced using Copernicus data and information funded by the + European Union - EU-DEM layers; +* Global ETOPO1 terrain data U.S. National Oceanic and Atmospheric Administration +* Mexico terrain data source: INEGI, Continental relief, 2016; +* New Zealand terrain data Copyright 2011 Crown copyright (c) Land Information New + Zealand and the New Zealand Government (All rights reserved); +* Norway terrain data © Kartverket; +* United Kingdom terrain data © Environment Agency copyright and/or database right + 2015. All rights reserved; +* United States 3DEP (formerly NED) and global GMTED2010 and SRTM terrain data + courtesy of the U.S. Geological Survey. +``` + +### Where to attribute + +Attribution needs to "appear in a place that is reasonable to the medium or means you are utilising." [Specific examples](http://wiki.osmfoundation.org/wiki/License#Where_to_put_it.3F) are given by the OSM Foundation and are generally best practices for giving credit to any source. More information on attribution is on our [rights](https://mapzen.com/rights) page. + +## The fine print + +### 3DEP + +3DEP (formerly NED) are several public domain datasets released by USGS: + +> All 3DEP products available through The National Map are in the public domain and may be used without restriction. + +However, the USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. + +`3DEP data courtesy of the U.S. Geological Survey` + +More details can be found on the NED/3DEP [FAQ](https://www2.usgs.gov/faq/categories/9865/5041) page, [info](http://nationalmap.gov/3DEP/3dep_prodserv.html) page, and general [acknowledging](https://www2.usgs.gov/visual-id/credit_usgs.html) page. + + +### SRTM + +SRTM is a public domain dataset released as follows, circa 2000 by NASA/NGA with several updates including in 2014, now distributed by USGS: + +> 3.15 +> +> a. Copyright protection is asserted for all products generated by these specifications which are distributed outside of the United States. No domestic copyright will be asserted. +> +> b. The copyright notice (with year of production inserted) states: +> +> © COPYRIGHT (year of production) BY THE UNITED STATES GOVERNMENT. +> NO COPYRIGHT CLAIMED UNDER TITLE 17 U.S.C. + +However, the USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. + +`SRTM data courtesy of the U.S. Geological Survey` + +More details can be found on the SRTM [about](https://lta.cr.usgs.gov/SRTM) from USGS and [about](http://www2.jpl.nasa.gov/srtm/) page from NASA, [data distribution policy](https://lta.cr.usgs.gov/srtm/data_distribution_policy) page, the [metadata](http://dds.cr.usgs.gov/srtm/version2_1/Documentation/MIL-PDF-89020B.pdf), and NASA/NGA original [memorandum of understanding](http://www2.jpl.nasa.gov/srtm/mou.html) and 2014 [press release](http://www.jpl.nasa.gov/news/news.php?release=2014-321) declassifying the global 30 meter data. + +### GMTED2010 + +GMTED2010 is a public domain dataset released, circa 2010 by USGS: + +> Access constraints: +> +> No restrictions, All GMTED2010 data products are publically available. Any acquisition or use of these data signifies a user's agreement to comprehension and compliance of the USGS Standard Disclaimer. Ensure all portions of metadata are read and clearly understood before using these data in order to protect both user and USGS interests. Please refer to http://www.usgs.gov/privacy.html for the USGS disclaimer. +> +> Use constraints: +> +> Although the USGS is making these data available to others who may find the data of value, USGS does not warrant, endorse, or recommend the use of these data for any given purpose. The user assumes the entire risk related to the use of these data. USGS is providing these data "as is", and USGS disclaims any and all warranties, whether expressed or implied, including (without limitation) any implied warranties of merchantability or fitness for a particular purpose. In no event will USGS be liable to you or to any third party for any direct, indirect, incidental, consequential, special, or exemplary damages or lost profits resulting from any use or misuse of these data. Acknowledgement of the U.S. Geological Survey would be appreciated in products derived from these data. Any user who modifies the data is obligated to describe the types of modifications they perform. User specifically agrees not to misrepresent the data, nor to imply that changes made were approved or endorsed by the USGS. + +The USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. + +`GMTED2010 data courtesy of the U.S. Geological Survey` + +More details can be found on the GMTED2010 [about](http://topotools.cr.usgs.gov/gmted_viewer/) page and [Open File Report](http://pubs.usgs.gov/of/2011/1073/pdf/of2011-1073.pdf) PDF, and updated [home](https://lta.cr.usgs.gov/GMTED2010) page. + +### ETOPO1 + +ETOPO1 is a public domain dataset released as follows, circa 2009 by NOAA: + +> Use Limitations +> +> Not to be used for navigation. Although these data are of high quality and useful for planning and modeling purposes, they are not suitable for navigation. For navigation, please refer to the NOS nautical chart series. +> +> Produced by the NOAA National Geophysical Data Center. Not subject to copyright protection within the United States. +> +> While every effort has been made to ensure that these data are accurate and reliable within the limits of the current state of the art, NOAA cannot assume liability for any damages caused by any errors or omissions in the data, nor as a result of the failure of the data to function on a particular system. NOAA makes no warranty, expressed or implied, nor does the fact of distribution constitute such a warranty. + +and + +> Copyright Notice +> +> As required by 17 U.S.C. 403, third parties producing copyrighted works consisting predominantly of the material produced by U.S. government agencies must provide notice with such work(s) identifying the U.S. Government material incorporated and stating that such material is not subject to copyright protection within the United States. + +Formal credit is specified as: + +> DOC/NOAA/NESDIS/NCEI > National Centers for Environmental Information, NESDIS, NOAA, U.S. Department of Commerce + +More details can be found on the ETOPO1 [about](https://www.ngdc.noaa.gov/mgg/global/global.html) page and [data sources](https://www.ngdc.noaa.gov/mgg/global/etopo1sources.html) page, and [description](https://www.ngdc.noaa.gov/docucomp/page?xml=NOAA/NESDIS/NGDC/MGG/DEM/iso/xml/316.xml&view=getDataView&header=none) page and [disclaimer](https://www.ngdc.noaa.gov/ngdcinfo/privacy.html#disclaimer) page. + +### Land Information New Zealand (LINZ) + +[Released](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/) by [LINZ](https://data.linz.govt.nz/), licensed under [Creative Commons Attribution 3.0 New Zealand](https://data.linz.govt.nz/license/attribution-3-0-new-zealand/). + +Attribution statement: + +> Copyright 2011 Crown copyright (c) Land Information New Zealand and the New Zealand Government. All rights reserved + +### data.gov.uk LIDAR Composite Digital Terrain Model + +[Released](https://data.gov.uk/dataset/lidar-composite-dtm-2m1) by [data.gov.uk](https://data.gov.uk/), licensed under the United Kingdom [Open Government License version 3](http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) + +Attribution statement: + +> © Environment Agency copyright and/or database right 2015. All rights reserved. + +### data.gv.at Digitales Geländemodell (DGM) Österreich + +[Released](https://www.data.gv.at/katalog/dataset/b5de6975-417b-4320-afdb-eb2a9e2a1dbf) by Austria's [data.gv.at](https://www.data.gv.at/), licensed under [Creative Commons Namensnennung 3.0 Österreich](https://creativecommons.org/licenses/by/3.0/at/deed.de) + +Attribution statement: + +> © offene Daten Österreichs – Digitales Geländemodell (DGM) Österreich. + +### data.kartverket.no Digital terrengmodell + +[Released](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33) by Austria's Kartverket, [licensed](http://www.kartverket.no/data/lisens/) under [Creative Commons Attribution 4.0 International (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/deed.no). + +Attribution statement: + +> © Kartverket + +### Arctic Digital Elevation Model (ArcticDEM) + +[Released](http://nga.maps.arcgis.com/apps/MapSeries/index.html?appid=cf2fba21df7540fb981f8836f2a97e25) without limitation. "ArcticDEM data is an unlicensed product and may be used, distributed, and modified without permission". + +Requested citation: + +> DEM(s) were created from DigitalGlobe, Inc., imagery and funded under National Science Foundation awards 1043681, 1559691, and 1542736. + +### Digital Terrain Model over Europe (EU-DEM) + +[Released](https://www.eea.europa.eu/data-and-maps/data/eu-dem#tab-metadata) by [European Environmental Agency](https://www.eea.europa.eu/). + +Attribution statement: + +> Produced using Copernicus data and information funded by the European Union - EU-DEM layers. + +### Canadian Digital Elevation Model (CDEM) + +[Released](http://ftp.geogratis.gc.ca/pub/nrcan_rncan/elevation/cdem_mnec/doc/CDEM_product_specs.pdf) under the [Open Government Licence Agreement for Unrestricted Use of Digital Data](http://open.canada.ca/en/open-government-licence-canada). + +Attribution statement: + +> Contains information licensed under the Open Government Licence – Canada. + +### National Institute of Statistics and Geography (INEGI) + +[Released](http://en.www.inegi.org.mx/temas/mapas/relieve/continental/) under Mexico's ["free use of information" license](http://en.www.inegi.org.mx/inegi/terminos.html). + +Attribution statement: + +> Source: INEGI, Continental relief, 2016 + +### Digital Elevation Model (DEM) of Australia derived from LiDAR 5 Metre Grid + +[Released](https://ecat.ga.gov.au/geonetwork/srv/eng/search#!22be4b55-2465-4320-e053-10a3070a5236) by Australia's [Geoscience Australia](http://www.ga.gov.au/), licensed under [Creative Commons Attribution 4.0 International Licence](https://creativecommons.org/licenses/by/4.0/) + +Attribution statement: + +> © Commonwealth of Australia (Geoscience Australia) 2017. + +### Digital Elevation Model (DEM) of Slovenia derived from LiDAR 1 Meter Grid + +[Released](https://gis.arso.gov.si/related/lidar_porocila/b_25_izdelava_izdelkov.pdf) by Slovenia's [GURS](https://www.e-prostor.gov.si/), licensed under [Creative Commons 4.0](https://creativecommons.org/licenses/by/4.0/deed.sl) + +Attribution statement: + +> Geodetska uprava Republike Slovenije, LiDAR, OTR 2015 +> © 2017 MOP - Geodetska uprava Republike Slovenije - Vse pravice pridržane. diff --git a/docs/data-sources.md b/docs/data-sources.md index b03ffef..bd71bfe 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -1,146 +1,146 @@ -# Data sources - -Mapzen Terrain Tiles are powered by several major open data sets and we owe a tremendous debt of gratitude to the individuals and communities which produced them. - -**Attribution is required** for some data providers. See the [Attribution](https://github.com/tilezen/joerd/blob/master/docs/attribution.md) document for more information. - -## List of sources - -The underlying data sources are a mix of: - -- [3DEP](http://nationalmap.gov/elevation.html) (formerly NED and NED Topobathy) in the United States, 10 meters outside of Alaska, 3 meter in select land and territorial water areas -- [ArcticDEM](http://nga.maps.arcgis.com/apps/MapSeries/index.html?appid=cf2fba21df7540fb981f8836f2a97e25) strips of 5 meter mosaics across all of the land north of 60° latitude, including Alaska, Canada, Greenland, Iceland, Norway, Russia, and Sweden -- [CDEM](http://geogratis.gc.ca/api/en/nrcan-rncan/ess-sst/c40acfba-c722-4be1-862e-146b80be738e.html) (Canadian Digital Elevation Model) in Canada, with variable spatial resolution (from 20-400 meters) depending on the latitude. -- [data.gov.uk](http://environment.data.gov.uk/ds/survey/index.jsp#/survey), 2 meters over most of the United Kingdom -- [data.gv.at](https://www.data.gv.at/katalog/dataset/b5de6975-417b-4320-afdb-eb2a9e2a1dbf), 10 meters over Austria -- [ETOPO1](https://www.ngdc.noaa.gov/mgg/global/global.html) for ocean bathymetry, 1 arc-minute resolution globally -- [EUDEM](https://www.eea.europa.eu/data-and-maps/data/eu-dem#tab-original-data) in most of Europe at 30 meter resolution, including Albania, Austria, Belgium, Bosnia and Herzegovina, Bulgaria, Croatia, Cyprus, Czechia, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Iceland, Ireland, Italy, Kosovo, Latvia, Liechtenstein, Lithuania, Luxembourg, Macedonia, Malta, Montenegro, Netherlands, Norway, Poland, Portugal, Romania, Serbia, Slovakia, Slovenia, Spain, Sweden, Switzerland, and United Kingdom -- Geoscience Australia's [DEM of Australia](https://ecat.ga.gov.au/geonetwork/srv/eng/search#!22be4b55-2465-4320-e053-10a3070a5236), 5 meters around coastal regions in South Australia, Victoria, and Northern Territory -- [GMTED](http://topotools.cr.usgs.gov/gmted_viewer/) globally, coarser resolutions at 7.5", 15", and 30" in land areas -- [INEGI](http://en.www.inegi.org.mx/temas/mapas/relieve/continental/)'s continental relief in Mexico -- [Kartverket](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33)'s Digital Terrain Model, 10 meters over Norway -- [LINZ](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/), 8 meters over New Zealand -- [SRTM](https://lta.cr.usgs.gov/SRTM) globally except high latitudes, 30 meters (90 meters nominal quality) in land areas -- [D96TM](http://gis.arso.gov.si/evode/profile.aspx?id=atlas_voda_Lidar%40Arso&initialExtent=499500.5%2C109841.5%2C264.58333) derived from pre-processed Lidar, which contains only the terrain at 1m resolution, 1200m-1400m over Slovenia - -### Footprints database - -These source images are composited to form tiles that make up the Mapzen Terrain Tiles service. To determine exactly which images contributed to Mapzen Terrain Tiles in a particular area, you can download the footprints database and use it with a GIS program like [QGIS](http://www.qgis.org/) to inspect which of these sources were used. - -![Preview Rendering of Footprints](images/footprints-preview.png) - -* [GeoJSON](https://s3.amazonaws.com/elevation-tiles-prod/docs/footprints.geojson.gz) (8.7MB, gzipped) -* [PostgreSQL Dump](https://s3.amazonaws.com/elevation-tiles-prod/docs/footprints.pgdump.gz) (14.5MB, gzipped) - -### Source headers - -To further assist in determining which sources contributed to an individual tile, the Mapzen Terrain Tiles service will respond with an [HTTP header](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Response_fields) listing the sources that contributed to that tile. The value of the `X-Imagery-Sources` HTTP header is a comma-separated list, where each entry follows the pattern `source/filename.tif`. - -For example, a tile might have the header `X-Imagery-Sources: srtm/N39W110.tif, srtm/N39W112.tif, gmted/30N120W_20101117_gmted_mea075.tif`, meaning that it was built from three source images. Two SRTM images and a GMTED image were composited together to generate the tile output. Using the footprint database dumps above you can gather more information about these source images, including the calculated resolution and footprint geometry. To find the entry in the database, look for an entry that has a matching `filename` attribute. - - -## What is the ground resolution? - -Ground resolution per tile pixel varies per zoom level, the given pixel cell's latitude, and input data source. - -This formula generates the following table: - -`ground_resolution = (cos(latitude * pi/180) * 2 * pi * 6378137 meters) / (256 * 2^zoom_level pixels)` - -Ground resolution per **zoom** in `meters` at a given _latitude_: - -zoom | _0°_ | _45°_ | _60°_ ------- | ---------- | ---------- | --------- -**0** | `156543.0` | `110692.6` | `78271.5` -**1** | `78271.5` | `55346.3` | `39135.8` -**2** | `39135.8` | `27673.2` | `19567.9` -**3** | `19567.9` | `13836.6` | `9783.9` -**4** | `9783.9` | `6918.3` | `4892.0` -**5** | `4892.0` | `3459.1` | `2446.0` -**6** | `2446.0` | `1729.6` | `1223.0` -**7** | `1223.0` | `864.8` | `611.5` -**8** | `611.5` | `432.4` | `305.7` -**9** | `305.7` | `216.2` | `152.9` -**10** | `152.9` | `108.1` | `76.4` -**11** | `76.4` | `54.0` | `38.2` -**12** | `38.2` | `27.0` | `19.1` -**13** | `19.1` | `13.5` | `9.6` -**14** | `9.6` | `6.8` | `4.8` -**15** | `4.8` | `3.4` | `2.4` - -**Note:** Esri has [documentation](https://blogs.esri.com/esri/arcgis/2009/03/19/how-can-you-tell-what-map-scales-are-shown-for-online-maps/) about converting web map zoom integers to conventional map scales. - -## What is sourced at what zooms? - -Generally speaking, **GMTED** is used at low-zooms, **ETOPO1** is used to show ocean bathymetry at all zooms (even bathymetry oversampled at zoom 15), and **[SRTM](http://www2.jpl.nasa.gov/srtm/)** is relied on in mid- and high-zooms on land. Some countries have higher resolution data available over land sourced from other open datasets. More information about these sources is available below. - -It should be noted that both `SRTM` and `GMTED` fill oceans and other bodies of water with a value of zero to indicate mean sea level; in these areas, `ETOPO1` provides bathymetry (as well as in regions which are not covered by `SRTM` and `GMTED`). - -**Data sources per zoom:** - -zoom | ocean | land ------- | -------- | ------- -**0** | `ETOPO1` | `ETOPO1` -**1** | `ETOPO1` | `ETOPO1` -**2** | `ETOPO1` | `ETOPO1` -**3** | `ETOPO1` | `ETOPO1` -**4** | `ETOPO1` | `GMTED` -**5** | `ETOPO1` | `GMTED` -**6** | `ETOPO1` | `GMTED` -**7** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60° -**8** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60° -**9** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, `EUDEM` in Europe, with `GMTED` in high latitudes above 60° -**10** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway -**11** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway -**12** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway -**13** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway -**14** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway -**15** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway - -## Sources native resolution - -You might be wondering why we source from different data sets at different zooms. Besides bandwidth reasons, it's helpful to know the native resolution of each data set which is expressed as a nominal arc resolution which maps roughly to a web map zoom for "best displayed at". - -In more practical terms, this results in some datasets being **"oversampled"** for a given zoom and map location. - -* In the water, most bathymetry values at zoom 15 for a pixel that has a ground resolution of 5 meter will actually be showing an oversampled zoom 6 `ETOPO1` value (nominally 2.5 km). - -* On the land, most elevation values at zoom 15 for a pixel that has a ground resolution of 5 meter will actually be showing an oversampled zoom 12 `SRTM` value (nominally 30 meters). - -This formula generates the following table: - -`ground_resolution = (cos(latitude * pi/180) * 2 * pi * 6378137 meters) / (256 * 2^zoom_level pixels) / 30.87 meters per arc second` - -Arc resolution per **zoom** and data sources, per pixel: - -zoom | meters at equator | arc seconds | nominal arc degrees minutes seconds | source | nominal ground units ------ | ---------- | -------- | ------------------- | -------- | -------------------- -**0** | _156543.0_ | `5071.0` | **1.5 arc degrees** | | -**1** | _78271.5_ | `2535.5` | **40 arc minutes** | | -**2** | _39135.8_ | `1267.8` | **20 arc minutes** | | -**3** | _19567.9_ | `633.9` | **10 arc minutes** | | -**4** | _9783.9_ | `316.9` | **5 arc minutes** | | -**5** | _4892.0_ | `158.5` | **2.5 arc minutes** | | -**6** | _2446.0_ | `79.2` | **1 arc minutes** | `ETOPO1` | 2.5 km -**7** | _1223.0_ | `39.6` | **30 arc seconds** | `GMTED2010` | 1km (not used) -**8** | _611.5_ | `19.8` | **15 arc seconds** | `GMTED2010` | 500m (not used) -**9** | _305.7_ | `9.9` | **7.5 arc seconds** | `GMTED2010` | 250m -**10** | _152.9_ | `5.0` | **5 arc seconds** | | -**11** | _76.4_ | `2.5` | **3 arc seconds** | Canada | 90m -**12** | _38.2_ | `1.2` | **1 arc seconds** | `SRTM`, Canada | 30m -**13** | _19.1_ | `0.6` | **2/3 arc seconds** | | -**14** | _9.6_ | `0.3` | **1/3 arc seconds** | `3DEP`, Austria, Australia, New Zealand, Norway | 10m -**15** | _4.8_ | `0.2` | **1/5 arc seconds** | Mexico, Arctic | -**16** | _2.4_ | `0.1` | **1/9 arc seconds** | `3DEP`, United Kingdom | 3m - -## Data updates - -Terrain tiles version 1 was built during 2016Q2 and released in 2016Q3 based on available sources at that time. Version 1.1 was built during 2017Q3 and released 2017Q4. Regular updates are not planned. - -Future updates will be on an as-needed basis for smaller regions to incorporate additional `3DEP` coverage in the United States and additional country specific data sources globally. - -We are always looking for better datasets. If you find a data issue or can suggest an open terrain datasets, please let us know by filing an issue in [tilezen/joerd](https://github.com/tilezen/joerd/issues/new). - -## Known issues - -Many classical DEM and LIDAR related issues occur in terrain tiles. It is not uncommon to see large variations in elevation in areas with large buildings and other such structures. Resampling and merging artifacts are also observed along coastlines (where different datasets are seamed together). +# Data sources + +Mapzen Terrain Tiles are powered by several major open data sets and we owe a tremendous debt of gratitude to the individuals and communities which produced them. + +**Attribution is required** for some data providers. See the [Attribution](https://github.com/tilezen/joerd/blob/master/docs/attribution.md) document for more information. + +## List of sources + +The underlying data sources are a mix of: + +- [3DEP](http://nationalmap.gov/elevation.html) (formerly NED and NED Topobathy) in the United States, 10 meters outside of Alaska, 3 meter in select land and territorial water areas +- [ArcticDEM](http://nga.maps.arcgis.com/apps/MapSeries/index.html?appid=cf2fba21df7540fb981f8836f2a97e25) strips of 5 meter mosaics across all of the land north of 60° latitude, including Alaska, Canada, Greenland, Iceland, Norway, Russia, and Sweden +- [CDEM](http://geogratis.gc.ca/api/en/nrcan-rncan/ess-sst/c40acfba-c722-4be1-862e-146b80be738e.html) (Canadian Digital Elevation Model) in Canada, with variable spatial resolution (from 20-400 meters) depending on the latitude. +- [data.gov.uk](http://environment.data.gov.uk/ds/survey/index.jsp#/survey), 2 meters over most of the United Kingdom +- [data.gv.at](https://www.data.gv.at/katalog/dataset/b5de6975-417b-4320-afdb-eb2a9e2a1dbf), 10 meters over Austria +- [ETOPO1](https://www.ngdc.noaa.gov/mgg/global/global.html) for ocean bathymetry, 1 arc-minute resolution globally +- [EUDEM](https://www.eea.europa.eu/data-and-maps/data/eu-dem#tab-original-data) in most of Europe at 30 meter resolution, including Albania, Austria, Belgium, Bosnia and Herzegovina, Bulgaria, Croatia, Cyprus, Czechia, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Iceland, Ireland, Italy, Kosovo, Latvia, Liechtenstein, Lithuania, Luxembourg, Macedonia, Malta, Montenegro, Netherlands, Norway, Poland, Portugal, Romania, Serbia, Slovakia, Slovenia, Spain, Sweden, Switzerland, and United Kingdom +- Geoscience Australia's [DEM of Australia](https://ecat.ga.gov.au/geonetwork/srv/eng/search#!22be4b55-2465-4320-e053-10a3070a5236), 5 meters around coastal regions in South Australia, Victoria, and Northern Territory +- [GMTED](http://topotools.cr.usgs.gov/gmted_viewer/) globally, coarser resolutions at 7.5", 15", and 30" in land areas +- [INEGI](http://en.www.inegi.org.mx/temas/mapas/relieve/continental/)'s continental relief in Mexico +- [Kartverket](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33)'s Digital Terrain Model, 10 meters over Norway +- [LINZ](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/), 8 meters over New Zealand +- [SRTM](https://lta.cr.usgs.gov/SRTM) globally except high latitudes, 30 meters (90 meters nominal quality) in land areas +- [OTR](http://gis.arso.gov.si/evode/profile.aspx?id=atlas_voda_Lidar%40Arso&initialExtent=499500.5%2C109841.5%2C264.58333) derived from pre-processed Lidar, which contains only the terrain at 1m resolution, 1200m-1400m over Slovenia + +### Footprints database + +These source images are composited to form tiles that make up the Mapzen Terrain Tiles service. To determine exactly which images contributed to Mapzen Terrain Tiles in a particular area, you can download the footprints database and use it with a GIS program like [QGIS](http://www.qgis.org/) to inspect which of these sources were used. + +![Preview Rendering of Footprints](images/footprints-preview.png) + +* [GeoJSON](https://s3.amazonaws.com/elevation-tiles-prod/docs/footprints.geojson.gz) (8.7MB, gzipped) +* [PostgreSQL Dump](https://s3.amazonaws.com/elevation-tiles-prod/docs/footprints.pgdump.gz) (14.5MB, gzipped) + +### Source headers + +To further assist in determining which sources contributed to an individual tile, the Mapzen Terrain Tiles service will respond with an [HTTP header](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Response_fields) listing the sources that contributed to that tile. The value of the `X-Imagery-Sources` HTTP header is a comma-separated list, where each entry follows the pattern `source/filename.tif`. + +For example, a tile might have the header `X-Imagery-Sources: srtm/N39W110.tif, srtm/N39W112.tif, gmted/30N120W_20101117_gmted_mea075.tif`, meaning that it was built from three source images. Two SRTM images and a GMTED image were composited together to generate the tile output. Using the footprint database dumps above you can gather more information about these source images, including the calculated resolution and footprint geometry. To find the entry in the database, look for an entry that has a matching `filename` attribute. + + +## What is the ground resolution? + +Ground resolution per tile pixel varies per zoom level, the given pixel cell's latitude, and input data source. + +This formula generates the following table: + +`ground_resolution = (cos(latitude * pi/180) * 2 * pi * 6378137 meters) / (256 * 2^zoom_level pixels)` + +Ground resolution per **zoom** in `meters` at a given _latitude_: + +zoom | _0°_ | _45°_ | _60°_ +------ | ---------- | ---------- | --------- +**0** | `156543.0` | `110692.6` | `78271.5` +**1** | `78271.5` | `55346.3` | `39135.8` +**2** | `39135.8` | `27673.2` | `19567.9` +**3** | `19567.9` | `13836.6` | `9783.9` +**4** | `9783.9` | `6918.3` | `4892.0` +**5** | `4892.0` | `3459.1` | `2446.0` +**6** | `2446.0` | `1729.6` | `1223.0` +**7** | `1223.0` | `864.8` | `611.5` +**8** | `611.5` | `432.4` | `305.7` +**9** | `305.7` | `216.2` | `152.9` +**10** | `152.9` | `108.1` | `76.4` +**11** | `76.4` | `54.0` | `38.2` +**12** | `38.2` | `27.0` | `19.1` +**13** | `19.1` | `13.5` | `9.6` +**14** | `9.6` | `6.8` | `4.8` +**15** | `4.8` | `3.4` | `2.4` + +**Note:** Esri has [documentation](https://blogs.esri.com/esri/arcgis/2009/03/19/how-can-you-tell-what-map-scales-are-shown-for-online-maps/) about converting web map zoom integers to conventional map scales. + +## What is sourced at what zooms? + +Generally speaking, **GMTED** is used at low-zooms, **ETOPO1** is used to show ocean bathymetry at all zooms (even bathymetry oversampled at zoom 15), and **[SRTM](http://www2.jpl.nasa.gov/srtm/)** is relied on in mid- and high-zooms on land. Some countries have higher resolution data available over land sourced from other open datasets. More information about these sources is available below. + +It should be noted that both `SRTM` and `GMTED` fill oceans and other bodies of water with a value of zero to indicate mean sea level; in these areas, `ETOPO1` provides bathymetry (as well as in regions which are not covered by `SRTM` and `GMTED`). + +**Data sources per zoom:** + +zoom | ocean | land +------ | -------- | ------- +**0** | `ETOPO1` | `ETOPO1` +**1** | `ETOPO1` | `ETOPO1` +**2** | `ETOPO1` | `ETOPO1` +**3** | `ETOPO1` | `ETOPO1` +**4** | `ETOPO1` | `GMTED` +**5** | `ETOPO1` | `GMTED` +**6** | `ETOPO1` | `GMTED` +**7** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60° +**8** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60° +**9** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, `EUDEM` in Europe, with `GMTED` in high latitudes above 60° +**10** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway +**11** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway +**12** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway +**13** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway +**14** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway +**15** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway + +## Sources native resolution + +You might be wondering why we source from different data sets at different zooms. Besides bandwidth reasons, it's helpful to know the native resolution of each data set which is expressed as a nominal arc resolution which maps roughly to a web map zoom for "best displayed at". + +In more practical terms, this results in some datasets being **"oversampled"** for a given zoom and map location. + +* In the water, most bathymetry values at zoom 15 for a pixel that has a ground resolution of 5 meter will actually be showing an oversampled zoom 6 `ETOPO1` value (nominally 2.5 km). + +* On the land, most elevation values at zoom 15 for a pixel that has a ground resolution of 5 meter will actually be showing an oversampled zoom 12 `SRTM` value (nominally 30 meters). + +This formula generates the following table: + +`ground_resolution = (cos(latitude * pi/180) * 2 * pi * 6378137 meters) / (256 * 2^zoom_level pixels) / 30.87 meters per arc second` + +Arc resolution per **zoom** and data sources, per pixel: + +zoom | meters at equator | arc seconds | nominal arc degrees minutes seconds | source | nominal ground units +----- | ---------- | -------- | ------------------- | -------- | -------------------- +**0** | _156543.0_ | `5071.0` | **1.5 arc degrees** | | +**1** | _78271.5_ | `2535.5` | **40 arc minutes** | | +**2** | _39135.8_ | `1267.8` | **20 arc minutes** | | +**3** | _19567.9_ | `633.9` | **10 arc minutes** | | +**4** | _9783.9_ | `316.9` | **5 arc minutes** | | +**5** | _4892.0_ | `158.5` | **2.5 arc minutes** | | +**6** | _2446.0_ | `79.2` | **1 arc minutes** | `ETOPO1` | 2.5 km +**7** | _1223.0_ | `39.6` | **30 arc seconds** | `GMTED2010` | 1km (not used) +**8** | _611.5_ | `19.8` | **15 arc seconds** | `GMTED2010` | 500m (not used) +**9** | _305.7_ | `9.9` | **7.5 arc seconds** | `GMTED2010` | 250m +**10** | _152.9_ | `5.0` | **5 arc seconds** | | +**11** | _76.4_ | `2.5` | **3 arc seconds** | Canada | 90m +**12** | _38.2_ | `1.2` | **1 arc seconds** | `SRTM`, Canada | 30m +**13** | _19.1_ | `0.6` | **2/3 arc seconds** | | +**14** | _9.6_ | `0.3` | **1/3 arc seconds** | `3DEP`, Austria, Australia, New Zealand, Norway | 10m +**15** | _4.8_ | `0.2` | **1/5 arc seconds** | Mexico, Arctic | +**16** | _2.4_ | `0.1` | **1/9 arc seconds** | `3DEP`, United Kingdom | 3m + +## Data updates + +Terrain tiles version 1 was built during 2016Q2 and released in 2016Q3 based on available sources at that time. Version 1.1 was built during 2017Q3 and released 2017Q4. Regular updates are not planned. + +Future updates will be on an as-needed basis for smaller regions to incorporate additional `3DEP` coverage in the United States and additional country specific data sources globally. + +We are always looking for better datasets. If you find a data issue or can suggest an open terrain datasets, please let us know by filing an issue in [tilezen/joerd](https://github.com/tilezen/joerd/issues/new). + +## Known issues + +Many classical DEM and LIDAR related issues occur in terrain tiles. It is not uncommon to see large variations in elevation in areas with large buildings and other such structures. Resampling and merging artifacts are also observed along coastlines (where different datasets are seamed together). diff --git a/joerd/source/otr.py b/joerd/source/otr.py new file mode 100644 index 0000000..53844b9 --- /dev/null +++ b/joerd/source/otr.py @@ -0,0 +1,260 @@ +from bs4 import BeautifulSoup +from joerd.util import BoundingBox +import joerd.download as download +import joerd.check as check +import joerd.srs as srs +import joerd.index as index +import joerd.mask as mask +import joerd.tmpdir as tmpdir +from joerd.mkdir_p import mkdir_p +from contextlib2 import closing, ExitStack +from shutil import copyfile, move +import os.path +import os +import io +import requests +import logging +import re +import tempfile +import sys +import zipfile +import traceback +import subprocess +import glob +from osgeo import gdal +from osgeo import ogr +from osgeo import osr +import pdal +import yaml +import time +import json +import subprocess + +class OTRTile(object): + def __init__(self, parent, link, name, block, bbox): + self.uri = parent.uri + self.download_options = parent.download_options + self.base_dir = parent.base_dir + self.name = name + self.block = block + self.link = link + self.bbox = bbox + + def __key(self): + return self.name + + def __eq__(a, b): + return isinstance(b, type(a)) and \ + a.__key() == b.__key() + + def __hash__(self): + return hash(self.__key()) + + def urls(self): + uri_list = [self.uri + "/" + self.link] + return uri_list + + + def verifier(self): + return is_las + + def options(self): + return self.download_options + + def output_file(self): + file_name = "TMR_" + self.name + ".tif" + return os.path.join(self.base_dir, file_name) + + def _tif_file(self): + # returns the name of the geotiff file within the distributed archive. + return "TMR_%(name)s.tif" % dict(name=self.name) + + def unpack(self, store, las_file): + tif_file = self._tif_file; + + with store.upload_dir() as target: + target_dir = os.path.join(target, self.base_dir) + mkdir_p(target_dir) + with tmpdir.tmpdir() as temp_dir: + + arr = [] + inp = { + "type": "readers.las", + "filename": "%(fname)s" % dict(fname=las_file.name), + "spatialreference":"EPSG:3794" + } + arr.append(inp) + par = { + "type": "writers.gdal", + "resolution": 1, + "radius": 7, + "filename": os.path.join(temp_dir, "TMR_%(name)s.tif" % dict(name=self.name)) + } + arr.append(par) + + astage = { + "pipeline" : arr + } + + + #j = json.dumps(astage) + #u = unicode(j, "utf-8") + + json_path = os.path.join(temp_dir, "TMR_%(name)s.json" % dict(name=self.name)) + + with open(json_path, 'w') as json_file: + json.dump(astage, json_file) + + #covert lidar to geotiff + #pipeline = pdal.Pipeline(u) + #count = pipeline.execute() + + subprocess.check_output('pdal pipeline {jfile}'.format(jfile=json_path), cwd=temp_dir, shell=True) + + output_file = os.path.join(target, self.output_file()) + mask.negative(os.path.join(temp_dir, "TMR_%(name)s.tif" % dict(name=self.name)), "GTiff", output_file) + + + def freeze_dry(self): + return dict(type='otr', link=self.link) + +IS_OTR_FILE = re.compile( + '^b_([0-9]{2})/D96TM/TMR_([0-9]{2,3})_([0-9]{2,3}).laz') + +def is_las(self): + def func(tmp): + if tmp.name.endswith(".laz"): + return True + else: + return False + return func + + +def _parse_otr_tile(link, parent): + m = IS_OTR_FILE.match(link) + + if not m: + return None + + d96_bottom = int(m.group(3)) * 1000 + d96_left = int(m.group(2)) * 1000 + d96_top = (int(m.group(3)) + 1) * 1000 + d96_right = (int(m.group(2)) + 1) * 1000 + + block = int(m.group(1)) + name = m.group(2) + "_" + m.group(3) + + #D96/TM (EPSG::3794) to WGS84 (EPSG::4326) + src = osr.SpatialReference() + tgt = osr.SpatialReference() + src.ImportFromEPSG(3794) + tgt.ImportFromEPSG(4326) + + transform = osr.CoordinateTransformation(src, tgt) + coords = transform.TransformPoint(d96_left, d96_bottom) + left,bottom = coords[0:2] + coords = transform.TransformPoint(d96_right, d96_top) + right,top= coords[0:2] + + bbox = BoundingBox(left, bottom, right, top) + + return OTRTile(parent, link, name, block, bbox) + +class OTR(object): + def __init__(self, options={}): + self.base_dir = options.get('base_dir', 'otr') + self.uri = options['uri'] + self.fishnet_url = options['fishnet_url'] + self.download_options = options + self.tile_index = None + + def get_index(self): + if not os.path.isdir(self.base_dir): + os.makedirs(self.base_dir) + index_name = 'index.yaml' + index_file = os.path.join(self.base_dir, index_name) + # if index doesn't exist, or is more than 24h old + if not os.path.isfile(index_file) or \ + time.time() > os.path.getmtime(index_file) + 86400: + self.download_index(index_file) + + def download_index(self, index_file): + logger = logging.getLogger('otr') + logger.info('Fetcthing D96TM index...') + links = [] + + fishnet = 'LIDAR_FISHNET_D96.shp' + + req = requests.get(self.fishnet_url, stream=True) + with tmpdir.tmpdir() as d: + with zipfile.ZipFile(io.BytesIO(req.content)) as zip_file: + zip_file.extractall(d) + + fishnet_file = os.path.join(d, fishnet) + driver = ogr.GetDriverByName("ESRI Shapefile") + dataSource = driver.Open(fishnet_file, 1) + layer = dataSource.GetLayer() + + for feature in layer: + links.append(feature.GetField("BLOK") + "/D96TM/TMR_" + feature.GetField("NAME") + ".laz") + layer.ResetReading() + + with open(index_file, 'w') as file: + file.write(yaml.dump(links)) + + def _ensure_tile_index(self): + if self.tile_index is None: + index_file = os.path.join(self.base_dir, 'index.yaml') + bbox = (13.39027778,45.40750000,16.62694444,46.88333333) + self.tile_index = index.create(index_file, bbox, _parse_otr_tile, self) + + return self.tile_index + + def existing_files(self): + for base, dirs, files in os.walk(self.base_dir): + for f in files: + if f.endswith('tif'): + yield os.path.join(base, f) + + def rehydrate(self, data): + assert data.get('type') == 'otr', \ + "Unable to rehydrate %r from Slovenia." %data + return _parse_otr_tile(data['link'], self) + + def downloads_for(self, tile): + tiles = set() + # if the tile scale is greater than 20x the D96TM scale, then there's no + # point in including D96TM, it'll be far too fine to make a difference. + # D96TM is 1m. + + if tile.max_resolution() > 20: + return tiles + + # buffer by 0.0075 degrees (81px) to grab neighbouring tiles and ensure + # some overlap to take care of boundary issues. + tile_bbox = tile.latlon_bbox().buffer(0.0075) + + tile_index = self._ensure_tile_index() + + for t in index.intersections(tile_index, tile_bbox): + tiles.add(t) + + return tiles + + def filter_type(self, src_res, dst_res): + return gdal.GRA_Lanczos if src_res > dst_res else gdal.GRA_Cubic + + def vrts_for(self, tile): + """ + Returns a list of sets of tiles, with each list element intended as a + separate VRT for use in GDAL. + + D96TM is non-overlapping. + """ + return [self.downloads_for(tile)] + + def srs(self): + return srs.d96() + +def create(options): + return OTR(options) \ No newline at end of file From e77e98c3475f7a948538d427a9aff68cafed6598 Mon Sep 17 00:00:00 2001 From: DavixDevelop Date: Fri, 30 Oct 2020 22:28:07 +0100 Subject: [PATCH 06/21] Renamed d96tm to otr --- joerd/source/d96tm.py | 261 ------------------------------------------ 1 file changed, 261 deletions(-) delete mode 100644 joerd/source/d96tm.py diff --git a/joerd/source/d96tm.py b/joerd/source/d96tm.py deleted file mode 100644 index 960c542..0000000 --- a/joerd/source/d96tm.py +++ /dev/null @@ -1,261 +0,0 @@ -from bs4 import BeautifulSoup -from joerd.util import BoundingBox -import joerd.download as download -import joerd.check as check -import joerd.srs as srs -import joerd.index as index -import joerd.mask as mask -import joerd.tmpdir as tmpdir -from joerd.mkdir_p import mkdir_p -from contextlib2 import closing, ExitStack -from shutil import copyfile, move -import os.path -import os -import io -import requests -import logging -import re -import tempfile -import sys -import zipfile -import traceback -import subprocess -import glob -from osgeo import gdal -from osgeo import ogr -from osgeo import osr -import pdal -import yaml -import time -import json -import subprocess - -class D96TMTile(object): - def __init__(self, parent, link, name, block, bbox): - self.uri = parent.uri - self.download_options = parent.download_options - self.base_dir = parent.base_dir - self.name = name - self.block = block - self.link = link - self.bbox = bbox - - def __key(self): - return self.name - - def __eq__(a, b): - return isinstance(b, type(a)) and \ - a.__key() == b.__key() - - def __hash__(self): - return hash(self.__key()) - - def urls(self): - uri_list = [self.uri + "/" + self.link] - return uri_list - - - def verifier(self): - return is_las - - def options(self): - return self.download_options - - def output_file(self): - file_name = "TMR_" + self.name + ".tif" - return os.path.join(self.base_dir, file_name) - - def _tif_file(self): - # returns the name of the geotiff file within the distributed archive. - return "TMR_%(name)s.tif" % dict(name=self.name) - - def unpack(self, store, las_file): - tif_file = self._tif_file; - - with store.upload_dir() as target: - target_dir = os.path.join(target, self.base_dir) - mkdir_p(target_dir) - with tmpdir.tmpdir() as temp_dir: - - arr = [] - inp = { - "type": "readers.las", - "filename": "%(fname)s" % dict(fname=las_file.name), - "spatialreference":"EPSG:3794" - } - arr.append(inp) - par = { - "type": "writers.gdal", - "resolution": 1, - "radius": 7, - "filename": os.path.join(temp_dir, "TMR_%(name)s.tif" % dict(name=self.name)) - } - arr.append(par) - - astage = { - "pipeline" : arr - } - - - #j = json.dumps(astage) - #u = unicode(j, "utf-8") - - json_path = os.path.join(temp_dir, "TMR_%(name)s.json" % dict(name=self.name)) - - with open(json_path, 'w') as json_file: - json.dump(astage, json_file) - - #covert lidar to geotiff - #pipeline = pdal.Pipeline(u) - #count = pipeline.execute() - - subprocess.check_output('pdal pipeline {jfile}'.format(jfile=json_path), cwd=temp_dir, shell=True) - - output_file = os.path.join(target, self.output_file()) - mask.negative(os.path.join(temp_dir, "TMR_%(name)s.tif" % dict(name=self.name)), "GTiff", output_file) - - - def freeze_dry(self): - return dict(type='d96tm', link=self.link) - -IS_D96TM_FILE = re.compile( - '^b_([0-9]{2})/D96TM/TMR_([0-9]{2,3})_([0-9]{2,3}).laz') - -def is_las(self): - def func(tmp): - if tmp.name.endswith(".laz"): - return True - else: - return False - return func - - -def _parse_d96tm_tile(link, parent): - m = IS_D96TM_FILE.match(link) - - if not m: - return None - - d96_bottom = int(m.group(3)) * 1000 - d96_left = int(m.group(2)) * 1000 - d96_top = (int(m.group(3)) + 1) * 1000 - d96_right = (int(m.group(2)) + 1) * 1000 - - block = int(m.group(1)) - name = m.group(2) + "_" + m.group(3) - - #D96 (EPSG::3794) to WGS84 (EPSG::4326) - src = osr.SpatialReference() - tgt = osr.SpatialReference() - src.ImportFromEPSG(3794) - tgt.ImportFromEPSG(4326) - - transform = osr.CoordinateTransformation(src, tgt) - coords = transform.TransformPoint(d96_left, d96_bottom) - left,bottom = coords[0:2] - coords = transform.TransformPoint(d96_right, d96_top) - right,top= coords[0:2] - - bbox = BoundingBox(left, bottom, right, top) - - return D96TMTile(parent, link, name, block, bbox) - -class D96TM(object): - def __init__(self, options={}): - self.base_dir = options.get('base_dir', 'd96tm') - self.uri = options['uri'] - self.fishnet_url = options['fishnet_url'] - self.download_options = options - self.tile_index = None - - def get_index(self): - if not os.path.isdir(self.base_dir): - os.makedirs(self.base_dir) - index_name = 'index.yaml' - index_file = os.path.join(self.base_dir, index_name) - # if index doesn't exist, or is more than 24h old - if not os.path.isfile(index_file) or \ - time.time() > os.path.getmtime(index_file) + 86400: - self.download_index(index_file) - - def download_index(self, index_file): - logger = logging.getLogger('d96tm') - logger.info('Fetcthing D96TM index...') - links = [] - - fishnet = 'LIDAR_FISHNET_D96.shp' - - req = requests.get(self.fishnet_url, stream=True) - with tmpdir.tmpdir() as d: - with zipfile.ZipFile(io.BytesIO(req.content)) as zip_file: - zip_file.extractall(d) - - fishnet_file = os.path.join(d, fishnet) - driver = ogr.GetDriverByName("ESRI Shapefile") - dataSource = driver.Open(fishnet_file, 1) - layer = dataSource.GetLayer() - - for feature in layer: - links.append(feature.GetField("BLOK") + "/D96TM/TMR_" + feature.GetField("NAME") + ".laz") - layer.ResetReading() - - with open(index_file, 'w') as file: - file.write(yaml.dump(links)) - - def _ensure_tile_index(self): - if self.tile_index is None: - index_file = os.path.join(self.base_dir, 'index.yaml') - #this cords are only for the test data and need to be changed for which region you want to download - bbox = (15.67583333,46.38861111,15.74166667,46.43305556) - self.tile_index = index.create(index_file, bbox, _parse_d96tm_tile, self) - - return self.tile_index - - def existing_files(self): - for base, dirs, files in os.walk(self.base_dir): - for f in files: - if f.endswith('tif'): - yield os.path.join(base, f) - - def rehydrate(self, data): - assert data.get('type') == 'd96tm', \ - "Unable to rehydrate %r from Slovenia." %data - return _parse_d96tm_tile(data['link'], self) - - def downloads_for(self, tile): - tiles = set() - # if the tile scale is greater than 20x the D96TM scale, then there's no - # point in including D96TM, it'll be far too fine to make a difference. - # D96TM is 1m. - - if tile.max_resolution() > 20: - return tiles - - # buffer by 0.0075 degrees (81px) to grab neighbouring tiles and ensure - # some overlap to take care of boundary issues. - tile_bbox = tile.latlon_bbox().buffer(0.0075) - - tile_index = self._ensure_tile_index() - - for t in index.intersections(tile_index, tile_bbox): - tiles.add(t) - - return tiles - - def filter_type(self, src_res, dst_res): - return gdal.GRA_Lanczos if src_res > dst_res else gdal.GRA_Cubic - - def vrts_for(self, tile): - """ - Returns a list of sets of tiles, with each list element intended as a - separate VRT for use in GDAL. - - D96TM is non-overlapping. - """ - return [self.downloads_for(tile)] - - def srs(self): - return srs.d96() - -def create(options): - return D96TM(options) \ No newline at end of file From 4c0e18e1acbdb50b208863f778adde2eea6fa34b Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Sat, 31 Oct 2020 00:36:26 +0100 Subject: [PATCH 07/21] Fixed indentation --- joerd/source/otr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/joerd/source/otr.py b/joerd/source/otr.py index 53844b9..ac73a80 100644 --- a/joerd/source/otr.py +++ b/joerd/source/otr.py @@ -231,8 +231,8 @@ def downloads_for(self, tile): return tiles # buffer by 0.0075 degrees (81px) to grab neighbouring tiles and ensure - # some overlap to take care of boundary issues. - tile_bbox = tile.latlon_bbox().buffer(0.0075) + # some overlap to take care of boundary issues. + tile_bbox = tile.latlon_bbox().buffer(0.0075) tile_index = self._ensure_tile_index() From 81e0b62fa718ff85d4967f8402d19e3c16401fa1 Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Sat, 31 Oct 2020 19:22:11 +0100 Subject: [PATCH 08/21] Removed otr source and replaced with otr --- .gitignore | 2 +- config.example.yaml | 4 +- config.slovenia.yaml | 4 +- config.slovenia_test.yaml | 14 +-- docs/attribution.md | 14 ++- docs/data-sources.md | 14 +-- joerd/index.py | 5 +- joerd/source/dmr1.py | 244 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 277 insertions(+), 24 deletions(-) create mode 100644 joerd/source/dmr1.py diff --git a/.gitignore b/.gitignore index 77440f6..05db451 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ gmted/ etopo1/ ned/ ned_topobathy/ -otr/ +dmr1/ # tile generation directories tiles/ diff --git a/config.example.yaml b/config.example.yaml index 9f08f84..55ddabd 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -43,8 +43,8 @@ sources: - type: ned_topobathy ftp_server: rockyftp.cr.usgs.gov base_path: vdelivery/Datasets/Staged/NED/19/IMG - - type: otr - uri: http://gis.arso.gov.si/lidar/otr/laz + - type: dmr1 + uri: http://gis.arso.gov.si/lidar/dmr1 fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip logging: config: logging.example.config diff --git a/config.slovenia.yaml b/config.slovenia.yaml index ca89421..3f8d7bf 100644 --- a/config.slovenia.yaml +++ b/config.slovenia.yaml @@ -13,7 +13,7 @@ regions: left: 13.39027778 bottom: 45.40750000 right: 16.62694444 - zoom_range: [0, 15] + zoom_range: [10, 15] outputs: - type: skadi - type: tiff @@ -26,7 +26,7 @@ sources: username: 'earthdata username' password: 'earthdata password' auth-url: https://urs.earthdata.nasa.gov - - type: otr + - type: dmr1 uri: http://gis.arso.gov.si/lidar/otr/laz fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip logging: diff --git a/config.slovenia_test.yaml b/config.slovenia_test.yaml index 539efd6..2651c30 100644 --- a/config.slovenia_test.yaml +++ b/config.slovenia_test.yaml @@ -9,11 +9,11 @@ regions: # zoom_range: [0, 16] slovenia-test: bbox: - top: 46.43305556 - left: 15.67583333 - bottom: 46.38861111 - right: 15.74166667 - zoom_range: [14, 15] + top: 46.239346 + left: 15.237007 + bottom: 46.205735 + right: 15.288334 + zoom_range: [10, 15] outputs: - type: skadi - type: tiff @@ -26,8 +26,8 @@ sources: username: 'earthdata username' password: 'earthdata password' auth-url: https://urs.earthdata.nasa.gov - - type: otr - uri: http://gis.arso.gov.si/lidar/otr/laz + - type: dmr1 + uri: http://gis.arso.gov.si/lidar/dmr1 fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip logging: config: logging.example.config diff --git a/docs/attribution.md b/docs/attribution.md index f00ebf7..adf8fde 100644 --- a/docs/attribution.md +++ b/docs/attribution.md @@ -201,11 +201,17 @@ Attribution statement: > © Commonwealth of Australia (Geoscience Australia) 2017. -### Digital Elevation Model (DEM) of Slovenia derived from LiDAR 1 Meter Grid +### Digital Elevation Model (DEM) of Slovenia derived from LiDAR 1 Meter Grid (DMR1) -[Released](https://gis.arso.gov.si/related/lidar_porocila/b_25_izdelava_izdelkov.pdf) by Slovenia's [GURS](https://www.e-prostor.gov.si/), licensed under [Creative Commons 4.0](https://creativecommons.org/licenses/by/4.0/deed.sl) +[Released](http://www.evode.gov.si/index.php?id=69) by Slovenia's [GURS](https://www.e-prostor.gov.si/), licensed under [Creative Commons 4.0](https://creativecommons.org/licenses/by/4.0/deed.sl) + +> Use Limitations: +> General conditions for the use of geodetic data: https://www.e-prostor.gov.si/dostop-do-podatkov/dostop-do-podatkov/#tab1-1029 +> +> Public access Restrictions: +> No restrictions Attribution statement: -> Geodetska uprava Republike Slovenije, LiDAR, OTR 2015 -> © 2017 MOP - Geodetska uprava Republike Slovenije - Vse pravice pridržane. +> Geodetska uprava Republike Slovenije, LiDAR, DMR1 2015 +> © 2017 MOP - Geodetska uprava Republike Slovenije - Vse pravice pridržane. \ No newline at end of file diff --git a/docs/data-sources.md b/docs/data-sources.md index bd71bfe..66142b0 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -21,7 +21,7 @@ The underlying data sources are a mix of: - [Kartverket](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33)'s Digital Terrain Model, 10 meters over Norway - [LINZ](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/), 8 meters over New Zealand - [SRTM](https://lta.cr.usgs.gov/SRTM) globally except high latitudes, 30 meters (90 meters nominal quality) in land areas -- [OTR](http://gis.arso.gov.si/evode/profile.aspx?id=atlas_voda_Lidar%40Arso&initialExtent=499500.5%2C109841.5%2C264.58333) derived from pre-processed Lidar, which contains only the terrain at 1m resolution, 1200m-1400m over Slovenia +- [DMR1](http://gis.arso.gov.si/evode/profile.aspx?id=atlas_voda_Lidar%40Arso&initialExtent=499500.5%2C109841.5%2C264.58333) derived from pre-processed Lidar, which contains only the terrain at 1m resolution, 1200m-1400m over Slovenia ### Footprints database @@ -90,12 +90,12 @@ zoom | ocean | land **7** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60° **8** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60° **9** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, `EUDEM` in Europe, with `GMTED` in high latitudes above 60° -**10** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway -**11** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway -**12** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway -**13** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway -**14** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway -**15** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway +**10** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia +**11** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia +**12** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia +**13** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia +**14** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia +**15** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia ## Sources native resolution diff --git a/joerd/index.py b/joerd/index.py index c247c3f..7b8f208 100644 --- a/joerd/index.py +++ b/joerd/index.py @@ -1,9 +1,11 @@ +from __future__ import print_function import pyqtree import yaml import logging import thread + # Create an index given a YAML file consisting of a list of strings and a # function to parse it. Extra, fixed arguments for the function can be also # be given. Each object returned from the `parse_fn` should have a member @@ -20,6 +22,7 @@ def create(index_file, bbox, parse_fn, *parse_args): if t: idx.insert(bbox=t.bbox.bounds, item=t) n += 1 + print("\r Loaded... %d objects" % n, end="") logger.info("Created index with %d objects." % n) return idx @@ -27,4 +30,4 @@ def create(index_file, bbox, parse_fn, *parse_args): # Returns a list of all objects intersecting the given bbox in the index. def intersections(idx, bbox): - return idx.intersect(bbox.bounds) + return idx.intersect(bbox.bounds) \ No newline at end of file diff --git a/joerd/source/dmr1.py b/joerd/source/dmr1.py new file mode 100644 index 0000000..82d6ec0 --- /dev/null +++ b/joerd/source/dmr1.py @@ -0,0 +1,244 @@ +from bs4 import BeautifulSoup +from joerd.util import BoundingBox +import joerd.download as download +import joerd.check as check +import joerd.srs as srs +import joerd.index as index +import joerd.mask as mask +import joerd.tmpdir as tmpdir +from joerd.mkdir_p import mkdir_p +from contextlib2 import closing, ExitStack +from shutil import copyfile, move +import os.path +import os +import io +import requests +import logging +import re +import tempfile +import sys +import zipfile +import traceback +import subprocess +import glob +from osgeo import gdal +from osgeo import ogr +from osgeo import osr +import pdal +import yaml +import time +import subprocess +import mimetypes + +#this needs to be changed to the region you want to download use +REGION_BBOX = BoundingBox(15.237007,46.205735,15.288334,46.239346) + +class DMR1Tile(object): + def __init__(self, parent, link, name, block, bbox): + self.uri = parent.uri + self.download_options = parent.download_options + self.base_dir = parent.base_dir + self.name = name + self.block = block + self.link = link + self.bbox = bbox + + def __key(self): + return self.name + + def __eq__(a, b): + return isinstance(b, type(a)) and \ + a.__key() == b.__key() + + def __hash__(self): + return hash(self.__key()) + + def urls(self): + uri_list = [self.uri + "/" + self.link] + return uri_list + + + def verifier(self): + return is_txt + + def options(self): + return self.download_options + + def output_file(self): + file_name = "TM1_" + self.name + ".tif" + return os.path.join(self.base_dir, file_name) + + def _tif_file(self): + # returns the name of the geotiff file within the distributed archive. + return "TM1_%(name)s.tif" % dict(name=self.name) + + def unpack(self, store, txt_file): + tif_file = self._tif_file(); + + with store.upload_dir() as target: + target_dir = os.path.join(target, self.base_dir) + mkdir_p(target_dir) + with tmpdir.tmpdir() as temp_dir: + + xyz_file = "TM1_%(name)s.xyz" % dict(name=self.name) + + #Flip x any y, as they are fliped in dmr1 + subprocess.check_output("sort -k2 -n -t';' -k1 {tfile} -o {ofile}".format(tfile=txt_file.name, ofile=xyz_file), cwd=temp_dir, shell=True) + + #Correct NS (North South) resolution and convert to xyz to tif + subprocess.check_output("gdalwarp -t_srs EPSG:3794 -overwrite {xfile} {tfile}".format(xfile=xyz_file,tfile=tif_file), cwd=temp_dir, shell=True) + + output_file = os.path.join(target, self.output_file()) + mask.negative(os.path.join(temp_dir, tif_file), "GTiff", output_file) + + + def freeze_dry(self): + return dict(type='dmr1', link=self.link) + +IS_DMR1_FILE = re.compile( + '^b_([0-9]{2})/D96TM/TM1_([0-9]{2,3})_([0-9]{2,3}).txt') + +def is_txt(self): + def func(tmp): + if mimetypes.guess_type(tmp.name)[0] == 'text/plain': + return True + else: + return False + return func + + +def _parse_dmr1_tile(link, parent): + m = IS_DMR1_FILE.match(link) + + if not m: + return None + + d96_bottom = int(m.group(3)) * 1000 + d96_left = int(m.group(2)) * 1000 + d96_top = (int(m.group(3)) + 1) * 1000 + d96_right = (int(m.group(2)) + 1) * 1000 + + block = int(m.group(1)) + name = m.group(2) + "_" + m.group(3) + + #D96/TM (EPSG::3794) to WGS84 (EPSG::4326) + src = osr.SpatialReference() + tgt = osr.SpatialReference() + src.ImportFromEPSG(3794) + tgt.ImportFromEPSG(4326) + + transform = osr.CoordinateTransformation(src, tgt) + coords = transform.TransformPoint(d96_left, d96_bottom) + left,bottom = coords[0:2] + coords = transform.TransformPoint(d96_right, d96_top) + right,top= coords[0:2] + + bbox = BoundingBox(left, bottom, right, top) + + #This is so the memory doesn't overflow, due to the big size of the index + #and is to be removed on a production server + if bbox.intersects(REGION_BBOX): + return DMR1Tile(parent, link, name, block, bbox) + + return None + +class DMR1(object): + def __init__(self, options={}): + self.base_dir = options.get('base_dir', 'dmr1') + self.uri = options['uri'] + self.fishnet_url = options['fishnet_url'] + self.download_options = options + self.tile_index = None + + def get_index(self): + if not os.path.isdir(self.base_dir): + os.makedirs(self.base_dir) + index_name = 'index.yaml' + index_file = os.path.join(self.base_dir, index_name) + # if index doesn't exist, or is more than 24h old + if not os.path.isfile(index_file) or \ + time.time() > os.path.getmtime(index_file) + 86400: + self.download_index(index_file) + + def download_index(self, index_file): + logger = logging.getLogger('dmr1') + logger.info('Fetcthing D96TM index...') + links = [] + + fishnet = 'LIDAR_FISHNET_D96.shp' + + req = requests.get(self.fishnet_url, stream=True) + with tmpdir.tmpdir() as d: + with zipfile.ZipFile(io.BytesIO(req.content)) as zip_file: + zip_file.extractall(d) + + fishnet_file = os.path.join(d, fishnet) + driver = ogr.GetDriverByName("ESRI Shapefile") + dataSource = driver.Open(fishnet_file, 1) + layer = dataSource.GetLayer() + + for feature in layer: + links.append(feature.GetField("BLOK") + "/D96TM/TM1_" + feature.GetField("NAME") + ".txt") + layer.ResetReading() + + with open(index_file, 'w') as file: + file.write(yaml.dump(links)) + + def _ensure_tile_index(self): + if self.tile_index is None: + index_file = os.path.join(self.base_dir, 'index.yaml') + + #this needs to be changed to the region you want to download use + bbox = (15.237007,46.205735,15.288334,46.239346) + self.tile_index = index.create(index_file, bbox, _parse_dmr1_tile, self) + + return self.tile_index + + def existing_files(self): + for base, dirs, files in os.walk(self.base_dir): + for f in files: + if f.endswith('tif'): + yield os.path.join(base, f) + + def rehydrate(self, data): + assert data.get('type') == 'dmr1', \ + "Unable to rehydrate %r from Slovenia." %data + return _parse_dmr1_tile(data['link'], self) + + def downloads_for(self, tile): + tiles = set() + # if the tile scale is greater than 20x the D96TM scale, then there's no + # point in including D96TM, it'll be far too fine to make a difference. + # D96TM is 1m. + + if tile.max_resolution() > 20: + return tiles + + # buffer by 0.0075 degrees (81px) to grab neighbouring tiles and ensure + # some overlap to take care of boundary issues. + tile_bbox = tile.latlon_bbox().buffer(0.0075) + + tile_index = self._ensure_tile_index() + + for t in index.intersections(tile_index, tile_bbox): + tiles.add(t) + + return tiles + + def filter_type(self, src_res, dst_res): + return gdal.GRA_Lanczos if src_res > dst_res else gdal.GRA_Cubic + + def vrts_for(self, tile): + """ + Returns a list of sets of tiles, with each list element intended as a + separate VRT for use in GDAL. + + D96TM is non-overlapping. + """ + return [self.downloads_for(tile)] + + def srs(self): + return srs.d96() + +def create(options): + return DMR1(options) \ No newline at end of file From 5c93d64ae6293a8d849bcb8caa2bbea99cca73d8 Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Sat, 31 Oct 2020 19:25:47 +0100 Subject: [PATCH 09/21] Updated docs --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 6875f49..6940eb2 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,6 @@ sudo apt-get install python-gdal python-bs4 python-numpy gdal-bin python-setupto (NOTE: not sure if this works: I installed GDAL-2.0.1 manually here, but I don't think it really needs it.) -(NOTE: if you want to use d96tm, you must install pdal, python-pdal, and LASzip or LAZperf) - You can then install it (recommended in a `virtualenv`) by running: ```sh From b2d4273e30b580291f5e194c61ac219341b39236 Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Sat, 31 Oct 2020 19:52:18 +0100 Subject: [PATCH 10/21] Fixed typo --- joerd/source/dmr1.py | 1 - 1 file changed, 1 deletion(-) diff --git a/joerd/source/dmr1.py b/joerd/source/dmr1.py index 82d6ec0..d379d65 100644 --- a/joerd/source/dmr1.py +++ b/joerd/source/dmr1.py @@ -24,7 +24,6 @@ from osgeo import gdal from osgeo import ogr from osgeo import osr -import pdal import yaml import time import subprocess From 4a366e4cc7502049180e656d57a4e2508241e3ee Mon Sep 17 00:00:00 2001 From: DavixDevelop Date: Sat, 31 Oct 2020 20:37:44 +0100 Subject: [PATCH 11/21] Delete config.render.slovenia_test.yaml --- config.render.slovenia_test.yaml | 39 -------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 config.render.slovenia_test.yaml diff --git a/config.render.slovenia_test.yaml b/config.render.slovenia_test.yaml deleted file mode 100644 index 3dfe089..0000000 --- a/config.render.slovenia_test.yaml +++ /dev/null @@ -1,39 +0,0 @@ ---- -regions: - # san-francisco-bay_california: - # bbox: - # top: 38.719 - # left: -123.640 - # bottom: 36.791 - # right: -121.025 - # zoom_range: [0, 16] - slovenia-test: - bbox: - top: 46.43305556 - left: 15.67583333 - bottom: 46.38861111 - right: 15.74166667 - zoom_range: [14, 15] -outputs: - - type: skadi - - type: tiff - - type: terrarium - - type: normal -sources: - - type: srtm - url: https://github.com/DavixDevelop/TerraLidar/raw/master/mapzen_data/tile - mask-url: https://github.com/DavixDevelop/TerraLidar/raw/master/mapzen_data/mask - - type: 'd96tm' - uri: 'http://gis.arso.gov.si/lidar/otr/laz' - fishnet_url: 'https://github.com/DavixDevelop/TerraLidar/raw/master/mapzen_data/lidar_fishnet_D96TM.zip' -logging: - config: logging.example.config -cluster: - queue: - type: fake -store: - type: file - base_dir: 'render' -source_store: - type: file - base_dir: 'source' \ No newline at end of file From 072ba3bd2c2ea096857d8cedff8125acaed9bf81 Mon Sep 17 00:00:00 2001 From: DavixDevelop Date: Sat, 31 Oct 2020 20:37:56 +0100 Subject: [PATCH 12/21] Delete config.download.slovenia.yaml --- config.download.slovenia.yaml | 36 ----------------------------------- 1 file changed, 36 deletions(-) delete mode 100644 config.download.slovenia.yaml diff --git a/config.download.slovenia.yaml b/config.download.slovenia.yaml deleted file mode 100644 index 1eb2e1d..0000000 --- a/config.download.slovenia.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -regions: - # san-francisco-bay_california: - # bbox: - # top: 38.719 - # left: -123.640 - # bottom: 36.791 - # right: -121.025 - # zoom_range: [0, 16] - slovenia: - bbox: - top: 46.88333333 - left: 13.39027778 - bottom: 45.40750000 - right: 16.62694444 - zoom_range: [0, 15] -outputs: - - type: skadi - - type: tiff - - type: terrarium - - type: normal -sources: - - type: d96tm - uri: http://gis.arso.gov.si/lidar/otr/laz - fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip -logging: - config: logging.example.config -cluster: - queue: - type: fake -store: - type: file - base_dir: 'render' -source_store: - type: file - base_dir: 'source' \ No newline at end of file From 6e3ed6c4305ee1086311f6af39f7212e0fec497b Mon Sep 17 00:00:00 2001 From: DavixDevelop Date: Sat, 31 Oct 2020 20:38:06 +0100 Subject: [PATCH 13/21] Delete config.download.slovenia_test.yaml --- config.download.slovenia_test.yaml | 36 ------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 config.download.slovenia_test.yaml diff --git a/config.download.slovenia_test.yaml b/config.download.slovenia_test.yaml deleted file mode 100644 index 55744e1..0000000 --- a/config.download.slovenia_test.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -regions: - # san-francisco-bay_california: - # bbox: - # top: 38.719 - # left: -123.640 - # bottom: 36.791 - # right: -121.025 - # zoom_range: [0, 16] - slovenia-test: - bbox: - top: 46.43305556 - left: 15.67583333 - bottom: 46.38861111 - right: 15.74166667 - zoom_range: [14, 15] -outputs: - - type: skadi - - type: tiff - - type: terrarium - - type: normal -sources: - - type: 'd96tm' - uri: 'http://gis.arso.gov.si/lidar/otr/laz' - fishnet_url: 'https://github.com/DavixDevelop/TerraLidar/raw/master/mapzen_data/lidar_fishnet_D96TM.zip' -logging: - config: logging.example.config -cluster: - queue: - type: fake -store: - type: file - base_dir: 'render' -source_store: - type: file - base_dir: 'source' \ No newline at end of file From 2890e8fd02922e9b44baa24c3fe133c884d563e2 Mon Sep 17 00:00:00 2001 From: DavixDevelop Date: Sat, 31 Oct 2020 20:38:19 +0100 Subject: [PATCH 14/21] Delete config.render.slovenia.yaml --- config.render.slovenia.yaml | 39 ------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 config.render.slovenia.yaml diff --git a/config.render.slovenia.yaml b/config.render.slovenia.yaml deleted file mode 100644 index 6cfe559..0000000 --- a/config.render.slovenia.yaml +++ /dev/null @@ -1,39 +0,0 @@ ---- -regions: - # san-francisco-bay_california: - # bbox: - # top: 38.719 - # left: -123.640 - # bottom: 36.791 - # right: -121.025 - # zoom_range: [0, 16] - slovenia: - bbox: - top: 46.88333333 - left: 13.39027778 - bottom: 45.40750000 - right: 16.62694444 - zoom_range: [0, 15] -outputs: - - type: skadi - - type: tiff - - type: terrarium - - type: normal -sources: - - type: srtm - url: https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 - mask-url: https://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 - - type: d96tm - uri: http://gis.arso.gov.si/lidar/otr/laz - fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip -logging: - config: logging.example.config -cluster: - queue: - type: fake -store: - type: file - base_dir: 'render' -source_store: - type: file - base_dir: 'source' \ No newline at end of file From 2ea1188198342909a4b45c96efcd052c956cdb1f Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Tue, 3 Nov 2020 00:25:24 +0100 Subject: [PATCH 15/21] Fixed diff --- .gitignore | 58 +++--- config.example.yaml | 100 +++++----- docs/attribution.md | 434 +++++++++++++++++++++---------------------- docs/data-sources.md | 292 ++++++++++++++--------------- 4 files changed, 442 insertions(+), 442 deletions(-) diff --git a/.gitignore b/.gitignore index 05db451..49530c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,29 +1,29 @@ -# editor temporaries -*~ - -# python byte code -*.pyc - -# setup.py stuff -build/ -dist/ -joerd.egg-info/ -/*.egg/ -/*.egg -/.eggs - -# data directories -srtm/ -gmted/ -etopo1/ -ned/ -ned_topobathy/ -dmr1/ - -# tile generation directories -tiles/ -terrarium_tiles/ -normal_tiles/ - -# macOS sillyness -.DS_Store +# editor temporaries +*~ + +# python byte code +*.pyc + +# setup.py stuff +build/ +dist/ +joerd.egg-info/ +/*.egg/ +/*.egg +/.eggs + +# data directories +srtm/ +gmted/ +etopo1/ +ned/ +ned_topobathy/ +dmr1/ + +# tile generation directories +tiles/ +terrarium_tiles/ +normal_tiles/ + +# macOS sillyness +.DS_Store diff --git a/config.example.yaml b/config.example.yaml index 55ddabd..ca91245 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1,50 +1,50 @@ ---- -regions: - # san-francisco-bay_california: - # bbox: - # top: 38.719 - # left: -123.640 - # bottom: 36.791 - # right: -121.025 - # zoom_range: [0, 16] - san-francisco-downtown: - bbox: - top: 37.7997 - left: -122.5109 - bottom: 37.7127 - right: -122.3636 - zoom_range: [0, 16] -outputs: - - type: skadi - - type: tiff - - type: terrarium - - type: normal -sources: - - type: etopo1 - url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip - - type: gmted - url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED - ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] - xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] - tries: 100 - - type: greatlakes - - type: srtm - url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 - mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 - username: 'earthdata username' - password: 'earthdata password' - auth-url: https://urs.earthdata.nasa.gov - - type: ned13 - ftp_server: rockyftp.cr.usgs.gov - base_path: vdelivery/Datasets/Staged/NED/13/IMG - - type: ned - ftp_server: rockyftp.cr.usgs.gov - base_path: vdelivery/Datasets/Staged/NED/19/IMG - - type: ned_topobathy - ftp_server: rockyftp.cr.usgs.gov - base_path: vdelivery/Datasets/Staged/NED/19/IMG - - type: dmr1 - uri: http://gis.arso.gov.si/lidar/dmr1 - fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip -logging: - config: logging.example.config +--- +regions: + # san-francisco-bay_california: + # bbox: + # top: 38.719 + # left: -123.640 + # bottom: 36.791 + # right: -121.025 + # zoom_range: [0, 16] + san-francisco-downtown: + bbox: + top: 37.7997 + left: -122.5109 + bottom: 37.7127 + right: -122.3636 + zoom_range: [0, 16] +outputs: + - type: skadi + - type: tiff + - type: terrarium + - type: normal +sources: + - type: etopo1 + url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip + - type: gmted + url: http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Global_tiles_GMTED + ys: [-90, -70, -50, -30, -10, 10, 30, 50, 70] + xs: [-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150] + tries: 100 + - type: greatlakes + - type: srtm + url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 + mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 + username: 'earthdata username' + password: 'earthdata password' + auth-url: https://urs.earthdata.nasa.gov + - type: ned13 + ftp_server: rockyftp.cr.usgs.gov + base_path: vdelivery/Datasets/Staged/NED/13/IMG + - type: ned + ftp_server: rockyftp.cr.usgs.gov + base_path: vdelivery/Datasets/Staged/NED/19/IMG + - type: ned_topobathy + ftp_server: rockyftp.cr.usgs.gov + base_path: vdelivery/Datasets/Staged/NED/19/IMG + - type: dmr1 + uri: http://gis.arso.gov.si/lidar/dmr1 + fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip +logging: + config: logging.example.config diff --git a/docs/attribution.md b/docs/attribution.md index adf8fde..60afeec 100644 --- a/docs/attribution.md +++ b/docs/attribution.md @@ -1,217 +1,217 @@ -# Attribution - -Attribution is required for many terrain tile data providers. Example language is provided below, but you are responsible for researching each project to follow their license terms. More details are available on the [Data Sources](data-sources.md) page at [Mapzen rights](https://mapzen.com/rights) for Mapzen's hosted service. - -***Required attribution:*** - -``` -* ArcticDEM terrain data DEM(s) were created from DigitalGlobe, Inc., imagery and - funded under National Science Foundation awards 1043681, 1559691, and 1542736; -* Australia terrain data © Commonwealth of Australia (Geoscience Australia) 2017; -* Austria terrain data © offene Daten Österreichs – Digitales Geländemodell (DGM) - Österreich; -* Canada terrain data contains information licensed under the Open Government - Licence – Canada; -* Europe terrain data produced using Copernicus data and information funded by the - European Union - EU-DEM layers; -* Global ETOPO1 terrain data U.S. National Oceanic and Atmospheric Administration -* Mexico terrain data source: INEGI, Continental relief, 2016; -* New Zealand terrain data Copyright 2011 Crown copyright (c) Land Information New - Zealand and the New Zealand Government (All rights reserved); -* Norway terrain data © Kartverket; -* United Kingdom terrain data © Environment Agency copyright and/or database right - 2015. All rights reserved; -* United States 3DEP (formerly NED) and global GMTED2010 and SRTM terrain data - courtesy of the U.S. Geological Survey. -``` - -***Required attribution when using Mapzen's hosted service:*** - -``` -* Mapzen -* ArcticDEM terrain data DEM(s) were created from DigitalGlobe, Inc., imagery and - funded under National Science Foundation awards 1043681, 1559691, and 1542736; -* Australia terrain data © Commonwealth of Australia (Geoscience Australia) 2017; -* Austria terrain data © offene Daten Österreichs – Digitales Geländemodell (DGM) - Österreich; -* Canada terrain data contains information licensed under the Open Government - Licence – Canada; -* Europe terrain data produced using Copernicus data and information funded by the - European Union - EU-DEM layers; -* Global ETOPO1 terrain data U.S. National Oceanic and Atmospheric Administration -* Mexico terrain data source: INEGI, Continental relief, 2016; -* New Zealand terrain data Copyright 2011 Crown copyright (c) Land Information New - Zealand and the New Zealand Government (All rights reserved); -* Norway terrain data © Kartverket; -* United Kingdom terrain data © Environment Agency copyright and/or database right - 2015. All rights reserved; -* United States 3DEP (formerly NED) and global GMTED2010 and SRTM terrain data - courtesy of the U.S. Geological Survey. -``` - -### Where to attribute - -Attribution needs to "appear in a place that is reasonable to the medium or means you are utilising." [Specific examples](http://wiki.osmfoundation.org/wiki/License#Where_to_put_it.3F) are given by the OSM Foundation and are generally best practices for giving credit to any source. More information on attribution is on our [rights](https://mapzen.com/rights) page. - -## The fine print - -### 3DEP - -3DEP (formerly NED) are several public domain datasets released by USGS: - -> All 3DEP products available through The National Map are in the public domain and may be used without restriction. - -However, the USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. - -`3DEP data courtesy of the U.S. Geological Survey` - -More details can be found on the NED/3DEP [FAQ](https://www2.usgs.gov/faq/categories/9865/5041) page, [info](http://nationalmap.gov/3DEP/3dep_prodserv.html) page, and general [acknowledging](https://www2.usgs.gov/visual-id/credit_usgs.html) page. - - -### SRTM - -SRTM is a public domain dataset released as follows, circa 2000 by NASA/NGA with several updates including in 2014, now distributed by USGS: - -> 3.15 -> -> a. Copyright protection is asserted for all products generated by these specifications which are distributed outside of the United States. No domestic copyright will be asserted. -> -> b. The copyright notice (with year of production inserted) states: -> -> © COPYRIGHT (year of production) BY THE UNITED STATES GOVERNMENT. -> NO COPYRIGHT CLAIMED UNDER TITLE 17 U.S.C. - -However, the USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. - -`SRTM data courtesy of the U.S. Geological Survey` - -More details can be found on the SRTM [about](https://lta.cr.usgs.gov/SRTM) from USGS and [about](http://www2.jpl.nasa.gov/srtm/) page from NASA, [data distribution policy](https://lta.cr.usgs.gov/srtm/data_distribution_policy) page, the [metadata](http://dds.cr.usgs.gov/srtm/version2_1/Documentation/MIL-PDF-89020B.pdf), and NASA/NGA original [memorandum of understanding](http://www2.jpl.nasa.gov/srtm/mou.html) and 2014 [press release](http://www.jpl.nasa.gov/news/news.php?release=2014-321) declassifying the global 30 meter data. - -### GMTED2010 - -GMTED2010 is a public domain dataset released, circa 2010 by USGS: - -> Access constraints: -> -> No restrictions, All GMTED2010 data products are publically available. Any acquisition or use of these data signifies a user's agreement to comprehension and compliance of the USGS Standard Disclaimer. Ensure all portions of metadata are read and clearly understood before using these data in order to protect both user and USGS interests. Please refer to http://www.usgs.gov/privacy.html for the USGS disclaimer. -> -> Use constraints: -> -> Although the USGS is making these data available to others who may find the data of value, USGS does not warrant, endorse, or recommend the use of these data for any given purpose. The user assumes the entire risk related to the use of these data. USGS is providing these data "as is", and USGS disclaims any and all warranties, whether expressed or implied, including (without limitation) any implied warranties of merchantability or fitness for a particular purpose. In no event will USGS be liable to you or to any third party for any direct, indirect, incidental, consequential, special, or exemplary damages or lost profits resulting from any use or misuse of these data. Acknowledgement of the U.S. Geological Survey would be appreciated in products derived from these data. Any user who modifies the data is obligated to describe the types of modifications they perform. User specifically agrees not to misrepresent the data, nor to imply that changes made were approved or endorsed by the USGS. - -The USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. - -`GMTED2010 data courtesy of the U.S. Geological Survey` - -More details can be found on the GMTED2010 [about](http://topotools.cr.usgs.gov/gmted_viewer/) page and [Open File Report](http://pubs.usgs.gov/of/2011/1073/pdf/of2011-1073.pdf) PDF, and updated [home](https://lta.cr.usgs.gov/GMTED2010) page. - -### ETOPO1 - -ETOPO1 is a public domain dataset released as follows, circa 2009 by NOAA: - -> Use Limitations -> -> Not to be used for navigation. Although these data are of high quality and useful for planning and modeling purposes, they are not suitable for navigation. For navigation, please refer to the NOS nautical chart series. -> -> Produced by the NOAA National Geophysical Data Center. Not subject to copyright protection within the United States. -> -> While every effort has been made to ensure that these data are accurate and reliable within the limits of the current state of the art, NOAA cannot assume liability for any damages caused by any errors or omissions in the data, nor as a result of the failure of the data to function on a particular system. NOAA makes no warranty, expressed or implied, nor does the fact of distribution constitute such a warranty. - -and - -> Copyright Notice -> -> As required by 17 U.S.C. 403, third parties producing copyrighted works consisting predominantly of the material produced by U.S. government agencies must provide notice with such work(s) identifying the U.S. Government material incorporated and stating that such material is not subject to copyright protection within the United States. - -Formal credit is specified as: - -> DOC/NOAA/NESDIS/NCEI > National Centers for Environmental Information, NESDIS, NOAA, U.S. Department of Commerce - -More details can be found on the ETOPO1 [about](https://www.ngdc.noaa.gov/mgg/global/global.html) page and [data sources](https://www.ngdc.noaa.gov/mgg/global/etopo1sources.html) page, and [description](https://www.ngdc.noaa.gov/docucomp/page?xml=NOAA/NESDIS/NGDC/MGG/DEM/iso/xml/316.xml&view=getDataView&header=none) page and [disclaimer](https://www.ngdc.noaa.gov/ngdcinfo/privacy.html#disclaimer) page. - -### Land Information New Zealand (LINZ) - -[Released](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/) by [LINZ](https://data.linz.govt.nz/), licensed under [Creative Commons Attribution 3.0 New Zealand](https://data.linz.govt.nz/license/attribution-3-0-new-zealand/). - -Attribution statement: - -> Copyright 2011 Crown copyright (c) Land Information New Zealand and the New Zealand Government. All rights reserved - -### data.gov.uk LIDAR Composite Digital Terrain Model - -[Released](https://data.gov.uk/dataset/lidar-composite-dtm-2m1) by [data.gov.uk](https://data.gov.uk/), licensed under the United Kingdom [Open Government License version 3](http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) - -Attribution statement: - -> © Environment Agency copyright and/or database right 2015. All rights reserved. - -### data.gv.at Digitales Geländemodell (DGM) Österreich - -[Released](https://www.data.gv.at/katalog/dataset/b5de6975-417b-4320-afdb-eb2a9e2a1dbf) by Austria's [data.gv.at](https://www.data.gv.at/), licensed under [Creative Commons Namensnennung 3.0 Österreich](https://creativecommons.org/licenses/by/3.0/at/deed.de) - -Attribution statement: - -> © offene Daten Österreichs – Digitales Geländemodell (DGM) Österreich. - -### data.kartverket.no Digital terrengmodell - -[Released](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33) by Austria's Kartverket, [licensed](http://www.kartverket.no/data/lisens/) under [Creative Commons Attribution 4.0 International (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/deed.no). - -Attribution statement: - -> © Kartverket - -### Arctic Digital Elevation Model (ArcticDEM) - -[Released](http://nga.maps.arcgis.com/apps/MapSeries/index.html?appid=cf2fba21df7540fb981f8836f2a97e25) without limitation. "ArcticDEM data is an unlicensed product and may be used, distributed, and modified without permission". - -Requested citation: - -> DEM(s) were created from DigitalGlobe, Inc., imagery and funded under National Science Foundation awards 1043681, 1559691, and 1542736. - -### Digital Terrain Model over Europe (EU-DEM) - -[Released](https://www.eea.europa.eu/data-and-maps/data/eu-dem#tab-metadata) by [European Environmental Agency](https://www.eea.europa.eu/). - -Attribution statement: - -> Produced using Copernicus data and information funded by the European Union - EU-DEM layers. - -### Canadian Digital Elevation Model (CDEM) - -[Released](http://ftp.geogratis.gc.ca/pub/nrcan_rncan/elevation/cdem_mnec/doc/CDEM_product_specs.pdf) under the [Open Government Licence Agreement for Unrestricted Use of Digital Data](http://open.canada.ca/en/open-government-licence-canada). - -Attribution statement: - -> Contains information licensed under the Open Government Licence – Canada. - -### National Institute of Statistics and Geography (INEGI) - -[Released](http://en.www.inegi.org.mx/temas/mapas/relieve/continental/) under Mexico's ["free use of information" license](http://en.www.inegi.org.mx/inegi/terminos.html). - -Attribution statement: - -> Source: INEGI, Continental relief, 2016 - -### Digital Elevation Model (DEM) of Australia derived from LiDAR 5 Metre Grid - -[Released](https://ecat.ga.gov.au/geonetwork/srv/eng/search#!22be4b55-2465-4320-e053-10a3070a5236) by Australia's [Geoscience Australia](http://www.ga.gov.au/), licensed under [Creative Commons Attribution 4.0 International Licence](https://creativecommons.org/licenses/by/4.0/) - -Attribution statement: - -> © Commonwealth of Australia (Geoscience Australia) 2017. - -### Digital Elevation Model (DEM) of Slovenia derived from LiDAR 1 Meter Grid (DMR1) - -[Released](http://www.evode.gov.si/index.php?id=69) by Slovenia's [GURS](https://www.e-prostor.gov.si/), licensed under [Creative Commons 4.0](https://creativecommons.org/licenses/by/4.0/deed.sl) - -> Use Limitations: -> General conditions for the use of geodetic data: https://www.e-prostor.gov.si/dostop-do-podatkov/dostop-do-podatkov/#tab1-1029 -> -> Public access Restrictions: -> No restrictions - -Attribution statement: - -> Geodetska uprava Republike Slovenije, LiDAR, DMR1 2015 -> © 2017 MOP - Geodetska uprava Republike Slovenije - Vse pravice pridržane. \ No newline at end of file +# Attribution + +Attribution is required for many terrain tile data providers. Example language is provided below, but you are responsible for researching each project to follow their license terms. More details are available on the [Data Sources](data-sources.md) page at [Mapzen rights](https://mapzen.com/rights) for Mapzen's hosted service. + +***Required attribution:*** + +``` +* ArcticDEM terrain data DEM(s) were created from DigitalGlobe, Inc., imagery and + funded under National Science Foundation awards 1043681, 1559691, and 1542736; +* Australia terrain data © Commonwealth of Australia (Geoscience Australia) 2017; +* Austria terrain data © offene Daten Österreichs – Digitales Geländemodell (DGM) + Österreich; +* Canada terrain data contains information licensed under the Open Government + Licence – Canada; +* Europe terrain data produced using Copernicus data and information funded by the + European Union - EU-DEM layers; +* Global ETOPO1 terrain data U.S. National Oceanic and Atmospheric Administration +* Mexico terrain data source: INEGI, Continental relief, 2016; +* New Zealand terrain data Copyright 2011 Crown copyright (c) Land Information New + Zealand and the New Zealand Government (All rights reserved); +* Norway terrain data © Kartverket; +* United Kingdom terrain data © Environment Agency copyright and/or database right + 2015. All rights reserved; +* United States 3DEP (formerly NED) and global GMTED2010 and SRTM terrain data + courtesy of the U.S. Geological Survey. +``` + +***Required attribution when using Mapzen's hosted service:*** + +``` +* Mapzen +* ArcticDEM terrain data DEM(s) were created from DigitalGlobe, Inc., imagery and + funded under National Science Foundation awards 1043681, 1559691, and 1542736; +* Australia terrain data © Commonwealth of Australia (Geoscience Australia) 2017; +* Austria terrain data © offene Daten Österreichs – Digitales Geländemodell (DGM) + Österreich; +* Canada terrain data contains information licensed under the Open Government + Licence – Canada; +* Europe terrain data produced using Copernicus data and information funded by the + European Union - EU-DEM layers; +* Global ETOPO1 terrain data U.S. National Oceanic and Atmospheric Administration +* Mexico terrain data source: INEGI, Continental relief, 2016; +* New Zealand terrain data Copyright 2011 Crown copyright (c) Land Information New + Zealand and the New Zealand Government (All rights reserved); +* Norway terrain data © Kartverket; +* United Kingdom terrain data © Environment Agency copyright and/or database right + 2015. All rights reserved; +* United States 3DEP (formerly NED) and global GMTED2010 and SRTM terrain data + courtesy of the U.S. Geological Survey. +``` + +### Where to attribute + +Attribution needs to "appear in a place that is reasonable to the medium or means you are utilising." [Specific examples](http://wiki.osmfoundation.org/wiki/License#Where_to_put_it.3F) are given by the OSM Foundation and are generally best practices for giving credit to any source. More information on attribution is on our [rights](https://mapzen.com/rights) page. + +## The fine print + +### 3DEP + +3DEP (formerly NED) are several public domain datasets released by USGS: + +> All 3DEP products available through The National Map are in the public domain and may be used without restriction. + +However, the USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. + +`3DEP data courtesy of the U.S. Geological Survey` + +More details can be found on the NED/3DEP [FAQ](https://www2.usgs.gov/faq/categories/9865/5041) page, [info](http://nationalmap.gov/3DEP/3dep_prodserv.html) page, and general [acknowledging](https://www2.usgs.gov/visual-id/credit_usgs.html) page. + + +### SRTM + +SRTM is a public domain dataset released as follows, circa 2000 by NASA/NGA with several updates including in 2014, now distributed by USGS: + +> 3.15 +> +> a. Copyright protection is asserted for all products generated by these specifications which are distributed outside of the United States. No domestic copyright will be asserted. +> +> b. The copyright notice (with year of production inserted) states: +> +> © COPYRIGHT (year of production) BY THE UNITED STATES GOVERNMENT. +> NO COPYRIGHT CLAIMED UNDER TITLE 17 U.S.C. + +However, the USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. + +`SRTM data courtesy of the U.S. Geological Survey` + +More details can be found on the SRTM [about](https://lta.cr.usgs.gov/SRTM) from USGS and [about](http://www2.jpl.nasa.gov/srtm/) page from NASA, [data distribution policy](https://lta.cr.usgs.gov/srtm/data_distribution_policy) page, the [metadata](http://dds.cr.usgs.gov/srtm/version2_1/Documentation/MIL-PDF-89020B.pdf), and NASA/NGA original [memorandum of understanding](http://www2.jpl.nasa.gov/srtm/mou.html) and 2014 [press release](http://www.jpl.nasa.gov/news/news.php?release=2014-321) declassifying the global 30 meter data. + +### GMTED2010 + +GMTED2010 is a public domain dataset released, circa 2010 by USGS: + +> Access constraints: +> +> No restrictions, All GMTED2010 data products are publically available. Any acquisition or use of these data signifies a user's agreement to comprehension and compliance of the USGS Standard Disclaimer. Ensure all portions of metadata are read and clearly understood before using these data in order to protect both user and USGS interests. Please refer to http://www.usgs.gov/privacy.html for the USGS disclaimer. +> +> Use constraints: +> +> Although the USGS is making these data available to others who may find the data of value, USGS does not warrant, endorse, or recommend the use of these data for any given purpose. The user assumes the entire risk related to the use of these data. USGS is providing these data "as is", and USGS disclaims any and all warranties, whether expressed or implied, including (without limitation) any implied warranties of merchantability or fitness for a particular purpose. In no event will USGS be liable to you or to any third party for any direct, indirect, incidental, consequential, special, or exemplary damages or lost profits resulting from any use or misuse of these data. Acknowledgement of the U.S. Geological Survey would be appreciated in products derived from these data. Any user who modifies the data is obligated to describe the types of modifications they perform. User specifically agrees not to misrepresent the data, nor to imply that changes made were approved or endorsed by the USGS. + +The USGS does request that proper credit be given when used for redistribution, resale, presentation or publication. + +`GMTED2010 data courtesy of the U.S. Geological Survey` + +More details can be found on the GMTED2010 [about](http://topotools.cr.usgs.gov/gmted_viewer/) page and [Open File Report](http://pubs.usgs.gov/of/2011/1073/pdf/of2011-1073.pdf) PDF, and updated [home](https://lta.cr.usgs.gov/GMTED2010) page. + +### ETOPO1 + +ETOPO1 is a public domain dataset released as follows, circa 2009 by NOAA: + +> Use Limitations +> +> Not to be used for navigation. Although these data are of high quality and useful for planning and modeling purposes, they are not suitable for navigation. For navigation, please refer to the NOS nautical chart series. +> +> Produced by the NOAA National Geophysical Data Center. Not subject to copyright protection within the United States. +> +> While every effort has been made to ensure that these data are accurate and reliable within the limits of the current state of the art, NOAA cannot assume liability for any damages caused by any errors or omissions in the data, nor as a result of the failure of the data to function on a particular system. NOAA makes no warranty, expressed or implied, nor does the fact of distribution constitute such a warranty. + +and + +> Copyright Notice +> +> As required by 17 U.S.C. 403, third parties producing copyrighted works consisting predominantly of the material produced by U.S. government agencies must provide notice with such work(s) identifying the U.S. Government material incorporated and stating that such material is not subject to copyright protection within the United States. + +Formal credit is specified as: + +> DOC/NOAA/NESDIS/NCEI > National Centers for Environmental Information, NESDIS, NOAA, U.S. Department of Commerce + +More details can be found on the ETOPO1 [about](https://www.ngdc.noaa.gov/mgg/global/global.html) page and [data sources](https://www.ngdc.noaa.gov/mgg/global/etopo1sources.html) page, and [description](https://www.ngdc.noaa.gov/docucomp/page?xml=NOAA/NESDIS/NGDC/MGG/DEM/iso/xml/316.xml&view=getDataView&header=none) page and [disclaimer](https://www.ngdc.noaa.gov/ngdcinfo/privacy.html#disclaimer) page. + +### Land Information New Zealand (LINZ) + +[Released](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/) by [LINZ](https://data.linz.govt.nz/), licensed under [Creative Commons Attribution 3.0 New Zealand](https://data.linz.govt.nz/license/attribution-3-0-new-zealand/). + +Attribution statement: + +> Copyright 2011 Crown copyright (c) Land Information New Zealand and the New Zealand Government. All rights reserved + +### data.gov.uk LIDAR Composite Digital Terrain Model + +[Released](https://data.gov.uk/dataset/lidar-composite-dtm-2m1) by [data.gov.uk](https://data.gov.uk/), licensed under the United Kingdom [Open Government License version 3](http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) + +Attribution statement: + +> © Environment Agency copyright and/or database right 2015. All rights reserved. + +### data.gv.at Digitales Geländemodell (DGM) Österreich + +[Released](https://www.data.gv.at/katalog/dataset/b5de6975-417b-4320-afdb-eb2a9e2a1dbf) by Austria's [data.gv.at](https://www.data.gv.at/), licensed under [Creative Commons Namensnennung 3.0 Österreich](https://creativecommons.org/licenses/by/3.0/at/deed.de) + +Attribution statement: + +> © offene Daten Österreichs – Digitales Geländemodell (DGM) Österreich. + +### data.kartverket.no Digital terrengmodell + +[Released](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33) by Austria's Kartverket, [licensed](http://www.kartverket.no/data/lisens/) under [Creative Commons Attribution 4.0 International (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/deed.no). + +Attribution statement: + +> © Kartverket + +### Arctic Digital Elevation Model (ArcticDEM) + +[Released](http://nga.maps.arcgis.com/apps/MapSeries/index.html?appid=cf2fba21df7540fb981f8836f2a97e25) without limitation. "ArcticDEM data is an unlicensed product and may be used, distributed, and modified without permission". + +Requested citation: + +> DEM(s) were created from DigitalGlobe, Inc., imagery and funded under National Science Foundation awards 1043681, 1559691, and 1542736. + +### Digital Terrain Model over Europe (EU-DEM) + +[Released](https://www.eea.europa.eu/data-and-maps/data/eu-dem#tab-metadata) by [European Environmental Agency](https://www.eea.europa.eu/). + +Attribution statement: + +> Produced using Copernicus data and information funded by the European Union - EU-DEM layers. + +### Canadian Digital Elevation Model (CDEM) + +[Released](http://ftp.geogratis.gc.ca/pub/nrcan_rncan/elevation/cdem_mnec/doc/CDEM_product_specs.pdf) under the [Open Government Licence Agreement for Unrestricted Use of Digital Data](http://open.canada.ca/en/open-government-licence-canada). + +Attribution statement: + +> Contains information licensed under the Open Government Licence – Canada. + +### National Institute of Statistics and Geography (INEGI) + +[Released](http://en.www.inegi.org.mx/temas/mapas/relieve/continental/) under Mexico's ["free use of information" license](http://en.www.inegi.org.mx/inegi/terminos.html). + +Attribution statement: + +> Source: INEGI, Continental relief, 2016 + +### Digital Elevation Model (DEM) of Australia derived from LiDAR 5 Metre Grid + +[Released](https://ecat.ga.gov.au/geonetwork/srv/eng/search#!22be4b55-2465-4320-e053-10a3070a5236) by Australia's [Geoscience Australia](http://www.ga.gov.au/), licensed under [Creative Commons Attribution 4.0 International Licence](https://creativecommons.org/licenses/by/4.0/) + +Attribution statement: + +> © Commonwealth of Australia (Geoscience Australia) 2017. + +### Digital Elevation Model (DEM) of Slovenia derived from LiDAR 1 Meter Grid (DMR1) + +[Released](http://www.evode.gov.si/index.php?id=69) by Slovenia's [GURS](https://www.e-prostor.gov.si/), licensed under [Creative Commons 4.0](https://creativecommons.org/licenses/by/4.0/deed.sl) + +> Use Limitations: +> General conditions for the use of geodetic data: https://www.e-prostor.gov.si/dostop-do-podatkov/dostop-do-podatkov/#tab1-1029 +> +> Public access Restrictions: +> No restrictions + +Attribution statement: + +> Geodetska uprava Republike Slovenije, LiDAR, DMR1 2015 +> © 2017 MOP - Geodetska uprava Republike Slovenije - Vse pravice pridržane. diff --git a/docs/data-sources.md b/docs/data-sources.md index 66142b0..2212743 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -1,146 +1,146 @@ -# Data sources - -Mapzen Terrain Tiles are powered by several major open data sets and we owe a tremendous debt of gratitude to the individuals and communities which produced them. - -**Attribution is required** for some data providers. See the [Attribution](https://github.com/tilezen/joerd/blob/master/docs/attribution.md) document for more information. - -## List of sources - -The underlying data sources are a mix of: - -- [3DEP](http://nationalmap.gov/elevation.html) (formerly NED and NED Topobathy) in the United States, 10 meters outside of Alaska, 3 meter in select land and territorial water areas -- [ArcticDEM](http://nga.maps.arcgis.com/apps/MapSeries/index.html?appid=cf2fba21df7540fb981f8836f2a97e25) strips of 5 meter mosaics across all of the land north of 60° latitude, including Alaska, Canada, Greenland, Iceland, Norway, Russia, and Sweden -- [CDEM](http://geogratis.gc.ca/api/en/nrcan-rncan/ess-sst/c40acfba-c722-4be1-862e-146b80be738e.html) (Canadian Digital Elevation Model) in Canada, with variable spatial resolution (from 20-400 meters) depending on the latitude. -- [data.gov.uk](http://environment.data.gov.uk/ds/survey/index.jsp#/survey), 2 meters over most of the United Kingdom -- [data.gv.at](https://www.data.gv.at/katalog/dataset/b5de6975-417b-4320-afdb-eb2a9e2a1dbf), 10 meters over Austria -- [ETOPO1](https://www.ngdc.noaa.gov/mgg/global/global.html) for ocean bathymetry, 1 arc-minute resolution globally -- [EUDEM](https://www.eea.europa.eu/data-and-maps/data/eu-dem#tab-original-data) in most of Europe at 30 meter resolution, including Albania, Austria, Belgium, Bosnia and Herzegovina, Bulgaria, Croatia, Cyprus, Czechia, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Iceland, Ireland, Italy, Kosovo, Latvia, Liechtenstein, Lithuania, Luxembourg, Macedonia, Malta, Montenegro, Netherlands, Norway, Poland, Portugal, Romania, Serbia, Slovakia, Slovenia, Spain, Sweden, Switzerland, and United Kingdom -- Geoscience Australia's [DEM of Australia](https://ecat.ga.gov.au/geonetwork/srv/eng/search#!22be4b55-2465-4320-e053-10a3070a5236), 5 meters around coastal regions in South Australia, Victoria, and Northern Territory -- [GMTED](http://topotools.cr.usgs.gov/gmted_viewer/) globally, coarser resolutions at 7.5", 15", and 30" in land areas -- [INEGI](http://en.www.inegi.org.mx/temas/mapas/relieve/continental/)'s continental relief in Mexico -- [Kartverket](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33)'s Digital Terrain Model, 10 meters over Norway -- [LINZ](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/), 8 meters over New Zealand -- [SRTM](https://lta.cr.usgs.gov/SRTM) globally except high latitudes, 30 meters (90 meters nominal quality) in land areas -- [DMR1](http://gis.arso.gov.si/evode/profile.aspx?id=atlas_voda_Lidar%40Arso&initialExtent=499500.5%2C109841.5%2C264.58333) derived from pre-processed Lidar, which contains only the terrain at 1m resolution, 1200m-1400m over Slovenia - -### Footprints database - -These source images are composited to form tiles that make up the Mapzen Terrain Tiles service. To determine exactly which images contributed to Mapzen Terrain Tiles in a particular area, you can download the footprints database and use it with a GIS program like [QGIS](http://www.qgis.org/) to inspect which of these sources were used. - -![Preview Rendering of Footprints](images/footprints-preview.png) - -* [GeoJSON](https://s3.amazonaws.com/elevation-tiles-prod/docs/footprints.geojson.gz) (8.7MB, gzipped) -* [PostgreSQL Dump](https://s3.amazonaws.com/elevation-tiles-prod/docs/footprints.pgdump.gz) (14.5MB, gzipped) - -### Source headers - -To further assist in determining which sources contributed to an individual tile, the Mapzen Terrain Tiles service will respond with an [HTTP header](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Response_fields) listing the sources that contributed to that tile. The value of the `X-Imagery-Sources` HTTP header is a comma-separated list, where each entry follows the pattern `source/filename.tif`. - -For example, a tile might have the header `X-Imagery-Sources: srtm/N39W110.tif, srtm/N39W112.tif, gmted/30N120W_20101117_gmted_mea075.tif`, meaning that it was built from three source images. Two SRTM images and a GMTED image were composited together to generate the tile output. Using the footprint database dumps above you can gather more information about these source images, including the calculated resolution and footprint geometry. To find the entry in the database, look for an entry that has a matching `filename` attribute. - - -## What is the ground resolution? - -Ground resolution per tile pixel varies per zoom level, the given pixel cell's latitude, and input data source. - -This formula generates the following table: - -`ground_resolution = (cos(latitude * pi/180) * 2 * pi * 6378137 meters) / (256 * 2^zoom_level pixels)` - -Ground resolution per **zoom** in `meters` at a given _latitude_: - -zoom | _0°_ | _45°_ | _60°_ ------- | ---------- | ---------- | --------- -**0** | `156543.0` | `110692.6` | `78271.5` -**1** | `78271.5` | `55346.3` | `39135.8` -**2** | `39135.8` | `27673.2` | `19567.9` -**3** | `19567.9` | `13836.6` | `9783.9` -**4** | `9783.9` | `6918.3` | `4892.0` -**5** | `4892.0` | `3459.1` | `2446.0` -**6** | `2446.0` | `1729.6` | `1223.0` -**7** | `1223.0` | `864.8` | `611.5` -**8** | `611.5` | `432.4` | `305.7` -**9** | `305.7` | `216.2` | `152.9` -**10** | `152.9` | `108.1` | `76.4` -**11** | `76.4` | `54.0` | `38.2` -**12** | `38.2` | `27.0` | `19.1` -**13** | `19.1` | `13.5` | `9.6` -**14** | `9.6` | `6.8` | `4.8` -**15** | `4.8` | `3.4` | `2.4` - -**Note:** Esri has [documentation](https://blogs.esri.com/esri/arcgis/2009/03/19/how-can-you-tell-what-map-scales-are-shown-for-online-maps/) about converting web map zoom integers to conventional map scales. - -## What is sourced at what zooms? - -Generally speaking, **GMTED** is used at low-zooms, **ETOPO1** is used to show ocean bathymetry at all zooms (even bathymetry oversampled at zoom 15), and **[SRTM](http://www2.jpl.nasa.gov/srtm/)** is relied on in mid- and high-zooms on land. Some countries have higher resolution data available over land sourced from other open datasets. More information about these sources is available below. - -It should be noted that both `SRTM` and `GMTED` fill oceans and other bodies of water with a value of zero to indicate mean sea level; in these areas, `ETOPO1` provides bathymetry (as well as in regions which are not covered by `SRTM` and `GMTED`). - -**Data sources per zoom:** - -zoom | ocean | land ------- | -------- | ------- -**0** | `ETOPO1` | `ETOPO1` -**1** | `ETOPO1` | `ETOPO1` -**2** | `ETOPO1` | `ETOPO1` -**3** | `ETOPO1` | `ETOPO1` -**4** | `ETOPO1` | `GMTED` -**5** | `ETOPO1` | `GMTED` -**6** | `ETOPO1` | `GMTED` -**7** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60° -**8** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60° -**9** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, `EUDEM` in Europe, with `GMTED` in high latitudes above 60° -**10** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia -**11** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia -**12** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia -**13** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia -**14** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia -**15** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia - -## Sources native resolution - -You might be wondering why we source from different data sets at different zooms. Besides bandwidth reasons, it's helpful to know the native resolution of each data set which is expressed as a nominal arc resolution which maps roughly to a web map zoom for "best displayed at". - -In more practical terms, this results in some datasets being **"oversampled"** for a given zoom and map location. - -* In the water, most bathymetry values at zoom 15 for a pixel that has a ground resolution of 5 meter will actually be showing an oversampled zoom 6 `ETOPO1` value (nominally 2.5 km). - -* On the land, most elevation values at zoom 15 for a pixel that has a ground resolution of 5 meter will actually be showing an oversampled zoom 12 `SRTM` value (nominally 30 meters). - -This formula generates the following table: - -`ground_resolution = (cos(latitude * pi/180) * 2 * pi * 6378137 meters) / (256 * 2^zoom_level pixels) / 30.87 meters per arc second` - -Arc resolution per **zoom** and data sources, per pixel: - -zoom | meters at equator | arc seconds | nominal arc degrees minutes seconds | source | nominal ground units ------ | ---------- | -------- | ------------------- | -------- | -------------------- -**0** | _156543.0_ | `5071.0` | **1.5 arc degrees** | | -**1** | _78271.5_ | `2535.5` | **40 arc minutes** | | -**2** | _39135.8_ | `1267.8` | **20 arc minutes** | | -**3** | _19567.9_ | `633.9` | **10 arc minutes** | | -**4** | _9783.9_ | `316.9` | **5 arc minutes** | | -**5** | _4892.0_ | `158.5` | **2.5 arc minutes** | | -**6** | _2446.0_ | `79.2` | **1 arc minutes** | `ETOPO1` | 2.5 km -**7** | _1223.0_ | `39.6` | **30 arc seconds** | `GMTED2010` | 1km (not used) -**8** | _611.5_ | `19.8` | **15 arc seconds** | `GMTED2010` | 500m (not used) -**9** | _305.7_ | `9.9` | **7.5 arc seconds** | `GMTED2010` | 250m -**10** | _152.9_ | `5.0` | **5 arc seconds** | | -**11** | _76.4_ | `2.5` | **3 arc seconds** | Canada | 90m -**12** | _38.2_ | `1.2` | **1 arc seconds** | `SRTM`, Canada | 30m -**13** | _19.1_ | `0.6` | **2/3 arc seconds** | | -**14** | _9.6_ | `0.3` | **1/3 arc seconds** | `3DEP`, Austria, Australia, New Zealand, Norway | 10m -**15** | _4.8_ | `0.2` | **1/5 arc seconds** | Mexico, Arctic | -**16** | _2.4_ | `0.1` | **1/9 arc seconds** | `3DEP`, United Kingdom | 3m - -## Data updates - -Terrain tiles version 1 was built during 2016Q2 and released in 2016Q3 based on available sources at that time. Version 1.1 was built during 2017Q3 and released 2017Q4. Regular updates are not planned. - -Future updates will be on an as-needed basis for smaller regions to incorporate additional `3DEP` coverage in the United States and additional country specific data sources globally. - -We are always looking for better datasets. If you find a data issue or can suggest an open terrain datasets, please let us know by filing an issue in [tilezen/joerd](https://github.com/tilezen/joerd/issues/new). - -## Known issues - -Many classical DEM and LIDAR related issues occur in terrain tiles. It is not uncommon to see large variations in elevation in areas with large buildings and other such structures. Resampling and merging artifacts are also observed along coastlines (where different datasets are seamed together). +# Data sources + +Mapzen Terrain Tiles are powered by several major open data sets and we owe a tremendous debt of gratitude to the individuals and communities which produced them. + +**Attribution is required** for some data providers. See the [Attribution](https://github.com/tilezen/joerd/blob/master/docs/attribution.md) document for more information. + +## List of sources + +The underlying data sources are a mix of: + +- [3DEP](http://nationalmap.gov/elevation.html) (formerly NED and NED Topobathy) in the United States, 10 meters outside of Alaska, 3 meter in select land and territorial water areas +- [ArcticDEM](http://nga.maps.arcgis.com/apps/MapSeries/index.html?appid=cf2fba21df7540fb981f8836f2a97e25) strips of 5 meter mosaics across all of the land north of 60° latitude, including Alaska, Canada, Greenland, Iceland, Norway, Russia, and Sweden +- [CDEM](http://geogratis.gc.ca/api/en/nrcan-rncan/ess-sst/c40acfba-c722-4be1-862e-146b80be738e.html) (Canadian Digital Elevation Model) in Canada, with variable spatial resolution (from 20-400 meters) depending on the latitude. +- [data.gov.uk](http://environment.data.gov.uk/ds/survey/index.jsp#/survey), 2 meters over most of the United Kingdom +- [data.gv.at](https://www.data.gv.at/katalog/dataset/b5de6975-417b-4320-afdb-eb2a9e2a1dbf), 10 meters over Austria +- [ETOPO1](https://www.ngdc.noaa.gov/mgg/global/global.html) for ocean bathymetry, 1 arc-minute resolution globally +- [EUDEM](https://www.eea.europa.eu/data-and-maps/data/eu-dem#tab-original-data) in most of Europe at 30 meter resolution, including Albania, Austria, Belgium, Bosnia and Herzegovina, Bulgaria, Croatia, Cyprus, Czechia, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Iceland, Ireland, Italy, Kosovo, Latvia, Liechtenstein, Lithuania, Luxembourg, Macedonia, Malta, Montenegro, Netherlands, Norway, Poland, Portugal, Romania, Serbia, Slovakia, Slovenia, Spain, Sweden, Switzerland, and United Kingdom +- Geoscience Australia's [DEM of Australia](https://ecat.ga.gov.au/geonetwork/srv/eng/search#!22be4b55-2465-4320-e053-10a3070a5236), 5 meters around coastal regions in South Australia, Victoria, and Northern Territory +- [GMTED](http://topotools.cr.usgs.gov/gmted_viewer/) globally, coarser resolutions at 7.5", 15", and 30" in land areas +- [INEGI](http://en.www.inegi.org.mx/temas/mapas/relieve/continental/)'s continental relief in Mexico +- [Kartverket](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33)'s Digital Terrain Model, 10 meters over Norway +- [LINZ](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/), 8 meters over New Zealand +- [SRTM](https://lta.cr.usgs.gov/SRTM) globally except high latitudes, 30 meters (90 meters nominal quality) in land areas +- [DMR1](http://gis.arso.gov.si/evode/profile.aspx?id=atlas_voda_Lidar%40Arso&initialExtent=499500.5%2C109841.5%2C264.58333) derived from pre-processed Lidar, which contains only the terrain at 1m resolution, 1200m-1400m over Slovenia + +### Footprints database + +These source images are composited to form tiles that make up the Mapzen Terrain Tiles service. To determine exactly which images contributed to Mapzen Terrain Tiles in a particular area, you can download the footprints database and use it with a GIS program like [QGIS](http://www.qgis.org/) to inspect which of these sources were used. + +![Preview Rendering of Footprints](images/footprints-preview.png) + +* [GeoJSON](https://s3.amazonaws.com/elevation-tiles-prod/docs/footprints.geojson.gz) (8.7MB, gzipped) +* [PostgreSQL Dump](https://s3.amazonaws.com/elevation-tiles-prod/docs/footprints.pgdump.gz) (14.5MB, gzipped) + +### Source headers + +To further assist in determining which sources contributed to an individual tile, the Mapzen Terrain Tiles service will respond with an [HTTP header](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Response_fields) listing the sources that contributed to that tile. The value of the `X-Imagery-Sources` HTTP header is a comma-separated list, where each entry follows the pattern `source/filename.tif`. + +For example, a tile might have the header `X-Imagery-Sources: srtm/N39W110.tif, srtm/N39W112.tif, gmted/30N120W_20101117_gmted_mea075.tif`, meaning that it was built from three source images. Two SRTM images and a GMTED image were composited together to generate the tile output. Using the footprint database dumps above you can gather more information about these source images, including the calculated resolution and footprint geometry. To find the entry in the database, look for an entry that has a matching `filename` attribute. + + +## What is the ground resolution? + +Ground resolution per tile pixel varies per zoom level, the given pixel cell's latitude, and input data source. + +This formula generates the following table: + +`ground_resolution = (cos(latitude * pi/180) * 2 * pi * 6378137 meters) / (256 * 2^zoom_level pixels)` + +Ground resolution per **zoom** in `meters` at a given _latitude_: + +zoom | _0°_ | _45°_ | _60°_ +------ | ---------- | ---------- | --------- +**0** | `156543.0` | `110692.6` | `78271.5` +**1** | `78271.5` | `55346.3` | `39135.8` +**2** | `39135.8` | `27673.2` | `19567.9` +**3** | `19567.9` | `13836.6` | `9783.9` +**4** | `9783.9` | `6918.3` | `4892.0` +**5** | `4892.0` | `3459.1` | `2446.0` +**6** | `2446.0` | `1729.6` | `1223.0` +**7** | `1223.0` | `864.8` | `611.5` +**8** | `611.5` | `432.4` | `305.7` +**9** | `305.7` | `216.2` | `152.9` +**10** | `152.9` | `108.1` | `76.4` +**11** | `76.4` | `54.0` | `38.2` +**12** | `38.2` | `27.0` | `19.1` +**13** | `19.1` | `13.5` | `9.6` +**14** | `9.6` | `6.8` | `4.8` +**15** | `4.8` | `3.4` | `2.4` + +**Note:** Esri has [documentation](https://blogs.esri.com/esri/arcgis/2009/03/19/how-can-you-tell-what-map-scales-are-shown-for-online-maps/) about converting web map zoom integers to conventional map scales. + +## What is sourced at what zooms? + +Generally speaking, **GMTED** is used at low-zooms, **ETOPO1** is used to show ocean bathymetry at all zooms (even bathymetry oversampled at zoom 15), and **[SRTM](http://www2.jpl.nasa.gov/srtm/)** is relied on in mid- and high-zooms on land. Some countries have higher resolution data available over land sourced from other open datasets. More information about these sources is available below. + +It should be noted that both `SRTM` and `GMTED` fill oceans and other bodies of water with a value of zero to indicate mean sea level; in these areas, `ETOPO1` provides bathymetry (as well as in regions which are not covered by `SRTM` and `GMTED`). + +**Data sources per zoom:** + +zoom | ocean | land +------ | -------- | ------- +**0** | `ETOPO1` | `ETOPO1` +**1** | `ETOPO1` | `ETOPO1` +**2** | `ETOPO1` | `ETOPO1` +**3** | `ETOPO1` | `ETOPO1` +**4** | `ETOPO1` | `GMTED` +**5** | `ETOPO1` | `GMTED` +**6** | `ETOPO1` | `GMTED` +**7** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60° +**8** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60° +**9** | `ETOPO1` | `SRTM`, `NRCAN` in Canada, `EUDEM` in Europe, with `GMTED` in high latitudes above 60° +**10** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia +**11** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia +**12** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia +**13** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia +**14** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia +**15** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia + +## Sources native resolution + +You might be wondering why we source from different data sets at different zooms. Besides bandwidth reasons, it's helpful to know the native resolution of each data set which is expressed as a nominal arc resolution which maps roughly to a web map zoom for "best displayed at". + +In more practical terms, this results in some datasets being **"oversampled"** for a given zoom and map location. + +* In the water, most bathymetry values at zoom 15 for a pixel that has a ground resolution of 5 meter will actually be showing an oversampled zoom 6 `ETOPO1` value (nominally 2.5 km). + +* On the land, most elevation values at zoom 15 for a pixel that has a ground resolution of 5 meter will actually be showing an oversampled zoom 12 `SRTM` value (nominally 30 meters). + +This formula generates the following table: + +`ground_resolution = (cos(latitude * pi/180) * 2 * pi * 6378137 meters) / (256 * 2^zoom_level pixels) / 30.87 meters per arc second` + +Arc resolution per **zoom** and data sources, per pixel: + +zoom | meters at equator | arc seconds | nominal arc degrees minutes seconds | source | nominal ground units +----- | ---------- | -------- | ------------------- | -------- | -------------------- +**0** | _156543.0_ | `5071.0` | **1.5 arc degrees** | | +**1** | _78271.5_ | `2535.5` | **40 arc minutes** | | +**2** | _39135.8_ | `1267.8` | **20 arc minutes** | | +**3** | _19567.9_ | `633.9` | **10 arc minutes** | | +**4** | _9783.9_ | `316.9` | **5 arc minutes** | | +**5** | _4892.0_ | `158.5` | **2.5 arc minutes** | | +**6** | _2446.0_ | `79.2` | **1 arc minutes** | `ETOPO1` | 2.5 km +**7** | _1223.0_ | `39.6` | **30 arc seconds** | `GMTED2010` | 1km (not used) +**8** | _611.5_ | `19.8` | **15 arc seconds** | `GMTED2010` | 500m (not used) +**9** | _305.7_ | `9.9` | **7.5 arc seconds** | `GMTED2010` | 250m +**10** | _152.9_ | `5.0` | **5 arc seconds** | | +**11** | _76.4_ | `2.5` | **3 arc seconds** | Canada | 90m +**12** | _38.2_ | `1.2` | **1 arc seconds** | `SRTM`, Canada | 30m +**13** | _19.1_ | `0.6` | **2/3 arc seconds** | | +**14** | _9.6_ | `0.3` | **1/3 arc seconds** | `3DEP`, Austria, Australia, New Zealand, Norway | 10m +**15** | _4.8_ | `0.2` | **1/5 arc seconds** | Mexico, Arctic | +**16** | _2.4_ | `0.1` | **1/9 arc seconds** | `3DEP`, United Kingdom | 3m + +## Data updates + +Terrain tiles version 1 was built during 2016Q2 and released in 2016Q3 based on available sources at that time. Version 1.1 was built during 2017Q3 and released 2017Q4. Regular updates are not planned. + +Future updates will be on an as-needed basis for smaller regions to incorporate additional `3DEP` coverage in the United States and additional country specific data sources globally. + +We are always looking for better datasets. If you find a data issue or can suggest an open terrain datasets, please let us know by filing an issue in [tilezen/joerd](https://github.com/tilezen/joerd/issues/new). + +## Known issues + +Many classical DEM and LIDAR related issues occur in terrain tiles. It is not uncommon to see large variations in elevation in areas with large buildings and other such structures. Resampling and merging artifacts are also observed along coastlines (where different datasets are seamed together). From dd3e7e49c13b8d3818fd3b10ed38aa5df3227326 Mon Sep 17 00:00:00 2001 From: DavixDevelop Date: Tue, 3 Nov 2020 00:27:57 +0100 Subject: [PATCH 16/21] Delete otr.py --- joerd/source/otr.py | 260 -------------------------------------------- 1 file changed, 260 deletions(-) delete mode 100644 joerd/source/otr.py diff --git a/joerd/source/otr.py b/joerd/source/otr.py deleted file mode 100644 index ac73a80..0000000 --- a/joerd/source/otr.py +++ /dev/null @@ -1,260 +0,0 @@ -from bs4 import BeautifulSoup -from joerd.util import BoundingBox -import joerd.download as download -import joerd.check as check -import joerd.srs as srs -import joerd.index as index -import joerd.mask as mask -import joerd.tmpdir as tmpdir -from joerd.mkdir_p import mkdir_p -from contextlib2 import closing, ExitStack -from shutil import copyfile, move -import os.path -import os -import io -import requests -import logging -import re -import tempfile -import sys -import zipfile -import traceback -import subprocess -import glob -from osgeo import gdal -from osgeo import ogr -from osgeo import osr -import pdal -import yaml -import time -import json -import subprocess - -class OTRTile(object): - def __init__(self, parent, link, name, block, bbox): - self.uri = parent.uri - self.download_options = parent.download_options - self.base_dir = parent.base_dir - self.name = name - self.block = block - self.link = link - self.bbox = bbox - - def __key(self): - return self.name - - def __eq__(a, b): - return isinstance(b, type(a)) and \ - a.__key() == b.__key() - - def __hash__(self): - return hash(self.__key()) - - def urls(self): - uri_list = [self.uri + "/" + self.link] - return uri_list - - - def verifier(self): - return is_las - - def options(self): - return self.download_options - - def output_file(self): - file_name = "TMR_" + self.name + ".tif" - return os.path.join(self.base_dir, file_name) - - def _tif_file(self): - # returns the name of the geotiff file within the distributed archive. - return "TMR_%(name)s.tif" % dict(name=self.name) - - def unpack(self, store, las_file): - tif_file = self._tif_file; - - with store.upload_dir() as target: - target_dir = os.path.join(target, self.base_dir) - mkdir_p(target_dir) - with tmpdir.tmpdir() as temp_dir: - - arr = [] - inp = { - "type": "readers.las", - "filename": "%(fname)s" % dict(fname=las_file.name), - "spatialreference":"EPSG:3794" - } - arr.append(inp) - par = { - "type": "writers.gdal", - "resolution": 1, - "radius": 7, - "filename": os.path.join(temp_dir, "TMR_%(name)s.tif" % dict(name=self.name)) - } - arr.append(par) - - astage = { - "pipeline" : arr - } - - - #j = json.dumps(astage) - #u = unicode(j, "utf-8") - - json_path = os.path.join(temp_dir, "TMR_%(name)s.json" % dict(name=self.name)) - - with open(json_path, 'w') as json_file: - json.dump(astage, json_file) - - #covert lidar to geotiff - #pipeline = pdal.Pipeline(u) - #count = pipeline.execute() - - subprocess.check_output('pdal pipeline {jfile}'.format(jfile=json_path), cwd=temp_dir, shell=True) - - output_file = os.path.join(target, self.output_file()) - mask.negative(os.path.join(temp_dir, "TMR_%(name)s.tif" % dict(name=self.name)), "GTiff", output_file) - - - def freeze_dry(self): - return dict(type='otr', link=self.link) - -IS_OTR_FILE = re.compile( - '^b_([0-9]{2})/D96TM/TMR_([0-9]{2,3})_([0-9]{2,3}).laz') - -def is_las(self): - def func(tmp): - if tmp.name.endswith(".laz"): - return True - else: - return False - return func - - -def _parse_otr_tile(link, parent): - m = IS_OTR_FILE.match(link) - - if not m: - return None - - d96_bottom = int(m.group(3)) * 1000 - d96_left = int(m.group(2)) * 1000 - d96_top = (int(m.group(3)) + 1) * 1000 - d96_right = (int(m.group(2)) + 1) * 1000 - - block = int(m.group(1)) - name = m.group(2) + "_" + m.group(3) - - #D96/TM (EPSG::3794) to WGS84 (EPSG::4326) - src = osr.SpatialReference() - tgt = osr.SpatialReference() - src.ImportFromEPSG(3794) - tgt.ImportFromEPSG(4326) - - transform = osr.CoordinateTransformation(src, tgt) - coords = transform.TransformPoint(d96_left, d96_bottom) - left,bottom = coords[0:2] - coords = transform.TransformPoint(d96_right, d96_top) - right,top= coords[0:2] - - bbox = BoundingBox(left, bottom, right, top) - - return OTRTile(parent, link, name, block, bbox) - -class OTR(object): - def __init__(self, options={}): - self.base_dir = options.get('base_dir', 'otr') - self.uri = options['uri'] - self.fishnet_url = options['fishnet_url'] - self.download_options = options - self.tile_index = None - - def get_index(self): - if not os.path.isdir(self.base_dir): - os.makedirs(self.base_dir) - index_name = 'index.yaml' - index_file = os.path.join(self.base_dir, index_name) - # if index doesn't exist, or is more than 24h old - if not os.path.isfile(index_file) or \ - time.time() > os.path.getmtime(index_file) + 86400: - self.download_index(index_file) - - def download_index(self, index_file): - logger = logging.getLogger('otr') - logger.info('Fetcthing D96TM index...') - links = [] - - fishnet = 'LIDAR_FISHNET_D96.shp' - - req = requests.get(self.fishnet_url, stream=True) - with tmpdir.tmpdir() as d: - with zipfile.ZipFile(io.BytesIO(req.content)) as zip_file: - zip_file.extractall(d) - - fishnet_file = os.path.join(d, fishnet) - driver = ogr.GetDriverByName("ESRI Shapefile") - dataSource = driver.Open(fishnet_file, 1) - layer = dataSource.GetLayer() - - for feature in layer: - links.append(feature.GetField("BLOK") + "/D96TM/TMR_" + feature.GetField("NAME") + ".laz") - layer.ResetReading() - - with open(index_file, 'w') as file: - file.write(yaml.dump(links)) - - def _ensure_tile_index(self): - if self.tile_index is None: - index_file = os.path.join(self.base_dir, 'index.yaml') - bbox = (13.39027778,45.40750000,16.62694444,46.88333333) - self.tile_index = index.create(index_file, bbox, _parse_otr_tile, self) - - return self.tile_index - - def existing_files(self): - for base, dirs, files in os.walk(self.base_dir): - for f in files: - if f.endswith('tif'): - yield os.path.join(base, f) - - def rehydrate(self, data): - assert data.get('type') == 'otr', \ - "Unable to rehydrate %r from Slovenia." %data - return _parse_otr_tile(data['link'], self) - - def downloads_for(self, tile): - tiles = set() - # if the tile scale is greater than 20x the D96TM scale, then there's no - # point in including D96TM, it'll be far too fine to make a difference. - # D96TM is 1m. - - if tile.max_resolution() > 20: - return tiles - - # buffer by 0.0075 degrees (81px) to grab neighbouring tiles and ensure - # some overlap to take care of boundary issues. - tile_bbox = tile.latlon_bbox().buffer(0.0075) - - tile_index = self._ensure_tile_index() - - for t in index.intersections(tile_index, tile_bbox): - tiles.add(t) - - return tiles - - def filter_type(self, src_res, dst_res): - return gdal.GRA_Lanczos if src_res > dst_res else gdal.GRA_Cubic - - def vrts_for(self, tile): - """ - Returns a list of sets of tiles, with each list element intended as a - separate VRT for use in GDAL. - - D96TM is non-overlapping. - """ - return [self.downloads_for(tile)] - - def srs(self): - return srs.d96() - -def create(options): - return OTR(options) \ No newline at end of file From 3cf1fc84bdedd5a510d27a62bd8edbcf4cc8d53d Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Tue, 3 Nov 2020 00:30:18 +0100 Subject: [PATCH 17/21] Fixed url for Slovenia source --- config.slovenia.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.slovenia.yaml b/config.slovenia.yaml index 3f8d7bf..6793be8 100644 --- a/config.slovenia.yaml +++ b/config.slovenia.yaml @@ -27,7 +27,7 @@ sources: password: 'earthdata password' auth-url: https://urs.earthdata.nasa.gov - type: dmr1 - uri: http://gis.arso.gov.si/lidar/otr/laz + uri: http://gis.arso.gov.si/lidar/dmr1 fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip logging: config: logging.example.config From f2001f641e42f04eed39dac382e5b43ec31b7d62 Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Tue, 3 Nov 2020 21:22:39 +0100 Subject: [PATCH 18/21] Fixed upack of dataset --- config.slovenia.yaml | 4 +++- config.slovenia_test.yaml | 4 +++- joerd/source/dmr1.py | 15 ++++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/config.slovenia.yaml b/config.slovenia.yaml index 6793be8..e09f60e 100644 --- a/config.slovenia.yaml +++ b/config.slovenia.yaml @@ -13,13 +13,15 @@ regions: left: 13.39027778 bottom: 45.40750000 right: 16.62694444 - zoom_range: [10, 15] + zoom_range: [10, 16] outputs: - type: skadi - type: tiff - type: terrarium - type: normal sources: + - type: etopo1 + url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip - type: srtm url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 diff --git a/config.slovenia_test.yaml b/config.slovenia_test.yaml index 2651c30..14162fd 100644 --- a/config.slovenia_test.yaml +++ b/config.slovenia_test.yaml @@ -13,13 +13,15 @@ regions: left: 15.237007 bottom: 46.205735 right: 15.288334 - zoom_range: [10, 15] + zoom_range: [10, 16] outputs: - type: skadi - type: tiff - type: terrarium - type: normal sources: + - type: etopo1 + url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip - type: srtm url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11 mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11 diff --git a/joerd/source/dmr1.py b/joerd/source/dmr1.py index d379d65..491c76f 100644 --- a/joerd/source/dmr1.py +++ b/joerd/source/dmr1.py @@ -29,7 +29,8 @@ import subprocess import mimetypes -#this needs to be changed to the region you want to download use +#this needs to be changed to the region you want to download and render +#Can be removed alongside line:141 and line:144 if memory usage due to the index is not important REGION_BBOX = BoundingBox(15.237007,46.205735,15.288334,46.239346) class DMR1Tile(object): @@ -85,10 +86,11 @@ def unpack(self, store, txt_file): subprocess.check_output("sort -k2 -n -t';' -k1 {tfile} -o {ofile}".format(tfile=txt_file.name, ofile=xyz_file), cwd=temp_dir, shell=True) #Correct NS (North South) resolution and convert to xyz to tif - subprocess.check_output("gdalwarp -t_srs EPSG:3794 -overwrite {xfile} {tfile}".format(xfile=xyz_file,tfile=tif_file), cwd=temp_dir, shell=True) + subprocess.check_output("gdalwarp -t_srs EPSG:3794 -ts 1001 1001 -overwrite {xfile} {tfile}".format(xfile=xyz_file,tfile=tif_file), cwd=temp_dir, shell=True) + #Move the file to the store output_file = os.path.join(target, self.output_file()) - mask.negative(os.path.join(temp_dir, tif_file), "GTiff", output_file) + move(os.path.join(temp_dir, tif_file), output_file) def freeze_dry(self): @@ -187,8 +189,7 @@ def _ensure_tile_index(self): if self.tile_index is None: index_file = os.path.join(self.base_dir, 'index.yaml') - #this needs to be changed to the region you want to download use - bbox = (15.237007,46.205735,15.288334,46.239346) + bbox = (13.39027778,45.40750000,16.62694444,46.88333333) self.tile_index = index.create(index_file, bbox, _parse_dmr1_tile, self) return self.tile_index @@ -208,9 +209,9 @@ def downloads_for(self, tile): tiles = set() # if the tile scale is greater than 20x the D96TM scale, then there's no # point in including D96TM, it'll be far too fine to make a difference. - # D96TM is 1m. + # D96TM is 1m (Aboue same as 1/9th arc second). - if tile.max_resolution() > 20: + if tile.max_resolution() > 20 * 1.0 / (3600 * 9): return tiles # buffer by 0.0075 degrees (81px) to grab neighbouring tiles and ensure From 6ecef46f82b1b8060e395d91ea6e5e66a14c2757 Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Tue, 3 Nov 2020 23:44:25 +0100 Subject: [PATCH 19/21] Fixed issues with overlapping and transparency by implementing vrts_for --- joerd/source/dmr1.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/joerd/source/dmr1.py b/joerd/source/dmr1.py index 491c76f..15781b9 100644 --- a/joerd/source/dmr1.py +++ b/joerd/source/dmr1.py @@ -28,6 +28,7 @@ import time import subprocess import mimetypes +from itertools import groupby #this needs to be changed to the region you want to download and render #Can be removed alongside line:141 and line:144 if memory usage due to the index is not important @@ -168,6 +169,18 @@ def download_index(self, index_file): fishnet = 'LIDAR_FISHNET_D96.shp' + #these tiles do not have dmr1 + blacklist = ['b_21/D96TM/TM1_393_35.txt', + 'b_21/D96TM/TM1_392_35.txt', + 'b_23/D96TM/TM1_509_169.txt' + 'b_24/D96TM/TM1_616_148.txt', + 'b_24/D96TM/TM1_617_148.txt', + 'b_24/D96TM/TM1_618_148.txt', + 'b_24/D96TM/TM1_622_147.txt', + 'b_24/D96TM/TM1_622_148.txt', + 'b_24/D96TM/TM1_623_148.txt', + 'b_24/D96TM/TM1_623_149.txt'] + req = requests.get(self.fishnet_url, stream=True) with tmpdir.tmpdir() as d: with zipfile.ZipFile(io.BytesIO(req.content)) as zip_file: @@ -179,7 +192,9 @@ def download_index(self, index_file): layer = dataSource.GetLayer() for feature in layer: - links.append(feature.GetField("BLOK") + "/D96TM/TM1_" + feature.GetField("NAME") + ".txt") + link = feature.GetField("BLOK") + "/D96TM/TM1_" + feature.GetField("NAME") + ".txt"; + if link not in blacklist: + links.append(link) layer.ResetReading() with open(index_file, 'w') as file: @@ -233,9 +248,18 @@ def vrts_for(self, tile): Returns a list of sets of tiles, with each list element intended as a separate VRT for use in GDAL. - D96TM is non-overlapping. + D96TM is overlapping. """ - return [self.downloads_for(tile)] + vrt = [] + tiles = self.downloads_for(tile) + + def func(tile): + return (tile.link) + + for k, t in groupby(sorted(tiles, key=func), func): + vrt.append(set(t)) + + return vrt def srs(self): return srs.d96() From 8e41ef9d6a30a7197f86c57ef469c3df7aa11036 Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Sat, 9 Jan 2021 22:47:42 +0100 Subject: [PATCH 20/21] Added bounding box option for dmr1 --- config.slovenia.yaml | 5 +++++ config.slovenia_test.yaml | 5 +++++ joerd/source/dmr1.py | 9 ++++----- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/config.slovenia.yaml b/config.slovenia.yaml index e09f60e..4d9f76a 100644 --- a/config.slovenia.yaml +++ b/config.slovenia.yaml @@ -31,6 +31,11 @@ sources: - type: dmr1 uri: http://gis.arso.gov.si/lidar/dmr1 fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip + bbox: + top: 46.88333333 + left: 13.39027778 + bottom: 45.40750000 + right: 16.62694444 logging: config: logging.example.config cluster: diff --git a/config.slovenia_test.yaml b/config.slovenia_test.yaml index 14162fd..249f701 100644 --- a/config.slovenia_test.yaml +++ b/config.slovenia_test.yaml @@ -31,6 +31,11 @@ sources: - type: dmr1 uri: http://gis.arso.gov.si/lidar/dmr1 fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip + bbox: + top: 46.239346 + left: 15.237007 + bottom: 46.205735 + right: 15.288334 logging: config: logging.example.config cluster: diff --git a/joerd/source/dmr1.py b/joerd/source/dmr1.py index 15781b9..cdce8da 100644 --- a/joerd/source/dmr1.py +++ b/joerd/source/dmr1.py @@ -30,10 +30,6 @@ import mimetypes from itertools import groupby -#this needs to be changed to the region you want to download and render -#Can be removed alongside line:141 and line:144 if memory usage due to the index is not important -REGION_BBOX = BoundingBox(15.237007,46.205735,15.288334,46.239346) - class DMR1Tile(object): def __init__(self, parent, link, name, block, bbox): self.uri = parent.uri @@ -43,6 +39,7 @@ def __init__(self, parent, link, name, block, bbox): self.block = block self.link = link self.bbox = bbox + self.region_bbox = parent.region_bbox def __key(self): return self.name @@ -139,7 +136,7 @@ def _parse_dmr1_tile(link, parent): #This is so the memory doesn't overflow, due to the big size of the index #and is to be removed on a production server - if bbox.intersects(REGION_BBOX): + if bbox.intersects(parent.region_bbox): return DMR1Tile(parent, link, name, block, bbox) return None @@ -149,6 +146,8 @@ def __init__(self, options={}): self.base_dir = options.get('base_dir', 'dmr1') self.uri = options['uri'] self.fishnet_url = options['fishnet_url'] + box = options['bbox'] + self.region_bbox = BoundingBox(box['left'], box['bottom'], box['right'],box['top']) self.download_options = options self.tile_index = None From 94b08e4bf4ade3f145d11e1f1cdbe8bdc36c46af Mon Sep 17 00:00:00 2001 From: David Rupnik Date: Mon, 22 Mar 2021 14:42:34 +0100 Subject: [PATCH 21/21] Fixed the conversion from dmr1 tile to GeoTIFF, removed fixed width and height of 1001 for tiles and fixed the incorrect tile bounds by switching to a pre-made index file (79.71% complete) --- config.example.yaml | 2 +- config.slovenia.yaml | 2 +- config.slovenia_test.yaml | 2 +- joerd/source/dmr1.py | 402 ++++++++++++++++++++------------------ 4 files changed, 213 insertions(+), 195 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index ca91245..de0bb3c 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -45,6 +45,6 @@ sources: base_path: vdelivery/Datasets/Staged/NED/19/IMG - type: dmr1 uri: http://gis.arso.gov.si/lidar/dmr1 - fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip + fishnet_url: https://raw.githubusercontent.com/DavixDevelop/TerraLidar/master/mapzen_data/whitelist.yaml logging: config: logging.example.config diff --git a/config.slovenia.yaml b/config.slovenia.yaml index 4d9f76a..fd3c6d1 100644 --- a/config.slovenia.yaml +++ b/config.slovenia.yaml @@ -30,7 +30,7 @@ sources: auth-url: https://urs.earthdata.nasa.gov - type: dmr1 uri: http://gis.arso.gov.si/lidar/dmr1 - fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip + fishnet_url: https://raw.githubusercontent.com/DavixDevelop/TerraLidar/master/mapzen_data/whitelist.yaml bbox: top: 46.88333333 left: 13.39027778 diff --git a/config.slovenia_test.yaml b/config.slovenia_test.yaml index 249f701..8206336 100644 --- a/config.slovenia_test.yaml +++ b/config.slovenia_test.yaml @@ -30,7 +30,7 @@ sources: auth-url: https://urs.earthdata.nasa.gov - type: dmr1 uri: http://gis.arso.gov.si/lidar/dmr1 - fishnet_url: http://gis.arso.gov.si/related/lidar_porocila/lidar_fishnet_D96TM.zip + fishnet_url: https://raw.githubusercontent.com/DavixDevelop/TerraLidar/master/mapzen_data/whitelist.yaml bbox: top: 46.239346 left: 15.237007 diff --git a/joerd/source/dmr1.py b/joerd/source/dmr1.py index cdce8da..b816e24 100644 --- a/joerd/source/dmr1.py +++ b/joerd/source/dmr1.py @@ -31,237 +31,255 @@ from itertools import groupby class DMR1Tile(object): - def __init__(self, parent, link, name, block, bbox): - self.uri = parent.uri - self.download_options = parent.download_options - self.base_dir = parent.base_dir - self.name = name - self.block = block - self.link = link - self.bbox = bbox - self.region_bbox = parent.region_bbox + def __init__(self, parent, tile, name, block, bbox): + self.uri = parent.uri + self.download_options = parent.download_options + self.base_dir = parent.base_dir + self.name = name + self.block = block + self.tile = tile + self.bbox = bbox + self.region_bbox = parent.region_bbox - def __key(self): - return self.name + def __key(self): + return self.name - def __eq__(a, b): - return isinstance(b, type(a)) and \ - a.__key() == b.__key() + def __eq__(a, b): + return isinstance(b, type(a)) and \ + a.__key() == b.__key() - def __hash__(self): - return hash(self.__key()) + def __hash__(self): + return hash(self.__key()) - def urls(self): - uri_list = [self.uri + "/" + self.link] - return uri_list + def urls(self): + uri_list = ["%(uri)s/b_%(block)s/D96TM/TM1_%(name)s.txt" % dict(uri=self.uri,block=self.block,name=self.name)] + return uri_list + def verifier(self): + return is_txt - def verifier(self): - return is_txt + def options(self): + return self.download_options - def options(self): - return self.download_options + def output_file(self): + file_name = "TM1_" + self.name + ".tif" + return os.path.join(self.base_dir, file_name) - def output_file(self): - file_name = "TM1_" + self.name + ".tif" - return os.path.join(self.base_dir, file_name) + def _tif_file(self): + # returns the name of the geotiff file within the distributed archive. + return "TM1_%(name)s.tif" % dict(name=self.name) - def _tif_file(self): - # returns the name of the geotiff file within the distributed archive. - return "TM1_%(name)s.tif" % dict(name=self.name) + def unpack(self, store, txt_file): - def unpack(self, store, txt_file): - tif_file = self._tif_file(); + with store.upload_dir() as target: + target_dir = os.path.join(target, self.base_dir) + mkdir_p(target_dir) + with tmpdir.tmpdir() as temp_dir: + tif_file = os.path.join(temp_dir, self._tif_file()) - with store.upload_dir() as target: - target_dir = os.path.join(target, self.base_dir) - mkdir_p(target_dir) - with tmpdir.tmpdir() as temp_dir: + xyz_file = os.path.join(temp_dir, "TM1_%(name)s.xyz" % dict(name=self.name)) - xyz_file = "TM1_%(name)s.xyz" % dict(name=self.name) + #Flip x any y, as they are fliped in dmr1 + data = [] + with open(txt_file.name, 'r') as dmr_file: + for l in dmr_file: + data.append(l.strip().split(";")) - #Flip x any y, as they are fliped in dmr1 - subprocess.check_output("sort -k2 -n -t';' -k1 {tfile} -o {ofile}".format(tfile=txt_file.name, ofile=xyz_file), cwd=temp_dir, shell=True) + data = sorted(data, key = lambda x: (float(x[1]), float(x[0]))) - #Correct NS (North South) resolution and convert to xyz to tif - subprocess.check_output("gdalwarp -t_srs EPSG:3794 -ts 1001 1001 -overwrite {xfile} {tfile}".format(xfile=xyz_file,tfile=tif_file), cwd=temp_dir, shell=True) + xyzlist = [] + for l in data: + xyzlist.append('{x};{y};{z}\n'.format(x=l[0],y=l[1],z=l[2])) + with open(xyz_file, 'w') as xyz: + xyz.writelines(xyzlist) - #Move the file to the store - output_file = os.path.join(target, self.output_file()) - move(os.path.join(temp_dir, tif_file), output_file) - + #Correct NS (North South) resolution and convert to xyz to tif + ds = gdal.Open(xyz_file) + ds = gdal.Warp(tif_file,ds,format="GTiff",dstSRS="EPSG:3794") + ds = None - def freeze_dry(self): - return dict(type='dmr1', link=self.link) + #subprocess.check_output("sort -k2 -n -t';' -k1 {tfile} -o {ofile}".format(tfile=txt_file.name, ofile=xyz_file), cwd=temp_dir, shell=True) + #subprocess.check_output("gdalwarp -t_srs EPSG:3794 -ts -overwrite {xfile} {tfile}".format(xfile=xyz_file,tfile=tif_file), cwd=temp_dir, shell=True) -IS_DMR1_FILE = re.compile( - '^b_([0-9]{2})/D96TM/TM1_([0-9]{2,3})_([0-9]{2,3}).txt') + #Move the file to the store + output_file = os.path.join(target, self.output_file()) + move(tif_file, output_file) + + + def freeze_dry(self): + return dict(type='dmr1', tile=self.tile) + +#IS_DMR1_FILE = re.compile('^b_([0-9]{2})/D96TM/TM1_([0-9]{2,3})_([0-9]{2,3}).txt') +IS_DMR1_FILE = re.compile(r'^b_([0-9]{2})\|TM1_([0-9]{2,3})_([0-9]{2,3})\|([\d*\.?\d*]+)\|([\d*\.?\d*]+)\|([\d*\.?\d*]+)\|([\d*\.?\d*]+)') def is_txt(self): - def func(tmp): - if mimetypes.guess_type(tmp.name)[0] == 'text/plain': - return True - else: - return False - return func - + def func(tmp): + if mimetypes.guess_type(tmp.name)[0] == 'text/plain': + return True + else: + return False + return func + -def _parse_dmr1_tile(link, parent): - m = IS_DMR1_FILE.match(link) +def _parse_dmr1_tile(tile, parent): + m = IS_DMR1_FILE.match(tile) - if not m: - return None + if not m: + return None - d96_bottom = int(m.group(3)) * 1000 - d96_left = int(m.group(2)) * 1000 - d96_top = (int(m.group(3)) + 1) * 1000 - d96_right = (int(m.group(2)) + 1) * 1000 + d96_left = float(m.group(4)) + d96_bottom = float(m.group(5)) + d96_right = float(m.group(6)) + d96_top = float(m.group(7)) - block = int(m.group(1)) - name = m.group(2) + "_" + m.group(3) + block = int(m.group(1)) + name = "%(x)s_%(y)s" % dict(x=str(m.group(2)),y=str(m.group(3))) - #D96/TM (EPSG::3794) to WGS84 (EPSG::4326) - src = osr.SpatialReference() - tgt = osr.SpatialReference() - src.ImportFromEPSG(3794) - tgt.ImportFromEPSG(4326) + #D96/TM (EPSG::3794) to WGS84 (EPSG::4326) + src = osr.SpatialReference() + tgt = osr.SpatialReference() + src.ImportFromEPSG(3794) + tgt.ImportFromEPSG(4326) - transform = osr.CoordinateTransformation(src, tgt) - coords = transform.TransformPoint(d96_left, d96_bottom) - left,bottom = coords[0:2] - coords = transform.TransformPoint(d96_right, d96_top) - right,top= coords[0:2] + transform = osr.CoordinateTransformation(src, tgt) + coords = transform.TransformPoint(d96_left, d96_bottom) + left,bottom = coords[0:2] + coords = transform.TransformPoint(d96_right, d96_top) + right,top= coords[0:2] - bbox = BoundingBox(left, bottom, right, top) + bbox = BoundingBox(left, bottom, right, top) - #This is so the memory doesn't overflow, due to the big size of the index - #and is to be removed on a production server - if bbox.intersects(parent.region_bbox): - return DMR1Tile(parent, link, name, block, bbox) + #This is so the memory doesn't overflow, due to the big size of the index + #and is to be removed on a production server + if bbox.intersects(parent.region_bbox): + return DMR1Tile(parent, tile, name, block, bbox) - return None + return None class DMR1(object): - def __init__(self, options={}): - self.base_dir = options.get('base_dir', 'dmr1') - self.uri = options['uri'] - self.fishnet_url = options['fishnet_url'] - box = options['bbox'] - self.region_bbox = BoundingBox(box['left'], box['bottom'], box['right'],box['top']) - self.download_options = options - self.tile_index = None - - def get_index(self): - if not os.path.isdir(self.base_dir): - os.makedirs(self.base_dir) - index_name = 'index.yaml' - index_file = os.path.join(self.base_dir, index_name) - # if index doesn't exist, or is more than 24h old - if not os.path.isfile(index_file) or \ - time.time() > os.path.getmtime(index_file) + 86400: - self.download_index(index_file) - - def download_index(self, index_file): - logger = logging.getLogger('dmr1') - logger.info('Fetcthing D96TM index...') - links = [] - - fishnet = 'LIDAR_FISHNET_D96.shp' - - #these tiles do not have dmr1 - blacklist = ['b_21/D96TM/TM1_393_35.txt', - 'b_21/D96TM/TM1_392_35.txt', - 'b_23/D96TM/TM1_509_169.txt' - 'b_24/D96TM/TM1_616_148.txt', - 'b_24/D96TM/TM1_617_148.txt', - 'b_24/D96TM/TM1_618_148.txt', - 'b_24/D96TM/TM1_622_147.txt', - 'b_24/D96TM/TM1_622_148.txt', - 'b_24/D96TM/TM1_623_148.txt', - 'b_24/D96TM/TM1_623_149.txt'] - - req = requests.get(self.fishnet_url, stream=True) - with tmpdir.tmpdir() as d: - with zipfile.ZipFile(io.BytesIO(req.content)) as zip_file: - zip_file.extractall(d) - - fishnet_file = os.path.join(d, fishnet) - driver = ogr.GetDriverByName("ESRI Shapefile") - dataSource = driver.Open(fishnet_file, 1) - layer = dataSource.GetLayer() - - for feature in layer: - link = feature.GetField("BLOK") + "/D96TM/TM1_" + feature.GetField("NAME") + ".txt"; - if link not in blacklist: - links.append(link) - layer.ResetReading() - - with open(index_file, 'w') as file: - file.write(yaml.dump(links)) - - def _ensure_tile_index(self): - if self.tile_index is None: - index_file = os.path.join(self.base_dir, 'index.yaml') - - bbox = (13.39027778,45.40750000,16.62694444,46.88333333) - self.tile_index = index.create(index_file, bbox, _parse_dmr1_tile, self) - - return self.tile_index - - def existing_files(self): - for base, dirs, files in os.walk(self.base_dir): - for f in files: - if f.endswith('tif'): - yield os.path.join(base, f) - - def rehydrate(self, data): - assert data.get('type') == 'dmr1', \ - "Unable to rehydrate %r from Slovenia." %data - return _parse_dmr1_tile(data['link'], self) - - def downloads_for(self, tile): - tiles = set() - # if the tile scale is greater than 20x the D96TM scale, then there's no - # point in including D96TM, it'll be far too fine to make a difference. - # D96TM is 1m (Aboue same as 1/9th arc second). - - if tile.max_resolution() > 20 * 1.0 / (3600 * 9): - return tiles - - # buffer by 0.0075 degrees (81px) to grab neighbouring tiles and ensure + def __init__(self, options={}): + self.base_dir = options.get('base_dir', 'dmr1') + self.uri = options['uri'] + self.fishnet_url = options['fishnet_url'] + box = options['bbox'] + self.region_bbox = BoundingBox(box['left'], box['bottom'], box['right'],box['top']) + self.download_options = options + self.tile_index = None + + def get_index(self): + if not os.path.isdir(self.base_dir): + os.makedirs(self.base_dir) + index_name = 'index.yaml' + index_file = os.path.join(self.base_dir, index_name) + # if index doesn't exist, or is more than 24h old + if not os.path.isfile(index_file) or \ + time.time() > os.path.getmtime(index_file) + 86400: + self.download_index(index_file) + + def download_index(self, index_file): + logger = logging.getLogger('dmr1') + logger.info('Fetcthing D96TM index...') + + """ + fishnet = 'LIDAR_FISHNET_D96.shp' + + #these tiles do not have dmr1 + blacklist = ['b_21/D96TM/TM1_393_35.txt', + 'b_21/D96TM/TM1_392_35.txt', + 'b_23/D96TM/TM1_509_169.txt' + 'b_24/D96TM/TM1_616_148.txt', + 'b_24/D96TM/TM1_617_148.txt', + 'b_24/D96TM/TM1_618_148.txt', + 'b_24/D96TM/TM1_622_147.txt', + 'b_24/D96TM/TM1_622_148.txt', + 'b_24/D96TM/TM1_623_148.txt', + 'b_24/D96TM/TM1_623_149.txt'] + + req = requests.get(self.fishnet_url, stream=True) + with tmpdir.tmpdir() as d: + with zipfile.ZipFile(io.BytesIO(req.content)) as zip_file: + zip_file.extractall(d) + + fishnet_file = os.path.join(d, fishnet) + driver = ogr.GetDriverByName("ESRI Shapefile") + dataSource = driver.Open(fishnet_file, 1) + layer = dataSource.GetLayer() + + for feature in layer: + link = feature.GetField("BLOK") + "/D96TM/TM1_" + feature.GetField("NAME") + ".txt"; + if link not in blacklist: + links.append(link) + layer.ResetReading() + """ + req = requests.get(self.fishnet_url, allow_redirects=True) + + with open(index_file, 'w') as file: + file.write(req.content) + #file.write(yaml.dump(whitelist)) + + def _ensure_tile_index(self): + if self.tile_index is None: + index_file = os.path.join(self.base_dir, 'index.yaml') + + bbox = (13.39027778,45.40750000,16.62694444,46.88333333) + self.tile_index = index.create(index_file, bbox, _parse_dmr1_tile, self) + + return self.tile_index + + def existing_files(self): + for base, dirs, files in os.walk(self.base_dir): + for f in files: + if f.endswith('tif'): + yield os.path.join(base, f) + + def rehydrate(self, data): + assert data.get('type') == 'dmr1', \ + "Unable to rehydrate %r from Slovenia." %data + return _parse_dmr1_tile(data['tile'], self) + + def downloads_for(self, tile): + tiles = set() + # if the tile scale is greater than 20x the D96TM scale, then there's no + # point in including D96TM, it'll be far too fine to make a difference. + # D96TM is 1m (Aboue same as 1/9th arc second). + + if tile.max_resolution() > 20 * 1.0 / (3600 * 9): + return tiles + + # buffer by 0.0075 degrees (81px) to grab neighbouring tiles and ensure # some overlap to take care of boundary issues. - tile_bbox = tile.latlon_bbox().buffer(0.0075) + tile_bbox = tile.latlon_bbox().buffer(0.0075) - tile_index = self._ensure_tile_index() + tile_index = self._ensure_tile_index() - for t in index.intersections(tile_index, tile_bbox): - tiles.add(t) + for t in index.intersections(tile_index, tile_bbox): + tiles.add(t) - return tiles + return tiles - def filter_type(self, src_res, dst_res): - return gdal.GRA_Lanczos if src_res > dst_res else gdal.GRA_Cubic + def filter_type(self, src_res, dst_res): + return gdal.GRA_Lanczos if src_res > dst_res else gdal.GRA_Cubic - def vrts_for(self, tile): - """ - Returns a list of sets of tiles, with each list element intended as a - separate VRT for use in GDAL. - - D96TM is overlapping. - """ - vrt = [] - tiles = self.downloads_for(tile) + def vrts_for(self, tile): + """ + Returns a list of sets of tiles, with each list element intended as a + separate VRT for use in GDAL. + + D96TM is overlapping. + """ + vrt = [] + tiles = self.downloads_for(tile) - def func(tile): - return (tile.link) + def func(tile): + return (tile.tile) - for k, t in groupby(sorted(tiles, key=func), func): - vrt.append(set(t)) + for k, t in groupby(sorted(tiles, key=func), func): + vrt.append(set(t)) - return vrt + return vrt - def srs(self): - return srs.d96() + def srs(self): + return srs.d96() def create(options): - return DMR1(options) \ No newline at end of file + return DMR1(options) \ No newline at end of file