Allow HTTP paths as layer sources

buildmap/input.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 import typing
 
 if typing.TYPE_CHECKING:
@@ -6,15 +7,32 @@
 
 class Input:
     def __init__(self, buildmap: "BuildMap", table_name: str, layer_config: dict):
+        self.buildmap = buildmap
         self.table = table_name
+        self.local_path = None
         if "path" not in layer_config:
             raise ValueError(f"Layer {table_name} is missing path")
-        self.path = buildmap.resolve_path(layer_config["path"])
-        if not self.path.is_file():
-            raise Exception("Source file %s does not exist" % self.path)
-
         self.config = layer_config
 
+    @property
+    def path(self):
+        path = self.config["path"]
+        # check if path is a URL
+        if path.startswith("http://") or path.startswith("https://"):
+            if self.local_path:
+                return self.local_path
+            else:
+                self.local_path = self.buildmap.download(path)
+                return self.local_path
+
+        path = self.buildmap.resolve_path(path)
+        if not path.is_file():
+            raise Exception("Source file %s does not exist" % self.path)
+        return path
+
     @property
     def file_type(self):
-        return self.path.suffix[1:].lower()
+        suffix = self.path.suffix[1:].lower()
+        if suffix == "json":
+            suffix = "geojson"
+        return suffix

buildmap/main.py

@@ -8,6 +8,7 @@
 import time
 import argparse
 import importlib
+import requests
 from json.decoder import JSONDecodeError
 from collections import defaultdict
 from typing import Union
@@ -314,3 +315,15 @@ def generate_static(self, dest_layers):
         else:
             self.log.error("Requested static layer (%s) not found", self.args.layer)
             return
+
+    def download(self, remote_path: str):
+        """Download a remote file into the temp directory, and return the path"""
+        file_name = remote_path.split("/")[-1]
+        self.log.info("Downloading %s...", remote_path)
+        with requests.get(remote_path, stream=True) as r:
+            r.raise_for_status()
+            with open(self.temp_dir / file_name, "wb") as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+
+        return self.temp_dir / file_name

setup.py

@@ -34,6 +34,7 @@
         "pydotplus",
         "mergedeep==1.3.4",
         "pint==0.18",
+        "requests==2.31.0",
     ],
     entry_points={"console_scripts": {"buildmap=buildmap.main:run"}},
 )