Merge pull request #450 from hotosm/fix/api

Fix/api

docs/Dev/bootstrapsh.md

@@ -55,3 +55,35 @@ ecuador-latests.osm.pbf
 ecuador.poly
 ```
 
+## OSM authentication for downloading data
+
+If you want to bootstrap your database with ChangeSet data downloaded from
+ GeoFabrik, you'll need to be authenticated with an OSM account.
+
+ A good utility for doing this from command line is in the sendfile_osm_oauth_protector
+ repository.
+
+ ```sh
+ git clone https://github.com/geofabrik/sendfile_osm_oauth_protector.git
+ cd sendfile_osm_oauth_protector
+ ```
+
+Edit `settings.json` with your credentials:
+
+ ```json
+ {
+  "user": "<your_osm_username>",
+  "password": "<your_osm_password>",
+  "osm_host": "https://www.openstreetmap.org",
+  "consumer_url": "https://osm-internal.download.geofabrik.de/get_cookie"
+}
+```
+
+And download your file:
+
+```sh
+curl https://osm-internal.download.geofabrik.de/africa/tanzania-latest-internal.osm.pbf \
+   --cookie "$(cat cookie_output_file.txt)" --output tanzania-latest.osm.pbf
+```
+
+Then you can move the pbf file to the `underpass/utils` directory and use the `-l yes` option.

docs/get-started/Setup.md

@@ -5,9 +5,13 @@ You can prepare your Underpass installation with data for a specific country.
 Go to the `utils` directory and run the boostrap script:
 
 ```
-./bootstrap.sh -r south-america -c ecuador
+./bootstrap.sh -r south-america -c uruguay
 ```
 
+Use `-p <PORT>` and `-u <USERNAME>` for the database.
+
+For example, if you installed Underpass using Docker:
+
 Regions (-r) are:
 
     africa
@@ -22,5 +26,5 @@ Countries (-c) is the name of the country inside the region.
 
 Data is downloaded from GeoFabrik, if you are not sure of what name you need to use, please check there.
 
-For advanced users, check the [boostrap documentation](/underpass/Dev/bootstrapsh).
+For advanced users, check the [boostrap script documentation](/underpass/Dev/bootstrapsh).
 

python/dbapi/api/db.py

@@ -49,10 +49,10 @@ def run(self, query, responseType = 'json', singleObject = False):
             try:
                 cur.execute(query)
             except Exception as e: 
-                print("*******" + "\n" + query + "\n")
+                print("\n******* \n" + query + "\n******* \n")
                 print(e)
                 cur.close()
-                return {"Error": "There was an error running the query."}
+                return None
 
             results = None
 

python/dbapi/api/raw.py

@@ -91,7 +91,6 @@ def listFeaturesQuery(
         dateTo = None,
         status = None,
         table = None,
-        orderBy = "created_at"
     ):
 
         geoType = getGeoType(table)
@@ -116,10 +115,12 @@ def listFeaturesQuery(
         return query
 
 def queryToJSON(query, orderBy, page):
-    return "with data AS (" + query + " {0}) , t_features AS ( \
-            SELECT to_jsonb(data) as feature from data \
-        ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;".format(
-            "ORDER BY " + orderBy + " DESC LIMIT " + str(RESULTS_PER_PAGE_LIST) + (" OFFSET {0}".format(page * RESULTS_PER_PAGE_LIST) if page else ""),
+    return "with data AS (" + query + ") , t_features AS ( \
+            SELECT to_jsonb(data) as feature from data {0} \
+        ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;" \
+        .format(
+            "WHERE " + orderBy + " IS NOT NULL ORDER BY " + orderBy + " DESC LIMIT " + str(RESULTS_PER_PAGE_LIST) + (" OFFSET {0}" \
+            .format(page * RESULTS_PER_PAGE_LIST) if page else ""),
         )
 
 class Raw:
@@ -236,13 +237,13 @@ def getAll(
 
         result = {'type': 'FeatureCollection', 'features': []}
 
-        if 'features' in polygons and polygons['features']:
+        if polygons and polygons['features']:
             result['features'] = result['features'] + polygons['features']
 
-        if 'features' in lines and lines['features']:
+        if lines and lines['features']:
             result['features'] = result['features'] + lines['features']
 
-        elif 'features' in nodes and nodes['features']:
+        elif nodes and nodes['features']:
             result['features'] = result['features'] + nodes['features']
 
         return result
@@ -291,7 +292,6 @@ def getLinesList(
             "ways_line"
         ), responseType, True)
 
-
     def getNodesList(
         self, 
         area = None,
@@ -324,6 +324,7 @@ def getAllList(
         dateFrom = None,
         dateTo = None,
         status = None,
+        orderBy = None,
         page = None
     ):
 
@@ -359,8 +360,8 @@ def getAllList(
 
         query = queryToJSON(
             " UNION ".join([queryPolygons, queryLines, queryNodes]),
-            "id",
-            page
+            orderBy or "id",
+            page or 0,
         )
 
         return self.underpassDB.run(query, responseType, True)

python/dbapi/api/stats.py

@@ -83,6 +83,9 @@ def getCount(
                     "AND status = '{0}'".format(status) if (status) else "",
                 )
 
-        return self.underpassDB.run(query, True)[0]
+        result = self.underpassDB.run(query, True)
+        if result:
+            return result[0]
+        return None
 
 

python/restapi/main.py

@@ -194,7 +194,7 @@ def getLines(request: RawRequest):
     return results
 
 @app.post("/raw/all")
-def getLines(request: RawRequest):
+def getAll(request: RawRequest):
     results = rawer.getAll(
         area = request.area,
         tags = request.tags or "",
@@ -241,6 +241,7 @@ def getAllList(request: RawRequest):
         dateFrom = request.dateFrom or "",
         dateTo = request.dateTo or "",
         status = request.status or "",
+        orderBy = request.orderBy or None,
         page = request.page,
     )
     return results

python/restapi/models.py

@@ -19,6 +19,7 @@ class RawRequest(BaseModel):
     dateFrom: str = None
     dateTo: str = None
     status: str = None
+    orderBy: str = None
     page: int = None
 
 class StatsRequest(BaseModel):