Merge pull request #171 from hotosm/feature/tippecanoe_docker

Feature : Tippecanoe Docker Support

API/api_worker.py

@@ -48,7 +48,6 @@ def process_raw_data(self, params):
         params.file_name = (
             format_file_name_str(params.file_name) if params.file_name else "Export"
         )
-        logging.info("Is Default Export ? : %s", params.uuid)
         exportname = f"{params.file_name}_{params.output_type}{f'_uid_{str(self.request.id)}' if params.uuid else ''}"
 
         logging.info("Request %s received", exportname)

API/tasks.py

@@ -1,3 +1,6 @@
+import re
+from typing import List
+
 from celery.result import AsyncResult
 from fastapi import APIRouter, Depends
 from fastapi.responses import JSONResponse
@@ -63,9 +66,43 @@ def inspect_workers():
         active: Current Active tasks ongoing on workers
     """
     inspected = celery.control.inspect()
-    return JSONResponse(
-        {"scheduled": str(inspected.scheduled()), "active": str(inspected.active())}
-    )
+
+    def extract_file_name(args: str) -> str:
+        """Extract file_name using a pattern match."""
+        match = re.search(r"file_name\s*=\s*['\"]([^'\"]+)['\"]", args)
+        return match.group(1) if match else None
+
+    def filter_task_details(tasks: List[dict]) -> List[dict]:
+        """Filter task details to include only id and file_name."""
+        filtered_tasks = {}
+        if tasks:
+            for worker in tasks:
+                filtered_tasks[worker] = {"total": 0, "detail": []}
+                if tasks[worker]:
+                    # key is worker name here
+                    for value_task in tasks[worker]:
+                        if value_task:
+                            if "id" in value_task:
+                                task_id = value_task.get("id")
+                                args = value_task.get("args")
+                                file_name = extract_file_name(str(args))
+                                if task_id:
+                                    filtered_tasks[worker]["total"] += 1
+                                    filtered_tasks[worker]["detail"].append(
+                                        {"id": task_id, "file_name": file_name}
+                                    )
+        return filtered_tasks
+
+    scheduled_tasks = inspected.scheduled()
+    # print(scheduled_tasks)
+    active_tasks = inspected.active()
+    # print(active_tasks)
+    response_data = {
+        "scheduled": filter_task_details(scheduled_tasks),
+        "active": filter_task_details(active_tasks),
+    }
+
+    return JSONResponse(content=response_data)
 
 
 @router.get("/ping/")

Dockerfile

@@ -5,7 +5,6 @@ FROM docker.io/python:${PYTHON_VERSION}-slim-bookworm as base
 ARG [email protected]
 ENV DEBIAN_FRONTEND=noninteractive
 
-
 FROM base as builder
 
 ENV PIP_NO_CACHE_DIR=1
@@ -26,7 +25,6 @@ RUN pip install --user --no-cache-dir --upgrade pip \
     && pip install --user --no-cache-dir -r requirements.txt \
     && pip install --user --no-cache-dir -e .
 
-
 FROM base as runner
 WORKDIR /home/appuser
 ENV PIP_NO_CACHE_DIR=1
@@ -48,13 +46,19 @@ COPY setup.py .
 COPY API/ ./API/
 COPY src/ ./src/
 
+# Use a separate stage to pull the tippecanoe image
+FROM docker.io/itskshitiz321/tippecanoe as tippecanoe-builder
+
+FROM runner as prod
+USER root
+
+# Copy tippecanoe binaries from the tippecanoe stage
+COPY --from=tippecanoe-builder /usr/local/bin/tippecanoe* /usr/local/bin/
+COPY --from=tippecanoe-builder /usr/local/bin/tile-join /usr/local/bin/
+
 RUN useradd --system --uid 900 --home-dir /home/appuser --shell /bin/false appuser \
     && chown -R appuser:appuser /home/appuser
 
-
-FROM runner as prod
 USER appuser
 
 CMD ["uvicorn", "API.main:app", "--reload", "--host", "0.0.0.0", "--port", "8000", "--no-use-colors", "--proxy-headers"]
-
-# HEALTHCHECK --interval=1m --timeout=3s CMD curl -f http://localhost:8000 || exit 1

backend/sql/update_countries.sql

@@ -1,3 +1 @@
-WITH t1 AS (SELECT osm_id, ST_Centroid(geom) AS geom FROM nodes wl WHERE country <@ Array[0]),
-t2 AS (SELECT t1.osm_id, CASE WHEN COUNT(cg.cid) = 0 THEN ARRAY[1000]::INTEGER[] ELSE ARRAY_AGG(COALESCE(cg.cid, 1000)) END AS aa_fids FROM t1 LEFT JOIN countries cg ON ST_Intersects(t1.geom, cg.geometry) GROUP BY t1.osm_id)
-UPDATE nodes uw SET country = t2.aa_fids FROM t2 WHERE t2.osm_id = uw.osm_id;
+WITH t1 AS (SELECT osm_id, ST_Centroid(geom) AS geom FROM relations wl WHERE country <@ Array[0]), t2 AS (SELECT t1.osm_id, CASE WHEN COUNT(cg.cid) = 0 THEN ARRAY[1000]::INTEGER[] ELSE ARRAY_AGG(COALESCE(cg.cid, 1000)) END AS aa_fids FROM t1 LEFT JOIN countries cg ON ST_Intersects(t1.geom, cg.geometry) GROUP BY t1.osm_id) UPDATE relations uw SET country = t2.aa_fids FROM t2 WHERE t2.osm_id = uw.osm_id;

src/app.py

@@ -597,7 +597,7 @@ def get_grid_id(geom, cur):
         if backend_match:
             countries = backend_match[0]
             country_export = True
-            logging.debug(f"Using Country Export Mode with id : {countries[0]}")
+            logging.info(f"Using Country Export Mode with id : {countries[0]}")
         # else:
         #     if int(geom_area) > int(index_threshold):
         #         # this will be applied only when polygon gets bigger we will be slicing index size to search