Merge pull request #10 from hotosm/setup-encode-databases

Setup encode databases

src/backend/app/db/database.py

@@ -1,25 +1,49 @@
 """Config for the DTM database connection."""
-
+from databases import Database
+from app.config import settings
 from sqlalchemy import create_engine
 from sqlalchemy.orm import declarative_base, sessionmaker
 
-from app.config import settings
-
-engine = create_engine(
-    settings.DTM_DB_URL.unicode_string(),
-    pool_size=20,
-    max_overflow=-1,
-)
-SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
-
 Base = declarative_base()
-DtmMetadata = Base.metadata
 
-
-def get_db():
-    """Create SQLAlchemy DB session."""
-    db = SessionLocal()
+class DatabaseConnection:
+  """Manages database connection (sqlalchemy & encode databases)"""
+  def __init__(self):
+    self.database = Database(settings.DTM_DB_URL.unicode_string(), min_size=5, max_size=20)
+    # self.database = Database(settings.DTM_DB_URL.unicode_string())
+    self.engine = create_engine(
+        settings.DTM_DB_URL.unicode_string(),
+        pool_size=20,
+        max_overflow=-1,
+    )
+    self.SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=self.engine)
+
+  async def connect(self):
+    """Connect to the database."""
+    await self.database.connect()
+
+  async def disconnect(self):
+    """Disconnect from the database."""
+    await self.database.disconnect()
+
+  def create_db_session(self):
+    """Create a new SQLAlchemy DB session."""
+    db = self.SessionLocal()
     try:
-        yield db
+      return db
     finally:
-        db.close()
+      db.close()
+
+db_connection = DatabaseConnection()  # Create a single instance
+
+def get_db():
+  """Yield a new database session."""
+  return db_connection.create_db_session()
+
+async def encode_db():
+  """Get the encode database connection"""
+  try:
+    await db_connection.connect()
+    yield db_connection.database
+  finally:
+    await db_connection.disconnect()

src/backend/app/projects/project_crud.py

@@ -7,12 +7,51 @@
 import shapely.wkb as wkblib
 from shapely.geometry import shape
 from fastapi import HTTPException
-from app.utils import geometry_to_geojson, merge_multipolygon
+from app.utils import geometry_to_geojson, merge_multipolygon, str_to_geojson
 from fmtm_splitter.splitter import split_by_square
 from fastapi.concurrency import run_in_threadpool
 from app.db import database
 from fastapi import Depends
 from asyncio import gather
+from databases import Database
+
+async def create_project_with_project_info(db: Database, project_metadata: project_schemas.ProjectIn):
+    """Create a project in database."""
+    query = f"""
+    INSERT INTO projects (
+        author_id, name, short_description, description, per_task_instructions, status, visibility, mapper_level, priority, outline, created
+    )
+    VALUES (
+        1, 
+        '{project_metadata.name}', 
+        '{project_metadata.short_description}', 
+        '{project_metadata.description}', 
+        '{project_metadata.per_task_instructions}', 
+        'DRAFT', 
+        'PUBLIC', 
+        'INTERMEDIATE', 
+        'MEDIUM', 
+        '{str(project_metadata.outline)}', 
+        CURRENT_TIMESTAMP
+    )
+    RETURNING id
+    """
+    new_project_id = await db.execute(query)
+
+    if not new_project_id:
+        raise HTTPException(
+            status_code=500,
+            detail="Project could not be created"
+        )
+    # Fetch the newly created project using the returned ID
+    select_query = f"""
+        SELECT id, name, short_description, description, per_task_instructions, outline
+        FROM projects
+        WHERE id = '{new_project_id}'
+    """
+    new_project = await db.fetch_one(query=select_query)
+    return new_project
+
 
 
 async def get_project_by_id(
@@ -26,41 +65,37 @@ async def get_project_by_id(
     )
     return await convert_to_app_project(db_project)
 
-
-async def convert_to_app_project(db_project: db_models.DbProject):
-    """Legacy function to convert db models --> Pydantic.
-
-    TODO refactor to use Pydantic model methods instead.
-    """
-    if not db_project:
-        log.debug("convert_to_app_project called, but no project provided")
-        return None
-
-    app_project = db_project
-
-    if db_project.outline:
-        app_project.outline_geojson = geometry_to_geojson(
-            db_project.outline, {"id": db_project.id}, db_project.id
-        )
-    app_project.tasks = db_project.tasks
-    return app_project
-
-
 async def get_projects(
-    db: Session,
+    db:Database,
     skip: int = 0,
     limit: int = 100,
 ):
     """Get all projects."""
-    db_projects = (
-        db.query(db_models.DbProject)
-        .order_by(db_models.DbProject.id.desc())
-        .offset(skip)
-        .limit(limit)
-        .all()
-    )
-    project_count = db.query(db_models.DbProject).count()
-    return project_count, await convert_to_app_projects(db_projects)
+    raw_sql = """
+        SELECT id, name, short_description, description, per_task_instructions, outline
+        FROM projects 
+        ORDER BY id DESC 
+        OFFSET :skip 
+        LIMIT :limit;
+        """
+    db_projects = await db.fetch_all(raw_sql, {'skip': skip, 'limit': limit})
+    return  await convert_to_app_projects(db_projects)
+
+# async def get_projects(
+#     db: Session,
+#     skip: int = 0,
+#     limit: int = 100,
+# ):
+#     """Get all projects."""
+#     db_projects = (
+#         db.query(db_models.DbProject)
+#         .order_by(db_models.DbProject.id.desc())
+#         .offset(skip)
+#         .limit(limit)
+#         .all()
+#     )
+#     project_count = db.query(db_models.DbProject).count()
+#     return project_count, await convert_to_app_projects(db_projects)
 
 
 async def convert_to_app_projects(
@@ -82,22 +117,22 @@ async def convert_project(project):
     else:
         return []
 
+async def convert_to_app_project(db_project: db_models.DbProject):
+    """Legacy function to convert db models --> Pydantic."""
+    if not db_project:
+        log.debug("convert_to_app_project called, but no project provided")
+        return None
+    app_project = db_project
 
-async def create_project_with_project_info(
-    db: Session, project_metadata: project_schemas.ProjectIn
-):
-    """Create a project in database."""
-    db_project = db_models.DbProject(
-        author_id=1, **project_metadata.model_dump(exclude=["outline_geojson"])
-    )
-    db.add(db_project)
-    db.commit()
-    db.refresh(db_project)
-    return db_project
-
+    if db_project.outline:
+
+        app_project.outline_geojson = str_to_geojson(
+            db_project.outline, {"id": db_project.id}, db_project.id
+        )
+    return app_project
 
 async def create_tasks_from_geojson(
-    db: Session,
+    db: Database,
     project_id: int,
     boundaries: str,
 ):
@@ -119,26 +154,18 @@ async def create_tasks_from_geojson(
                 polygon["geometry"]["type"] = "Polygon"
                 polygon["geometry"]["coordinates"] = polygon["geometry"]["coordinates"][
                     0
-                ]
-
-            db_task = db_models.DbTask(
-                project_id=project_id,
-                outline=wkblib.dumps(shape(polygon["geometry"]), hex=True),
-                project_task_index=index + 1,
-            )
-            db.add(db_task)
-            log.debug(
-                "Created database task | "
-                f"Project ID {project_id} | "
-                f"Task index {index}"
-            )
-
-        # Commit all tasks and update project location in db
-        db.commit()
-
-        log.debug("COMPLETE: creating project boundary, based on task boundaries")
-
-        return True
+                ]                  
+            query = f""" INSERT INTO tasks (project_id,outline,project_task_index) VALUES ( '{project_id}', '{wkblib.dumps(shape(polygon["geometry"]), hex=True)}', '{index + 1}');"""
+
+            result = await db.execute(query)
+            if result:
+                log.debug(
+                    "Created database task | "
+                    f"Project ID {project_id} | "
+                    f"Task index {index}"
+                )
+                log.debug("COMPLETE: creating project boundary, based on task boundaries")
+                return True
     except Exception as e:
         log.exception(e)
         raise HTTPException(e) from e

src/backend/app/projects/project_routes.py

@@ -13,9 +13,9 @@
 from app.utils import multipolygon_to_polygon
 from app.s3 import s3_client
 from app.config import settings
+from databases import Database
 from app.db import db_models
 
-
 router = APIRouter(
     prefix=f"{settings.API_PREFIX}/projects",
     responses={404: {"description": "Not found"}},
@@ -66,29 +66,22 @@ def delete_project_by_id(project_id: int, db: Session = Depends(database.get_db)
 )
 async def create_project(
     project_info: project_schemas.ProjectIn,
-    db: Session = Depends(database.get_db),
+    db: Database = Depends(database.encode_db),
 ):
     """Create a project in  database."""
-
-    log.info(
-        f"Attempting creation of project "
-        f"{project_info.name} in organisation {project_info.organisation_id}"
-    )
-
     project = await project_crud.create_project_with_project_info(db, project_info)
     if not project:
         raise HTTPException(
             status_code=HTTPStatus.BAD_REQUEST, detail="Project creation failed"
         )
-
     return project
 
 
 @router.post("/{project_id}/upload-task-boundaries", tags=["Projects"])
 async def upload_project_task_boundaries(
     project_id: int,
     task_geojson: UploadFile = File(...),
-    db: Session = Depends(database.get_db),
+    db: Database = Depends(database.encode_db),
 ):
     """Set project task boundaries using split GeoJSON from frontend.
 
@@ -100,8 +93,7 @@ async def upload_project_task_boundaries(
 
     Returns:
         dict: JSON containing success message, project ID, and number of tasks.
-    """
-
+    """    
     # read entire file
     content = await task_geojson.read()
     task_boundaries = json.loads(content)
@@ -173,10 +165,10 @@ async def generate_presigned_url(data: project_schemas.PresignedUrlRequest):
 async def read_projects(
     skip: int = 0,
     limit: int = 100,
-    db: Session = Depends(database.get_db),
+    db: Database = Depends(database.encode_db)
 ):
-    """Return all projects."""
-    total_count, projects = await project_crud.get_projects(db, skip, limit)
+    "Return all projects"
+    projects = await project_crud.get_projects(db, skip, limit)
     return projects
 
 

src/backend/app/projects/project_schemas.py

@@ -8,14 +8,15 @@
     geojson_to_geometry,
     read_wkb,
     merge_multipolygon,
+    str_to_geojson,
     write_wkb,
     geometry_to_geojson,
 )
 
 
 class ProjectInfo(BaseModel):
     """Basic project info."""
-
+    id: int
     name: str
     short_description: str
     description: str
@@ -69,9 +70,11 @@ def outline_geojson(self) -> Optional[Feature]:
         """Compute the geojson outline from WKBElement outline."""
         if not self.outline:
             return None
-        geometry = wkb.loads(bytes(self.outline.data))
-        bbox = geometry.bounds  # Calculate bounding box
-        return geometry_to_geojson(self.outline, {"id": self.id, "bbox": bbox}, self.id)
+        wkb_data = bytes.fromhex(self.outline)
+        geom = wkb.loads(wkb_data)
+        # geometry = wkb.loads(bytes(self.outline.data))
+        bbox = geom.bounds  # Calculate bounding box
+        return str_to_geojson(self.outline, {"id": self.id, "bbox": bbox}, self.id)
 
 
 class PresignedUrlRequest(BaseModel):