Merge pull request #5 from UoA-eResearch/IDS-952-decide-and-implement…

…-persistence

IDS-952 Handle and store archiving request

pyproject.toml

@@ -7,6 +7,8 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = "^3.11"
 fastapi = {extras = ["standard"], version = "^0.115.0"}
+sqlmodel = "^0.0.22"
+sqlalchemy = "^2.0.36"
 
 
 [tool.poetry.group.dev.dependencies]

src/api/main.py

@@ -1,14 +1,60 @@
 """Definition of endpoints/routers for the webserver."""
 
 import re
-from typing import Annotated, Any
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Annotated, AsyncGenerator, Iterable
 
-from fastapi import FastAPI, Security
+from fastapi import Depends, FastAPI, Security
 from pydantic.functional_validators import AfterValidator
+from sqlalchemy.exc import IntegrityError
+from sqlmodel import Session, SQLModel, create_engine
 
 from api.security import ApiKey, validate_api_key, validate_permissions
+from models.member import Member
+from models.person import Person
+from models.project import InputProject, Project
+from models.role import prepopulate_roles
+from models.services import ResearchDriveService, Services
 
-app = FastAPI()
+# Ensure driveoff directory is created
+(Path.home() / ".driveoff").mkdir(exist_ok=True)
+DB_FILE_NAME = Path.home() / ".driveoff" / "database.db"
+DB_URL = f"sqlite:///{DB_FILE_NAME}"
+
+connect_args = {"check_same_thread": False}
+engine = create_engine(DB_URL, connect_args=connect_args, echo=True)
+
+
+def create_db_and_tables() -> None:
+    """Create database structure and pre-populate with fixtures."""
+    SQLModel.metadata.create_all(engine)
+    with Session(engine) as session:
+        roles = prepopulate_roles()
+        session.add_all(roles)
+        try:
+            session.commit()
+        except IntegrityError:
+            pass  # Roles already inserted, skip.
+
+
+def get_session() -> Iterable[Session]:
+    """Return a Session object."""
+    with Session(engine) as session:
+        yield session
+
+
+SessionDep = Annotated[Session, Depends(get_session)]
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
+    """Lifecycle method for the API"""
+    create_db_and_tables()
+    yield
+
+
+app = FastAPI(lifespan=lifespan)
 
 
 RESEARCH_DRIVE_REGEX = re.compile(r"res[a-z]{3}[0-9]{9}-[a-zA-Z0-9-_]+")
@@ -29,18 +75,45 @@ def validate_resdrive_identifier(drive_id: str) -> str:
 
 @app.post(ENDPOINT_PREFIX + "/resdriveinfo")
 async def set_drive_info(
-    drive_id: ResearchDriveID,
-    ro_crate_metadata: dict[str, Any],
+    input_project: InputProject,
+    session: SessionDep,
     api_key: ApiKey = Security(validate_api_key),
-) -> dict[str, str]:
+) -> Project:
     """Submit initial RO-Crate metadata. NOTE: this may also need to accept the manifest data."""
-
     validate_permissions("POST", api_key)
-
-    _ = ro_crate_metadata
-    return {
-        "message": f"Received RO-Crate metadata for {drive_id}.",
-    }
+    project = Project(
+        id=input_project.id,
+        title=input_project.title,
+        description=input_project.description,
+        division=input_project.division,
+        start_date=input_project.start_date,
+        end_date=input_project.end_date,
+        codes=input_project.codes,
+    )
+    # Break up role and person information.
+    members: list[Member] = []
+    for input_member in input_project.members:
+        person = Person(
+            id=input_member.id,
+            email=input_member.email,
+            full_name=input_member.full_name,
+            username=input_member.identities.items[0].username,
+        )
+        member = Member(project=project, person=person, role_id=input_member.role.id)
+        members.append(member)
+    # Add the drive info into services.
+    drives = [
+        ResearchDriveService.model_validate(drive)
+        for drive in input_project.services.research_drive
+    ]
+    stored_services = Services(research_drive=drives)
+    # Add the validated services and members into the project
+    project.services = stored_services
+    project.members = members
+    # Upsert the project.
+    session.merge(project)
+    session.commit()
+    return project
 
 
 @app.put(ENDPOINT_PREFIX + "/resdriveinfo")
@@ -62,6 +135,7 @@ async def append_drive_info(
 @app.get(ENDPOINT_PREFIX + "/resdriveinfo")
 async def get_drive_info(
     drive_id: ResearchDriveID,
+    # session: SessionDep,
     api_key: ApiKey = Security(validate_api_key),
 ) -> dict[str, str]:
     """Retrieve information about the specified Research Drive."""

src/api/tests/test_requests.http

@@ -29,3 +29,116 @@ GET {{server_url}}/api/v1/resdriveinfo
 x-api-key: {{api_key}}
 
 ###
+
+POST http://localhost:8000/api/v1/resdriveinfo
+    ?drive_id={{drive_id}}
+x-api-key: {{api_key}}
+
+{
+    "id": 383,
+    "title": "Monteith Research Art Archive",
+    "description": "<span>The works often explore the political dimensions of culture engaged in turmoil over land ownership, history and occupation. Her work traverses political movements, contemporary sports, culture and social activities. Alex\u2019s projects often take place in large-scale or extreme geographies. Her surfing-related projects connect museum spaces directly to local geography through participatory performance projects.<br><br></span>An archive of UofA funded visual art video, performance and photo work I have produced since in appointment in 2008. Bring together 2008 to 2014 archive already held on tape backup at UofA, with current research projects, to make the older work accessible to me for re-publishing, and to back-up currently exposed research created from 2014 to 2018.",
+    "division": "CAI",
+    "codes": [
+        {
+            "code": "cer00383",
+            "href": "/project/383/code/766",
+            "id": 766
+        },
+        {
+            "code": "rescai201800001",
+            "href": "/project/383/code/767",
+            "id": 767
+        }
+    ],
+    "status": {
+        "href": "/projectstatus/1",
+        "id": 1,
+        "name": "Open"
+    },
+    "start_date": "2018-04-09",
+    "end_date": "2023-04-09",
+    "next_review_date": "2023-04-09",
+    "last_modified": "2022-11-16T00:18:14Z",
+    "requirements": "",
+    "services": {
+        "dropbox": [],
+        "href": "/project/383/service",
+        "mytardis": [],
+        "nectar": [],
+        "research_drive": [
+            {
+                "allocated_gb": 25600.0,
+                "archived": 0,
+                "date": "2024-10-13",
+                "deleted": 0,
+                "first_day": "2018-04-09",
+                "free_gb": 24894.5,
+                "id": 138,
+                "last_day": null,
+                "name": "rescai201800001-MonteithResearchArt",
+                "num_files": 50102,
+                "percentage_used": 2.75578,
+                "project_code": "rescai201800001",
+                "used_gb": 705.479
+            }
+        ],
+        "uoaivm": [],
+        "vis": [],
+        "vm": []
+    },
+    "members": [
+        {
+            "id": 1421,
+            "person.email": "[email protected]",
+            "person.full_name": "Alex Monteith",
+            "person.identities": {
+                "href": "/person/120/identity",
+                "items": [
+                    {
+                        "href": "/person/120/identity/235",
+                        "id": 235,
+                        "username": "amon011"
+                    }
+                ]
+            },
+            "person.status": {
+                "href": "/personstatus/1",
+                "id": 1,
+                "name": "Active"
+            },
+            "role": {
+                "href": "/personrole/1",
+                "id": 1,
+                "name": "Project Owner"
+            },
+            "notes": ""
+        },
+        {
+            "id": 1420,
+            "person.email": "[email protected]",
+            "person.full_name": "Yvette Wharton",
+            "person.identities": {
+                "href": "/person/12/identity",
+                "items": [
+                    {
+                        "href": "/person/12/identity/20",
+                        "id": 20,
+                        "username": "ywha001"
+                    }
+                ]
+            },
+            "person.status": {
+                "href": "/personstatus/1",
+                "id": 1,
+                "name": "Active"
+            },
+            "role": {
+                "href": "/personrole/7",
+                "id": 7,
+                "name": "Primary Adviser"
+            },
+            "notes": ""
+        }
+    ]
+}

src/models/member.py

@@ -0,0 +1,23 @@
+"""Data models representing project members - person in a project and their role."""
+
+from sqlmodel import Field, Relationship, SQLModel
+
+from models.person import Person
+from models.project import Project
+from models.role import Role
+
+
+class Member(SQLModel, table=True):
+    """Linking table between projects, people and their roles."""
+
+    project_id: int | None = Field(
+        default=None, foreign_key="project.id", primary_key=True
+    )
+    person_id: int | None = Field(
+        default=None, foreign_key="person.id", primary_key=True
+    )
+    role_id: int | None = Field(default=None, foreign_key="role.id", primary_key=True)
+
+    role: "Role" = Relationship()
+    project: "Project" = Relationship(back_populates="members")
+    person: "Person" = Relationship()

src/models/person.py

@@ -0,0 +1,39 @@
+"""Data models representing people and their identities."""
+
+from typing import Optional
+
+from sqlmodel import Field, SQLModel
+
+from models.role import Role
+
+
+class InputIdentity(SQLModel):
+    """Data class for the identity list in POST request."""
+
+    username: str
+
+
+class InputIdentityResultItems(SQLModel):
+    """The set of result items from Project DB API."""
+
+    href: str
+    items: list[InputIdentity]
+
+
+class InputPerson(SQLModel):
+    "Data class for a Person model in POST request."
+    id: Optional[int] = Field(default=None, primary_key=True)
+    email: Optional[str] = Field(schema_extra={"validation_alias": "person.email"})
+    full_name: str = Field(schema_extra={"validation_alias": "person.full_name"})
+    identities: InputIdentityResultItems = Field(
+        schema_extra={"validation_alias": "person.identities"}
+    )
+    role: Role
+
+
+class Person(SQLModel, table=True):
+    "Data class for a Person model in database."
+    id: Optional[int] = Field(default=None, primary_key=True)
+    email: Optional[str]
+    full_name: str
+    username: str

src/models/project.py

@@ -0,0 +1,62 @@
+"""Data models representing projects."""
+
+from datetime import datetime
+from typing import TYPE_CHECKING, Optional
+
+from sqlmodel import Field, Relationship, SQLModel
+
+from models.person import InputPerson
+from models.services import InputServices, Services
+
+# Only import Member during typechecking to prevent circular dependency error.
+if TYPE_CHECKING:
+    from models.member import Member
+
+
+class Code(SQLModel, table=True):
+    """Model for project codes."""
+
+    id: Optional[int] = Field(primary_key=True)
+    code: str
+
+
+class BaseProject(SQLModel):
+    """Base model for describing a project."""
+
+    title: str
+    description: str
+    division: str
+    start_date: datetime
+    end_date: datetime
+
+
+class InputProject(BaseProject):
+    """Input project model for data received from POST"""
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    members: list[InputPerson]
+    codes: list[Code]
+    services: InputServices
+
+
+class ProjectCodeLink(SQLModel, table=True):
+    """Linking table between project and codes"""
+
+    code_id: int | None = Field(default=None, foreign_key="code.id", primary_key=True)
+    project_id: int | None = Field(
+        default=None, foreign_key="project.id", primary_key=True
+    )
+
+
+class Project(BaseProject, table=True):
+    """Project model for data stored in database"""
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    services_id: int | None = Field(default=None, foreign_key="services.id")
+    codes: list[Code] = Relationship(link_model=ProjectCodeLink)
+    services: Services = Relationship()
+    members: list["Member"] = Relationship(
+        # cascade_delete enabled so session.merge() works for project save.
+        back_populates="project",
+        cascade_delete=True,
+    )