add input data route + tests (#340)

* add status head route + tests

* isort

* only update if date is greater than current time

* udpate

* use temp file

requirements.txt

@@ -17,3 +17,4 @@ structlog
 sentry-sdk
 slowapi
 pathy==0.10.3
+fsspec

src/status.py

@@ -3,9 +3,14 @@
 import os
 from datetime import datetime, timedelta, timezone
 
+import fsspec
 import structlog
 from fastapi import APIRouter, Depends, HTTPException, Request
-from nowcasting_datamodel.models import ForecastSQL, Status
+from nowcasting_datamodel.models import ForecastSQL, GSPYieldSQL, Status
+from nowcasting_datamodel.read.read import (
+    get_latest_input_data_last_updated,
+    update_latest_input_data_last_updated,
+)
 from sqlalchemy.exc import NoResultFound
 from sqlalchemy.orm.session import Session
 
@@ -63,3 +68,55 @@ def check_last_forecast(request: Request, session: Session = Depends(get_session
 
     logger.debug(f"Last forecast time was {forecast.forecast_creation_time}")
     return forecast.forecast_creation_time
+
+
+@router.get("/update_last_data", include_in_schema=False)
+@limiter.limit(f"{N_CALLS_PER_HOUR}/hour")
+def update_last_data(
+    request: Request, component: str, file: str = None, session: Session = Depends(get_session)
+) -> datetime:
+    """Update InputDataLastUpdatedSQL table"""
+
+    save_api_call_to_db(session=session, request=request)
+
+    assert component in ["gsp", "nwp", "satellite"]
+
+    logger.debug("Check to see when the last forecast run was ")
+
+    if component == "gsp":
+        # get last gsp yield in database
+        query = session.query(GSPYieldSQL)
+        query = query.order_by(GSPYieldSQL.created_utc.desc())
+        query = query.limit(1)
+        try:
+            gsp = query.one()
+        except NoResultFound:
+            raise HTTPException(status_code=404, detail="There are no gsp yields")
+
+        modified_date = gsp.created_utc
+
+    elif component in ["nwp", "satellite"]:
+
+        assert file is not None
+
+        # get modified date, this will probably be in s3
+        fs = fsspec.open(file).fs
+        modified_date = fs.modified(file)
+
+    # get last value
+    latest_input_data = get_latest_input_data_last_updated(session=session)
+
+    update = True
+    if latest_input_data is not None:
+        if hasattr(latest_input_data, component):
+            current_date = getattr(latest_input_data, component)
+            if current_date >= modified_date:
+                update = False
+
+    if update:
+        # update the database
+        update_latest_input_data_last_updated(
+            session=session, component=component, update_datetime=modified_date
+        )
+
+    return modified_date

src/tests/test_status.py

@@ -1,10 +1,22 @@
 """ Test for main app """
 
+import os
+import tempfile
 from datetime import datetime, timedelta, timezone
 
+import fsspec
 from fastapi.testclient import TestClient
 from freezegun import freeze_time
-from nowcasting_datamodel.models import APIRequestSQL, ForecastSQL, Status, UserSQL
+from nowcasting_datamodel.models import (
+    APIRequestSQL,
+    ForecastSQL,
+    GSPYield,
+    InputDataLastUpdatedSQL,
+    Location,
+    LocationSQL,
+    Status,
+    UserSQL,
+)
 
 from database import get_session
 from main import app
@@ -63,3 +75,66 @@ def test_check_last_forecast_error(db_session):
 
     response = client.get("/v0/solar/GB/check_last_forecast_run")
     assert response.status_code == 404
+
+
+@freeze_time("2023-01-03")
+def test_check_last_forecast_gsp(db_session):
+    """Check check_last_forecast_run works fine"""
+
+    gsp_yield_1 = GSPYield(datetime_utc=datetime(2022, 1, 2), solar_generation_kw=1)
+    gsp_yield_1_sql = gsp_yield_1.to_orm()
+
+    gsp_sql_1: LocationSQL = Location(
+        gsp_id=0, label="national", status_interval_minutes=5
+    ).to_orm()
+    gsp_yield_1_sql.location = gsp_sql_1
+
+    # add to database
+    db_session.add_all([gsp_yield_1_sql, gsp_sql_1])
+
+    app.dependency_overrides[get_session] = lambda: db_session
+
+    response = client.get("/v0/solar/GB/update_last_data?component=gsp")
+    assert response.status_code == 200, response.text
+
+    data = db_session.query(InputDataLastUpdatedSQL).all()
+    assert len(data) == 1
+    assert data[0].gsp.isoformat() == datetime(2023, 1, 3, tzinfo=timezone.utc).isoformat()
+
+    # check no updates is made, as file modified datetime is the same
+    response = client.get("/v0/solar/GB/update_last_data?component=gsp")
+    assert response.status_code == 200, response.text
+    data = db_session.query(InputDataLastUpdatedSQL).all()
+    assert len(data) == 1
+
+
+def test_check_last_forecast_file(db_session):
+    """Check check_last_forecast_run works fine"""
+
+    gsp_yield_1 = GSPYield(datetime_utc=datetime(2022, 1, 2), solar_generation_kw=1)
+    gsp_yield_1_sql = gsp_yield_1.to_orm()
+
+    gsp_sql_1: LocationSQL = Location(
+        gsp_id=0, label="national", status_interval_minutes=5
+    ).to_orm()
+    gsp_yield_1_sql.location = gsp_sql_1
+
+    # add to database
+    db_session.add_all([gsp_yield_1_sql, gsp_sql_1])
+
+    app.dependency_overrides[get_session] = lambda: db_session
+
+    # create temp file
+    with tempfile.TemporaryDirectory() as tmp:
+        filename = os.path.join(tmp, "text.txt")
+        with open(filename, "w") as f:
+            f.write("test")
+        fs = fsspec.open(filename).fs
+        modified_date = fs.modified(filename)
+
+        response = client.get(f"/v0/solar/GB/update_last_data?component=nwp&file={filename}")
+        assert response.status_code == 200
+
+        data = db_session.query(InputDataLastUpdatedSQL).all()
+        assert len(data) == 1
+        assert data[0].nwp.isoformat() == modified_date.isoformat()