diff --git a/requirements.txt b/requirements.txt index 3b9e40e..3f9eb73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ structlog sentry-sdk slowapi pathy==0.10.3 +fsspec diff --git a/src/status.py b/src/status.py index 867fedd..c78eb23 100644 --- a/src/status.py +++ b/src/status.py @@ -3,9 +3,14 @@ import os from datetime import datetime, timedelta, timezone +import fsspec import structlog from fastapi import APIRouter, Depends, HTTPException, Request -from nowcasting_datamodel.models import ForecastSQL, Status +from nowcasting_datamodel.models import ForecastSQL, GSPYieldSQL, Status +from nowcasting_datamodel.read.read import ( + get_latest_input_data_last_updated, + update_latest_input_data_last_updated, +) from sqlalchemy.exc import NoResultFound from sqlalchemy.orm.session import Session @@ -63,3 +68,55 @@ def check_last_forecast(request: Request, session: Session = Depends(get_session logger.debug(f"Last forecast time was {forecast.forecast_creation_time}") return forecast.forecast_creation_time + + +@router.get("/update_last_data", include_in_schema=False) +@limiter.limit(f"{N_CALLS_PER_HOUR}/hour") +def update_last_data( + request: Request, component: str, file: str = None, session: Session = Depends(get_session) +) -> datetime: + """Update InputDataLastUpdatedSQL table""" + + save_api_call_to_db(session=session, request=request) + + assert component in ["gsp", "nwp", "satellite"] + + logger.debug("Check to see when the last forecast run was ") + + if component == "gsp": + # get last gsp yield in database + query = session.query(GSPYieldSQL) + query = query.order_by(GSPYieldSQL.created_utc.desc()) + query = query.limit(1) + try: + gsp = query.one() + except NoResultFound: + raise HTTPException(status_code=404, detail="There are no gsp yields") + + modified_date = gsp.created_utc + + elif component in ["nwp", "satellite"]: + + assert file is not None + + # get modified date, this will probably be in s3 + fs = fsspec.open(file).fs + modified_date = fs.modified(file) + + # get last value + latest_input_data = get_latest_input_data_last_updated(session=session) + + update = True + if latest_input_data is not None: + if hasattr(latest_input_data, component): + current_date = getattr(latest_input_data, component) + if current_date >= modified_date: + update = False + + if update: + # update the database + update_latest_input_data_last_updated( + session=session, component=component, update_datetime=modified_date + ) + + return modified_date diff --git a/src/tests/test_status.py b/src/tests/test_status.py index 87e0823..fd26e35 100644 --- a/src/tests/test_status.py +++ b/src/tests/test_status.py @@ -1,10 +1,22 @@ """ Test for main app """ +import os +import tempfile from datetime import datetime, timedelta, timezone +import fsspec from fastapi.testclient import TestClient from freezegun import freeze_time -from nowcasting_datamodel.models import APIRequestSQL, ForecastSQL, Status, UserSQL +from nowcasting_datamodel.models import ( + APIRequestSQL, + ForecastSQL, + GSPYield, + InputDataLastUpdatedSQL, + Location, + LocationSQL, + Status, + UserSQL, +) from database import get_session from main import app @@ -63,3 +75,66 @@ def test_check_last_forecast_error(db_session): response = client.get("/v0/solar/GB/check_last_forecast_run") assert response.status_code == 404 + + +@freeze_time("2023-01-03") +def test_check_last_forecast_gsp(db_session): + """Check check_last_forecast_run works fine""" + + gsp_yield_1 = GSPYield(datetime_utc=datetime(2022, 1, 2), solar_generation_kw=1) + gsp_yield_1_sql = gsp_yield_1.to_orm() + + gsp_sql_1: LocationSQL = Location( + gsp_id=0, label="national", status_interval_minutes=5 + ).to_orm() + gsp_yield_1_sql.location = gsp_sql_1 + + # add to database + db_session.add_all([gsp_yield_1_sql, gsp_sql_1]) + + app.dependency_overrides[get_session] = lambda: db_session + + response = client.get("/v0/solar/GB/update_last_data?component=gsp") + assert response.status_code == 200, response.text + + data = db_session.query(InputDataLastUpdatedSQL).all() + assert len(data) == 1 + assert data[0].gsp.isoformat() == datetime(2023, 1, 3, tzinfo=timezone.utc).isoformat() + + # check no updates is made, as file modified datetime is the same + response = client.get("/v0/solar/GB/update_last_data?component=gsp") + assert response.status_code == 200, response.text + data = db_session.query(InputDataLastUpdatedSQL).all() + assert len(data) == 1 + + +def test_check_last_forecast_file(db_session): + """Check check_last_forecast_run works fine""" + + gsp_yield_1 = GSPYield(datetime_utc=datetime(2022, 1, 2), solar_generation_kw=1) + gsp_yield_1_sql = gsp_yield_1.to_orm() + + gsp_sql_1: LocationSQL = Location( + gsp_id=0, label="national", status_interval_minutes=5 + ).to_orm() + gsp_yield_1_sql.location = gsp_sql_1 + + # add to database + db_session.add_all([gsp_yield_1_sql, gsp_sql_1]) + + app.dependency_overrides[get_session] = lambda: db_session + + # create temp file + with tempfile.TemporaryDirectory() as tmp: + filename = os.path.join(tmp, "text.txt") + with open(filename, "w") as f: + f.write("test") + fs = fsspec.open(filename).fs + modified_date = fs.modified(filename) + + response = client.get(f"/v0/solar/GB/update_last_data?component=nwp&file={filename}") + assert response.status_code == 200 + + data = db_session.query(InputDataLastUpdatedSQL).all() + assert len(data) == 1 + assert data[0].nwp.isoformat() == modified_date.isoformat()