From e67c25da6150b8063c6891ba68bd056b70bc2cac Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Wed, 29 May 2024 18:10:22 +0100 Subject: [PATCH 1/7] add status head route + tests --- requirements.txt | 1 + src/status.py | 45 ++++++++++++++++++++++++++- src/tests/test_status.py | 67 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 111 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3b9e40e..3f9eb73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ structlog sentry-sdk slowapi pathy==0.10.3 +fsspec diff --git a/src/status.py b/src/status.py index 867fedd..0a8c53e 100644 --- a/src/status.py +++ b/src/status.py @@ -3,9 +3,11 @@ import os from datetime import datetime, timedelta, timezone +import fsspec import structlog from fastapi import APIRouter, Depends, HTTPException, Request -from nowcasting_datamodel.models import ForecastSQL, Status +from nowcasting_datamodel.models import ForecastSQL, GSPYieldSQL, Status +from nowcasting_datamodel.read.read import update_latest_input_data_last_updated from sqlalchemy.exc import NoResultFound from sqlalchemy.orm.session import Session @@ -63,3 +65,44 @@ def check_last_forecast(request: Request, session: Session = Depends(get_session logger.debug(f"Last forecast time was {forecast.forecast_creation_time}") return forecast.forecast_creation_time + + +@router.get("/update_last_data", include_in_schema=False) +@limiter.limit(f"{N_CALLS_PER_HOUR}/hour") +def update_last_data( + request: Request, component: str, file: str = None, session: Session = Depends(get_session) +) -> datetime: + """Update InputDataLastUpdatedSQL table""" + + save_api_call_to_db(session=session, request=request) + + assert component in ["gsp", "nwp", "satellite"] + + logger.debug("Check to see when the last forecast run was ") + + if component == "gsp": + # get last gsp yield in database + query = session.query(GSPYieldSQL) + query = query.order_by(GSPYieldSQL.created_utc.desc()) + query = query.limit(1) + try: + gsp = query.one() + except NoResultFound: + raise HTTPException(status_code=404, detail="There are no gsp yields") + + modified_date = gsp.created_utc + + elif component in ["nwp", "satellite"]: + + assert file is not None + + # get modified date, this will probably be in s3 + fs = fsspec.open(file).fs + modified_date = fs.modified(file) + + # update the database + update_latest_input_data_last_updated( + session=session, component=component, update_datetime=modified_date + ) + + return modified_date diff --git a/src/tests/test_status.py b/src/tests/test_status.py index 87e0823..c6e94f2 100644 --- a/src/tests/test_status.py +++ b/src/tests/test_status.py @@ -2,9 +2,19 @@ from datetime import datetime, timedelta, timezone +import fsspec from fastapi.testclient import TestClient from freezegun import freeze_time -from nowcasting_datamodel.models import APIRequestSQL, ForecastSQL, Status, UserSQL +from nowcasting_datamodel.models import ( + APIRequestSQL, + ForecastSQL, + GSPYield, + Location, + LocationSQL, + InputDataLastUpdatedSQL, + Status, + UserSQL, +) from database import get_session from main import app @@ -63,3 +73,58 @@ def test_check_last_forecast_error(db_session): response = client.get("/v0/solar/GB/check_last_forecast_run") assert response.status_code == 404 + + +@freeze_time("2023-01-03") +def test_check_last_forecast_gsp(db_session): + """Check check_last_forecast_run works fine""" + + gsp_yield_1 = GSPYield(datetime_utc=datetime(2022, 1, 2), solar_generation_kw=1) + gsp_yield_1_sql = gsp_yield_1.to_orm() + + gsp_sql_1: LocationSQL = Location( + gsp_id=0, label="national", status_interval_minutes=5 + ).to_orm() + gsp_yield_1_sql.location = gsp_sql_1 + + # add to database + db_session.add_all([gsp_yield_1_sql, gsp_sql_1]) + + app.dependency_overrides[get_session] = lambda: db_session + + response = client.get("/v0/solar/GB/update_last_data?component=gsp") + assert response.status_code == 200, response.text + + data = db_session.query(InputDataLastUpdatedSQL).all() + assert len(data) == 1 + assert data[0].gsp.isoformat() == datetime(2023, 1, 3, tzinfo=timezone.utc).isoformat() + + +def test_check_last_forecast_file(db_session): + """Check check_last_forecast_run works fine""" + + gsp_yield_1 = GSPYield(datetime_utc=datetime(2022, 1, 2), solar_generation_kw=1) + gsp_yield_1_sql = gsp_yield_1.to_orm() + + gsp_sql_1: LocationSQL = Location( + gsp_id=0, label="national", status_interval_minutes=5 + ).to_orm() + gsp_yield_1_sql.location = gsp_sql_1 + + # add to database + db_session.add_all([gsp_yield_1_sql, gsp_sql_1]) + + app.dependency_overrides[get_session] = lambda: db_session + + # create temp file + with open("test.txt", "w") as f: + f.write("test") + fs = fsspec.open("test.txt").fs + modified_date = fs.modified("test.txt") + + response = client.get("/v0/solar/GB/update_last_data?component=nwp&file=test.txt") + assert response.status_code == 200 + + data = db_session.query(InputDataLastUpdatedSQL).all() + assert len(data) == 1 + assert data[0].nwp.isoformat() == modified_date.isoformat() From bcc20e19f9babc4a999ee6d771030f04b737fda8 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Wed, 29 May 2024 18:13:21 +0100 Subject: [PATCH 2/7] isort --- src/tests/test_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_status.py b/src/tests/test_status.py index c6e94f2..390c944 100644 --- a/src/tests/test_status.py +++ b/src/tests/test_status.py @@ -9,9 +9,9 @@ APIRequestSQL, ForecastSQL, GSPYield, + InputDataLastUpdatedSQL, Location, LocationSQL, - InputDataLastUpdatedSQL, Status, UserSQL, ) From d05ccc41f838ff6c717de4a80811dc5c71bd3e3c Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Wed, 29 May 2024 20:38:20 +0100 Subject: [PATCH 3/7] only update if date is greater than current time --- src/status.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/status.py b/src/status.py index 0a8c53e..5d9b409 100644 --- a/src/status.py +++ b/src/status.py @@ -7,7 +7,10 @@ import structlog from fastapi import APIRouter, Depends, HTTPException, Request from nowcasting_datamodel.models import ForecastSQL, GSPYieldSQL, Status -from nowcasting_datamodel.read.read import update_latest_input_data_last_updated +from nowcasting_datamodel.read.read import ( + get_latest_input_data_last_updated, + update_latest_input_data_last_updated, +) from sqlalchemy.exc import NoResultFound from sqlalchemy.orm.session import Session @@ -100,9 +103,14 @@ def update_last_data( fs = fsspec.open(file).fs modified_date = fs.modified(file) - # update the database - update_latest_input_data_last_updated( - session=session, component=component, update_datetime=modified_date - ) + # get last value + latest_input_data = get_latest_input_data_last_updated(session=session) + current_date = getattr(latest_input_data, component) + + if current_date < modified_date: + # update the database + update_latest_input_data_last_updated( + session=session, component=component, update_datetime=modified_date + ) return modified_date From 79588476f6bff279fb0391990d8a63dd2bf525fe Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Wed, 29 May 2024 20:53:54 +0100 Subject: [PATCH 4/7] udpate --- src/status.py | 10 ++++++++-- src/tests/test_status.py | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/status.py b/src/status.py index 5d9b409..c78eb23 100644 --- a/src/status.py +++ b/src/status.py @@ -105,9 +105,15 @@ def update_last_data( # get last value latest_input_data = get_latest_input_data_last_updated(session=session) - current_date = getattr(latest_input_data, component) - if current_date < modified_date: + update = True + if latest_input_data is not None: + if hasattr(latest_input_data, component): + current_date = getattr(latest_input_data, component) + if current_date >= modified_date: + update = False + + if update: # update the database update_latest_input_data_last_updated( session=session, component=component, update_datetime=modified_date diff --git a/src/tests/test_status.py b/src/tests/test_status.py index 390c944..7b0bfaf 100644 --- a/src/tests/test_status.py +++ b/src/tests/test_status.py @@ -99,6 +99,12 @@ def test_check_last_forecast_gsp(db_session): assert len(data) == 1 assert data[0].gsp.isoformat() == datetime(2023, 1, 3, tzinfo=timezone.utc).isoformat() + # check no updates is made, as file modified datetime is the same + response = client.get("/v0/solar/GB/update_last_data?component=gsp") + assert response.status_code == 200, response.text + data = db_session.query(InputDataLastUpdatedSQL).all() + assert len(data) == 1 + def test_check_last_forecast_file(db_session): """Check check_last_forecast_run works fine""" From bc99ef36b519324239de7ad2b59bcc54ca7a4258 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 30 May 2024 10:32:20 +0100 Subject: [PATCH 5/7] use temp file --- src/tests/test_status.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/tests/test_status.py b/src/tests/test_status.py index 7b0bfaf..fd26e35 100644 --- a/src/tests/test_status.py +++ b/src/tests/test_status.py @@ -1,5 +1,7 @@ """ Test for main app """ +import os +import tempfile from datetime import datetime, timedelta, timezone import fsspec @@ -123,14 +125,16 @@ def test_check_last_forecast_file(db_session): app.dependency_overrides[get_session] = lambda: db_session # create temp file - with open("test.txt", "w") as f: - f.write("test") - fs = fsspec.open("test.txt").fs - modified_date = fs.modified("test.txt") - - response = client.get("/v0/solar/GB/update_last_data?component=nwp&file=test.txt") - assert response.status_code == 200 - - data = db_session.query(InputDataLastUpdatedSQL).all() - assert len(data) == 1 - assert data[0].nwp.isoformat() == modified_date.isoformat() + with tempfile.TemporaryDirectory() as tmp: + filename = os.path.join(tmp, "text.txt") + with open(filename, "w") as f: + f.write("test") + fs = fsspec.open(filename).fs + modified_date = fs.modified(filename) + + response = client.get(f"/v0/solar/GB/update_last_data?component=nwp&file={filename}") + assert response.status_code == 200 + + data = db_session.query(InputDataLastUpdatedSQL).all() + assert len(data) == 1 + assert data[0].nwp.isoformat() == modified_date.isoformat() From c36b8fe2132004dfc6105dbd3fbd3115119ec689 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Thu, 30 May 2024 12:13:04 +0100 Subject: [PATCH 6/7] add s3fs to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 3f9eb73..43dbe64 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ sentry-sdk slowapi pathy==0.10.3 fsspec +s3fs From bee48e06fd35c7ef553c80d9193abbab96827d98 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 11:14:05 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f6409a8..43dbe64 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,4 +19,3 @@ slowapi pathy==0.10.3 fsspec s3fs -