From 02f153cb3bd63f4741c25e2270d4f60fdec49e46 Mon Sep 17 00:00:00 2001 From: Jamie Kennea Date: Wed, 21 Feb 2024 15:36:57 -0500 Subject: [PATCH] Add BurstCube TOO API endpoints Add Validation to AstropyTime Add FOV check to BurstCubeTOO and other fixes Change healpix_order to healpix_scheme Compare ephem times in unix time as comparing freeform astropy time object can cause this to fail due to minute differences that create an out of bounds issue with the assertion in get_slice Fixes for modified_on/by Remove history table stuff for now Fix security for burstcube TOO Updates for mypy and cleanups Update import sorts Import sort Remove debug print statement Simplify database write/update Simplify database write/update Fix check for existing TOO Change BurstCubeTOO PUT to have the same arguments as POST, so we can use it to upload updated HEALPix Remove excess docstring from schema Reconfigure BurstCubeTOO to redo constraints check if PUT updates coordinates Change how uploads are done in the API Update TOO PUT schema Trap uploads of bad HEALPix files Fix bug Make schema and api return astropy native values Simplify requests.py Move to astropy native in schema. Clean up docstrings Clean up code reuse in BurstCubeTOO Json email selections (#1977) Bump usehooks-ts from 2.15.0 to 2.15.1 Bumps [usehooks-ts](https://github.com/juliencrn/usehooks-ts) from 2.15.0 to 2.15.1. - [Release notes](https://github.com/juliencrn/usehooks-ts/releases) - [Commits](https://github.com/juliencrn/usehooks-ts/compare/usehooks-ts@2.15.0...usehooks-ts@2.15.1) --- updated-dependencies: - dependency-name: usehooks-ts dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Bump tiny-invariant from 1.3.1 to 1.3.3 Bumps [tiny-invariant](https://github.com/alexreardon/tiny-invariant) from 1.3.1 to 1.3.3. - [Release notes](https://github.com/alexreardon/tiny-invariant/releases) - [Commits](https://github.com/alexreardon/tiny-invariant/compare/v1.3.1...v1.3.3) --- updated-dependencies: - dependency-name: tiny-invariant dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Support for reading gzipped fits files. Refactor reused code. Remove superfluous entries in app.arc Make BurstCubeTOO a dataclass to remove the init stuff Differentiate username and created by Back to username Remove a space in a comment. Compare jd instead of unix as it should always be consistent Put table into a property that can be set for pytest Don't hardcode TOO table name Clean up docstring Move table write into BurstCubeTOO Move BurstCubeTOOModel into schema.py Fix imports Remove types in docstrings Adds a confirmation alert (#1957) * Adds a confirmation alert * Return intent and rename newItem to newCircular Better sentence text, adds heading Bump es5-ext from 0.10.61 to 0.10.63 Bumps [es5-ext](https://github.com/medikoo/es5-ext) from 0.10.61 to 0.10.63. - [Release notes](https://github.com/medikoo/es5-ext/releases) - [Changelog](https://github.com/medikoo/es5-ext/blob/main/CHANGELOG.md) - [Commits](https://github.com/medikoo/es5-ext/compare/v0.10.61...v0.10.63) --- updated-dependencies: - dependency-name: es5-ext dependency-type: indirect ... Signed-off-by: dependabot[bot] Announcement template (#1958) Code of Conduct (#1635) Docstring cleanups Fix use of old style Config in Pydantic model Add pytest to check CRUD operations on BurstCubeTOO Add check for SPACETRACK credentials Fix reject_reason Add old TOO rejection pytest Add test for double submission Add test for double submission Mock BurstCubeTLE so that we don't have to spam celestak or spacetrack Add pytest-mock requirement for testing Revert tle.py Mypy fixes Docstring fix Fix test time to a value that matches the test TLE Remove epoch for TLE Revert to Time.now() otherwise toos will get rejected a few days from now --- app.arc | 3 + python/across_api/base/api.py | 172 ++++++++- python/across_api/base/constraints.py | 2 +- python/across_api/base/database.py | 14 + python/across_api/base/schema.py | 156 +++++++- python/across_api/burstcube/api.py | 202 +++++++++++ python/across_api/burstcube/requests.py | 115 ++++++ python/across_api/burstcube/schema.py | 219 ++++++++++++ python/across_api/burstcube/toorequest.py | 338 ++++++++++++++++++ python/requirements.in | 1 + python/requirements.txt | 2 + python/tests/burstcubetoo/conftest.py | 118 ++++++ .../tests/burstcubetoo/test_burstcube_too.py | 70 ++++ requirements.txt | 2 + 14 files changed, 1406 insertions(+), 8 deletions(-) create mode 100644 python/across_api/base/database.py create mode 100644 python/across_api/burstcube/requests.py create mode 100644 python/across_api/burstcube/schema.py create mode 100644 python/across_api/burstcube/toorequest.py create mode 100644 python/tests/burstcubetoo/conftest.py create mode 100644 python/tests/burstcubetoo/test_burstcube_too.py diff --git a/app.arc b/app.arc index 2b700dd3ed..b21ce79beb 100644 --- a/app.arc +++ b/app.arc @@ -49,6 +49,9 @@ acrossapi_tle tle1 String tle2 String +burstcube_too + id *String + sessions _idx *String _ttl TTL diff --git a/python/across_api/base/api.py b/python/across_api/base/api.py index ece647dcb0..e81fbf96f4 100644 --- a/python/across_api/base/api.py +++ b/python/across_api/base/api.py @@ -4,10 +4,14 @@ """ Base API definitions for ACROSS API. This module is imported by all other API -modules. Contains the FastAPI app definition. +modules. Contains the FastAPI app definition and global depends definition. """ +from astropy.time import Time # type: ignore[import] +from datetime import datetime +from typing import Annotated, Optional +from fastapi import Depends, FastAPI, Path, Query +import astropy.units as u # type: ignore[import] -from fastapi import FastAPI # FastAPI app definition app = FastAPI( @@ -19,3 +23,167 @@ }, root_path="/labs/api/v1", ) + +# FastAPI Depends definitions + + +# Depends functions for FastAPI calls. +async def optional_daterange( + begin: Annotated[ + Optional[datetime], + Query( + description="Start time of period to be calculated.", + title="Begin", + ), + ] = None, + end: Annotated[ + Optional[datetime], + Query( + description="Start time of period to be calculated.", + title="End", + ), + ] = None, +) -> dict: + """ + Helper function to convert begin and end to datetime objects. + """ + if begin is None or end is None: + return {"begin": None, "end": None} + return {"begin": Time(begin), "end": Time(end)} + + +OptionalDateRangeDep = Annotated[dict, Depends(optional_daterange)] + + +# Depends functions for FastAPI calls. +async def optional_length( + length: Annotated[ + Optional[float], + Query( + description="Length of time (days).", + title="Length", + ), + ] = None, +) -> Optional[u.Quantity]: + """ + Helper function to convert begin and end to datetime objects. + """ + if length is None: + return None + return length * u.day + + +OptionalLengthDep = Annotated[dict, Depends(optional_length)] + + +async def optional_limit( + limit: Annotated[ + Optional[int], + Query( + ge=0, + title="Limit", + description="Maximum number of results to return.", + ), + ] = None, +) -> Optional[int]: + return limit + + +LimitDep = Annotated[Optional[int], Depends(optional_limit)] + + +async def error_radius( + error_radius: Annotated[ + Optional[float], + Query( + ge=0, + title="Error Radius", + description="Error radius in degrees.", + ), + ] = None, +) -> Optional[float]: + if error_radius is None: + return None + return error_radius * u.deg + + +ErrorRadiusDep = Annotated[float, Depends(error_radius)] + + +async def exposure( + exposure: Annotated[ + float, + Query( + ge=0, + title="Exposure", + description="Exposure time in seconds.", + ), + ] = 200, +) -> u.Quantity: + return exposure * u.s + + +ExposureDep = Annotated[float, Depends(exposure)] + + +async def offset( + offset: Annotated[ + float, + Query( + ge=-200, + le=200, + title="Offset", + description="Offset start of dump window from T0 by this amount (seconds).", + ), + ] = -50, +) -> u.Quantity: + return offset * u.s + + +OffsetDep = Annotated[float, Depends(offset)] + + +IdDep = Annotated[str, Path(description="TOO ID string")] + + +async def trigger_time( + trigger_time: Annotated[ + datetime, + Query( + title="Trigger Time", + description="Time of trigger in UTC or ISO format.", + ), + ], +) -> Optional[datetime]: + return Time(trigger_time) + + +TriggerTimeDep = Annotated[datetime, Depends(trigger_time)] + + +async def optional_ra_dec( + ra: Annotated[ + Optional[float], + Query( + ge=0, + lt=360, + title="RA (J2000)", + description="Right Ascenscion in J2000 coordinates and units of decimal degrees.", + ), + ] = None, + dec: Annotated[ + Optional[float], + Query( + ge=-90, + le=90, + title="Dec (J2000)", + description="Declination in J2000 coordinates in units of decimal degrees.", + ), + ] = None, +) -> Optional[dict]: + if ra is None or dec is None: + return {"ra": None, "dec": None} + return {"ra": ra * u.deg, "dec": dec * u.deg} + + +OptionalRaDecDep = Annotated[dict, Depends(optional_ra_dec)] diff --git a/python/across_api/base/constraints.py b/python/across_api/base/constraints.py index fcfed27a65..ccc4107864 100644 --- a/python/across_api/base/constraints.py +++ b/python/across_api/base/constraints.py @@ -41,7 +41,7 @@ def get_slice(time: Time, ephem: EphemBase) -> slice: else: # Check that the time range is within the ephemeris range, as above. assert ( - time[0] >= ephem.begin and time[-1] <= ephem.end + time[0].jd >= ephem.begin.jd and time[-1].jd <= ephem.end.jd ), "Time outside of ephemeris of range" # Find the indices for the start and end of the time range and return a diff --git a/python/across_api/base/database.py b/python/across_api/base/database.py new file mode 100644 index 0000000000..ad4a74afca --- /dev/null +++ b/python/across_api/base/database.py @@ -0,0 +1,14 @@ +from typing import Any +import arc # type: ignore[import] +import boto3 # type: ignore[import] +import os + + +def dynamodb_table(tablename) -> Any: + """If running in Architect, return tables.table, else return boto3 dynamodb + table. This enables the use of moto to mock the dynamodb table in tests.""" + if os.environ.get("ARC_ENV") is not None: + return arc.tables.table(tablename) + else: + session = boto3.Session() + return session.resource("dynamodb", region_name="us-east-1").Table(tablename) diff --git a/python/across_api/base/schema.py b/python/across_api/base/schema.py index a20ffd18b7..a19a6b3683 100644 --- a/python/across_api/base/schema.py +++ b/python/across_api/base/schema.py @@ -4,30 +4,84 @@ from datetime import datetime -from typing import Annotated, Any, List, Optional - +import json +from typing import Annotated, Any, Dict, List, Optional, Union +from astropy.coordinates import Latitude, Longitude # type: ignore[import] import astropy.units as u # type: ignore -from arc import tables # type: ignore from astropy.time import Time # type: ignore from pydantic import ( BaseModel, + BeforeValidator, ConfigDict, Field, PlainSerializer, + WithJsonSchema, computed_field, model_validator, ) +from .database import dynamodb_table + # Define a Pydantic type for astropy Time objects, which will be serialized as # a naive UTC datetime object, or a string in ISO format for JSON. AstropyTime = Annotated[ Time, + BeforeValidator(lambda x: Time(x) if type(x) is not Time else x), PlainSerializer( lambda x: x.utc.datetime, return_type=datetime, ), + WithJsonSchema( + {"type": "string", "format": "date-time"}, + mode="serialization", + ), + WithJsonSchema( + {"type": "string", "format": "date-time"}, + mode="validation", + ), +] + + +# Pydantic type for a Astropy Time in seconds +AstropySeconds = Annotated[ + u.Quantity, + BeforeValidator(lambda x: x * u.s if type(x) is not u.Quantity else x.to(u.s)), + PlainSerializer( + lambda x: x.to(u.s).value, + return_type=float, + ), + WithJsonSchema( + {"type": "number"}, + mode="serialization", + ), + WithJsonSchema( + {"type": "number"}, + mode="validation", + ), +] + + +# Pydantic type for a scalar astropy Quantity/Latitude/Longitude in degrees +AstropyAngle = Annotated[ + Union[Latitude, Longitude, u.Quantity[u.deg]], + BeforeValidator(lambda x: x * u.deg if isinstance(x, (int, float)) else x), + PlainSerializer( + lambda x: x.to_value(u.deg), + return_type=float, + ), + WithJsonSchema( + {"type": "number"}, + mode="serialization", + ), + WithJsonSchema( + {"type": "number"}, + mode="validation", + ), ] +# Define a pydantic type for a dictionary that will be serialized as a JSON +JsonStr = Annotated[Dict, PlainSerializer(lambda x: json.dumps(x), return_type=str)] + class BaseSchema(BaseModel): """ @@ -75,6 +129,98 @@ def check_dates(cls, data: Any) -> Any: return data +class OptionalDateRangeSchema(BaseSchema): + """Schema that defines date range, which is optional + + Parameters + ---------- + begin + The beginning date of the range, by default None + end + The end date of the range, by default None + + Methods + ------- + check_dates(data: Any) -> Any + Validates the date range and ensures that the begin and end dates are set correctly. + + """ + + begin: Optional[AstropyTime] = None + end: Optional[AstropyTime] = None + + @model_validator(mode="after") + @classmethod + def check_dates(cls, data: Any) -> Any: + """Validates the date range and ensures that the begin and end dates are set correctly. + + Parameters + ---------- + data + The data to be validated. + + Returns + ------- + Any + The validated data. + + Raises + ------ + AssertionError + If the begin and end dates are not both set or both not set. + If the end date is before the begin date. + + """ + if data.begin is None or data.end is None: + assert ( + data.begin == data.end + ), "Begin/End should both be set, or both not set" + if data.begin != data.end: + assert data.begin <= data.end, "End date should not be before begin" + + return data + + +class OptionalPositionSchema(BaseSchema): + """ + Schema for representing position information with an error radius. + + Attributes + ---------- + error + The error associated with the position. Defaults to None. + """ + + ra: Optional[AstropyAngle] = Field(ge=0 * u.deg, lt=360 * u.deg, default=None) + dec: Optional[AstropyAngle] = Field(ge=-90 * u.deg, le=90 * u.deg, default=None) + error: Optional[AstropyAngle] = None + + @model_validator(mode="after") + @classmethod + def check_ra_dec(cls, data: Any) -> Any: + """Validates that RA and Dec are both set or both not set. + + Parameters + ---------- + data + The data to be validated. + + Returns + ------- + Any + The validated data. + + Raises + ------ + AssertionError + If RA and Dec are not both set or both not set. + + """ + if data.ra is None or data.dec is None: + assert data.ra == data.dec, "RA/Dec should both be set, or both not set" + return data + + class TLEGetSchema(BaseSchema): epoch: AstropyTime @@ -151,7 +297,7 @@ def find_tles_between_epochs( ------- A list of TLEEntry objects between the specified epochs. """ - table = tables.table(cls.__tablename__) + table = dynamodb_table(cls.__tablename__) # Query the table for TLEs between the two epochs response = table.query( @@ -168,7 +314,7 @@ def find_tles_between_epochs( def write(self) -> None: """Write the TLE entry to the database.""" - table = tables.table(self.__tablename__) + table = dynamodb_table(self.__tablename__) table.put_item(Item=self.model_dump(mode="json")) diff --git a/python/across_api/burstcube/api.py b/python/across_api/burstcube/api.py index ff1d227489..3542432dcb 100644 --- a/python/across_api/burstcube/api.py +++ b/python/across_api/burstcube/api.py @@ -1,3 +1,205 @@ # Copyright © 2023 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. + +import gzip +from datetime import datetime +from typing import Annotated, BinaryIO, Optional, Tuple, Union + +from astropy.io import fits # type: ignore +from astropy.time import Time # type: ignore +from fastapi import Depends, File, HTTPException, Query, Security, UploadFile, status + +from ..auth.api import claims, scope_authorize +from ..base.api import ( + ErrorRadiusDep, + ExposureDep, + IdDep, + LimitDep, + OffsetDep, + OptionalDateRangeDep, + OptionalLengthDep, + OptionalRaDecDep, + TriggerTimeDep, + app, +) +from .requests import BurstCubeTOORequests +from .schema import BurstCubeTOORequestsSchema, BurstCubeTOOSchema, BurstCubeTriggerInfo +from .toorequest import BurstCubeTOO + + +# BurstCube Deps +async def optional_trigger_time( + trigger_time: Annotated[ + Optional[datetime], + Query( + title="Trigger Time", + description="Time of trigger in UTC or ISO format.", + ), + ] = None, +) -> Optional[datetime]: + if trigger_time is None: + return None + return Time(trigger_time) + + +OptionalTriggerTimeDep = Annotated[datetime, Depends(optional_trigger_time)] + + +def read_healpix_file(healpix_file: UploadFile) -> Tuple[fits.FITS_rec, str]: + """Read in a HEALPix file in FITS format and return the HDUList. Supports + gzipped FITS files""" + + # Type hint for the file object + file: Union[gzip.GzipFile, BinaryIO] + + # If the file is a gzip file, open it with gzip.open, otherwise just + # pass along the filehandle + if healpix_file.content_type == "application/x-gzip": + file = gzip.open(healpix_file.file, "rb") + else: + file = healpix_file.file + + # Open HEALPix fits file and extract data and ordering scheme + try: + with fits.open(file) as hdu: + healpix_data = hdu[1].data + healpix_scheme = hdu[1].header["ORDERING"] + except (OSError, KeyError, IndexError): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid HEALPix file.", + ) + return healpix_data, healpix_scheme + + +@app.post( + "/burstcube/too", + status_code=status.HTTP_201_CREATED, + dependencies=[ + Security(scope_authorize, scopes=["gcn.nasa.gov/kafka-public-consumer"]) + ], +) +async def burstcube_too_submit( + credential: Annotated[dict, Depends(claims)], + ra_dec: OptionalRaDecDep, + error_radius: ErrorRadiusDep, + trigger_time: TriggerTimeDep, + trigger_info: BurstCubeTriggerInfo, + exposure: ExposureDep, + offset: OffsetDep, + healpix_file: UploadFile = File( + None, description="HEALPix file describing the localization." + ), +) -> BurstCubeTOOSchema: + """ + Resolve the name of an astronomical object to its coordinates. + """ + # Construct the TOO object. + too = BurstCubeTOO( + username=credential["sub"], + ra=ra_dec["ra"], + dec=ra_dec["dec"], + error_radius=error_radius, + trigger_time=trigger_time, + trigger_info=trigger_info, + exposure=exposure, + offset=offset, + ) + # If a HEALpix file was uploaded, open it and set the healpix_loc + # and healpix_scheme attributes. + if healpix_file is not None: + too.healpix_loc, too.healpix_scheme = read_healpix_file(healpix_file) + too.post() + return too.schema + + +@app.put( + "/burstcube/too/{id}", + status_code=status.HTTP_201_CREATED, + dependencies=[ + Security(scope_authorize, scopes=["gcn.nasa.gov/kafka-public-consumer"]) + ], +) +async def burstcube_too_update( + id: IdDep, + credential: Annotated[dict, Depends(claims)], + ra_dec: OptionalRaDecDep, + error_radius: ErrorRadiusDep, + trigger_time: TriggerTimeDep, + trigger_info: BurstCubeTriggerInfo, + exposure: ExposureDep, + offset: OffsetDep, + healpix_file: UploadFile = File( + None, description="HEALPix file describing the localization." + ), +) -> BurstCubeTOOSchema: + """ + Update a BurstCube TOO object with the given ID number. + """ + # Update the TOO object. + too = BurstCubeTOO( + id=id, + username=credential["sub"], + ra=ra_dec["ra"], + dec=ra_dec["dec"], + error_radius=error_radius, + trigger_time=trigger_time, + trigger_info=trigger_info, + exposure=exposure, + offset=offset, + ) + # If a HEALpix file was uploaded, open it and set the healpix_loc + # and healpix_scheme attributes. + if healpix_file is not None: + too.healpix_loc, too.healpix_scheme = read_healpix_file(healpix_file) + too.put() + return too.schema + + +@app.get("/burstcube/too/", status_code=status.HTTP_200_OK) +async def burstcube_too_requests( + daterange: OptionalDateRangeDep, + length: OptionalLengthDep, + limit: LimitDep, +) -> BurstCubeTOORequestsSchema: + """ + Endpoint to retrieve BurstCube multiple TOO requests. + """ + return BurstCubeTOORequests( + begin=daterange["begin"], + end=daterange["end"], + length=length, + limit=limit, + ).schema + + +@app.get("/burstcube/too/{id}", status_code=status.HTTP_200_OK) +async def burstcube_too( + id: IdDep, +) -> BurstCubeTOOSchema: + """ + Retrieve a BurstCube Target of Opportunity (TOO) by ID. + """ + too = BurstCubeTOO(id=id) + too.get() + return too.schema + + +@app.delete( + "/burstcube/too/{id}", + status_code=status.HTTP_200_OK, + dependencies=[ + Security(scope_authorize, scopes=["gcn.nasa.gov/kafka-public-consumer"]) + ], +) +async def burstcube_delete_too( + credential: Annotated[dict, Depends(claims)], + id: IdDep, +) -> BurstCubeTOOSchema: + """ + Delete a BurstCube Target of Opportunity (TOO) with the given ID. + """ + too = BurstCubeTOO(username=credential["sub"], id=id) + too.delete() + return too.schema diff --git a/python/across_api/burstcube/requests.py b/python/across_api/burstcube/requests.py new file mode 100644 index 0000000000..cfa9ffcd7e --- /dev/null +++ b/python/across_api/burstcube/requests.py @@ -0,0 +1,115 @@ +# Copyright © 2023 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. + +from typing import Optional + +import astropy.units as u # type: ignore +from arc import tables # type: ignore +from astropy.time import Time # type: ignore +from boto3.dynamodb.conditions import Key # type: ignore + +from ..base.common import ACROSSAPIBase +from .schema import ( + BurstCubeTOOModel, + BurstCubeTOORequestsGetSchema, + BurstCubeTOORequestsSchema, + BurstCubeTOOSchema, +) + + +class BurstCubeTOORequests(ACROSSAPIBase): + """ + Class to fetch multiple BurstCubeTOO requests, based on various filters. + + Note that the filtering right now is based on DynamoDB scan, which is not + very efficient. This should be replaced with a query at some point. + + Parameters + ---------- + begin + Start time of plan search + end + End time of plan search + limit + Limit number of searches + length + Length of time to search from now + + Attributes + ---------- + entries + List of BurstCubeTOO requests + status + Status of BurstCubeTOO query + """ + + _schema = BurstCubeTOORequestsSchema + _get_schema = BurstCubeTOORequestsGetSchema + mission = "ACROSS" + + def __getitem__(self, i): + return self.entries[i] + + def __len__(self): + return len(self.entries) + + def __init__( + self, + begin: Optional[Time] = None, + end: Optional[Time] = None, + length: Optional[u.Quantity] = None, + limit: Optional[int] = None, + ): + # Default parameters + self.limit = limit + self.begin = begin + self.end = end + self.length = length + # Attributes + self.entries: list = [] + + # Parse Arguments + if self.validate_get(): + self.get() + + def get(self) -> bool: + """ + Get a list of BurstCubeTOO requests + + Returns + ------- + bool + Did this work? True | False + """ + # Validate query + if not self.validate_get(): + return False + table = tables.table(BurstCubeTOOModel.__tablename__) + + if self.length is not None: + self.begin = Time.now() + self.end = self.begin - self.length + + # Search for events that overlap a given date range + if self.begin is not None and self.end is not None: + toos = table.scan( + FilterExpression=Key("created_on").between( + str(self.begin), str(self.end) + ) + ) + else: + toos = table.scan() + + # Convert entries for return + self.entries = [BurstCubeTOOSchema.model_validate(too) for too in toos["Items"]] + + # Sort and limit the results + self.entries.sort(key=lambda x: x.trigger_time, reverse=True) + self.entries = self.entries[: self.limit] + + return True + + +# Short aliases for classes +TOORequests = BurstCubeTOORequests diff --git a/python/across_api/burstcube/schema.py b/python/across_api/burstcube/schema.py new file mode 100644 index 0000000000..707320f627 --- /dev/null +++ b/python/across_api/burstcube/schema.py @@ -0,0 +1,219 @@ +# Copyright © 2023 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. + + +import hashlib +import json +from decimal import Decimal +from enum import Enum +from typing import List, Optional + +import astropy.units as u # type: ignore +from astropy.time import Time # type: ignore +from pydantic import ConfigDict, computed_field, model_validator # type: ignore + +from ..base.schema import ( + AstropySeconds, + AstropyTime, + BaseSchema, + JsonStr, + OptionalDateRangeSchema, + OptionalPositionSchema, +) + + +class TOOReason(str, Enum): + """ + Reasons for rejecting TOO observations + + Attributes + ---------- + saa + In SAA + earth_occult + Earth occulted + moon_occult + Moon occulted + sun_occult + Sun occulted + too_old + Too old + other + Other + none + None + """ + + saa = "In SAA" + earth_occult = "Earth occulted" + moon_occult = "Moon occulted" + sun_occult = "Sun occulted" + too_old = "Too old" + other = "Other" + none = "None" + + +class TOOStatus(str, Enum): + """ + Enumeration class representing the status of a Target of Opportunity (TOO) request. + + Attributes: + requested + The TOO request has been submitted. + rejected + The TOO request has been rejected. + declined + The TOO request has been declined. + approved + The TOO request has been approved. + executed + The TOO request has been executed. + other + The TOO request has a status other than the predefined ones. + """ + + requested = "Requested" + rejected = "Rejected" + declined = "Declined" + approved = "Approved" + executed = "Executed" + deleted = "Deleted" + other = "Other" + + +class BurstCubeTriggerInfo(BaseSchema): + """ + Metadata schema for the BurstCube Target of Opportunity (TOO) request. Note + that this schema is not strictly defined, keys are only suggested, and + additional keys can be added as needed. + """ + + trigger_name: Optional[str] = None + trigger_mission: Optional[str] = None + trigger_instrument: Optional[str] = None + trigger_id: Optional[str] = None + trigger_duration: Optional[AstropySeconds] = None + classification: Optional[str] = None + justification: Optional[str] = None + + model_config = ConfigDict(extra="allow") + + @model_validator(mode="before") + def convert_json_string_to_dict(cls, data): + if isinstance(data, str): + return json.loads(data) + return data + + +class BurstCubeTOOSchema(OptionalPositionSchema): + """ + Schema describing a BurstCube TOO Request. + """ + + id: Optional[str] = None + created_by: str + created_on: AstropyTime + modified_by: Optional[str] = None + modified_on: Optional[AstropyTime] = None + trigger_time: AstropyTime + trigger_info: BurstCubeTriggerInfo + exposure: AstropySeconds + offset: AstropySeconds + reject_reason: TOOReason = TOOReason.none + status: TOOStatus = TOOStatus.requested + too_info: str = "" + + +class BurstCubeTOODelSchema(BaseSchema): + """ + Schema for BurstCubeTOO DELETE API call. + + Attributes + ---------- + id + The ID of the BurstCubeTOODel object. + """ + + id: str + + +class BurstCubeTOOPostSchema(OptionalPositionSchema): + """ + Schema to submit a TOO request for BurstCube. + """ + + trigger_time: AstropyTime + trigger_info: BurstCubeTriggerInfo + exposure: AstropySeconds = 200 * u.s + offset: AstropySeconds = -50 * u.s + + +class BurstCubeTOOGetSchema(BaseSchema): + """ + Schema for BurstCubeTOO GET request. + """ + + id: str + + +class BurstCubeTOOPutSchema(BurstCubeTOOPostSchema): + """ + Schema for BurstCubeTOO PUT request. + """ + + id: str + + +class BurstCubeTOORequestsGetSchema(OptionalDateRangeSchema): + """ + Schema for GET requests to retrieve BurstCube Target of Opportunity (TOO) requests. + """ + + length: Optional[u.Quantity] = None + limit: Optional[int] = None + + @model_validator(mode="after") + def check_begin_and_end_or_length_set(self): + if self.begin is not None and self.end is not None and self.length is not None: + raise ValueError("Cannot set both begin and end and length.") + elif self.begin is not None and self.length is not None: + self.end = self.begin + self.length + elif self.begin is None and self.end is None and self.length is not None: + self.end = Time.now() + self.begin = self.end - self.length + + +class BurstCubeTOORequestsSchema(BaseSchema): + """ + Schema for BurstCube TOO requests. + """ + + entries: List[BurstCubeTOOSchema] + + +class BurstCubeTOOModel(BaseSchema): + """Database Model Schema for BurstCube TOO requests.""" + + __tablename__ = "burstcube_too" + ra: Optional[Decimal] = None + dec: Optional[Decimal] = None + error_radius: Optional[Decimal] = None + created_by: str + created_on: str + modified_by: Optional[str] = None + modified_on: Optional[str] = None + trigger_time: str + trigger_info: JsonStr + exposure: Decimal = Decimal(200) + offset: Decimal = Decimal(-50) + reject_reason: str = "None" + status: str = "Requested" + too_info: str = "" + + @computed_field # type: ignore + @property + def id(self) -> str: + return hashlib.md5( + f"{self.trigger_time}{self.exposure}{self.offset}".encode() + ).hexdigest() diff --git a/python/across_api/burstcube/toorequest.py b/python/across_api/burstcube/toorequest.py new file mode 100644 index 0000000000..5fc3b3f1f6 --- /dev/null +++ b/python/across_api/burstcube/toorequest.py @@ -0,0 +1,338 @@ +# Copyright © 2023 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. + +from dataclasses import dataclass # type: ignore[import] +from functools import cached_property +from typing import Optional, Union + +import astropy.units as u # type: ignore[import] +import botocore # type: ignore[import] +import numpy as np +from astropy.coordinates import ( # type: ignore[import] + Latitude, + Longitude, + SkyCoord, +) +from astropy.time import Time # type: ignore[import] +from fastapi import HTTPException + +from ..base.database import dynamodb_table + +from ..base.common import ACROSSAPIBase, round_time +from .constraints import burstcube_saa_constraint +from .ephem import BurstCubeEphem +from .fov import BurstCubeFOV +from .schema import ( + BurstCubeTOODelSchema, + BurstCubeTOOGetSchema, + BurstCubeTOOModel, + BurstCubeTOOPostSchema, + BurstCubeTOOPutSchema, + BurstCubeTOOSchema, + BurstCubeTriggerInfo, + TOOReason, + TOOStatus, +) + + +@dataclass +class BurstCubeTOO(ACROSSAPIBase): + """ + Class to handle BurstCube Target of Opportunity Requests + + Parameters + ---------- + username + Username of user making request + id + ID of BurstCubeTOO to fetch, by default None + created_by + Username of user who created the BurstCubeTOO, by default None + created_on + Time BurstCubeTOO was created, by default None + modified_by + Username of user who last modified the BurstCubeTOO, by default None + modified_on + Time BurstCubeTOO was last modified, by default None + trigger_time + Time of trigger, by default None + trigger_info + Information about the trigger, by default None + exposure + Exposure time, by default 200 * u.s + offset + Offset time, by default -50 * u.s + ra + Right Ascension, by default None + dec + Declination, by default None + error_radius + Error radius, by default None + healpix_loc + HEALPix location, by default None + healpix_scheme + HEALPix scheme, by default "nested" + reject_reason + Reason for rejection, by default `TOOReason.none` + status + Status of request, by default "Requested" + too_info + Information about the TOO, by default "" + min_prob + Minimum probability in FOV to accept a TOO request, by default 0.10 + """ + + _schema = BurstCubeTOOSchema + _get_schema = BurstCubeTOOGetSchema + _put_schema = BurstCubeTOOPutSchema + _del_schema = BurstCubeTOODelSchema + _post_schema = BurstCubeTOOPostSchema + + id: Optional[str] = None + username: Optional[str] = None + created_by: Optional[str] = None + created_on: Optional[Time] = None + modified_by: Optional[str] = None + modified_on: Optional[Time] = None + trigger_time: Optional[Time] = None + trigger_info: Optional[BurstCubeTriggerInfo] = None + exposure: u.Quantity[u.s] = 200 * u.s + offset: u.Quantity[u.s] = -50 * u.s + ra: Union[u.Quantity[u.deg], Longitude, None] = None + dec: Union[u.Quantity[u.deg], Latitude, None] = None + error_radius: Optional[u.Quantity[u.deg]] = None + healpix_loc: Optional[np.ndarray] = None + healpix_scheme: str = "nested" + reject_reason: TOOReason = TOOReason.none + status: TOOStatus = TOOStatus.requested + too_info: str = "" + min_prob: float = 0.10 # 10% of probability in FOV + + # @property + # def table(self): + # """Return the table for the BurstCubeTOO.""" + # if not hasattr(self, "_table"): + # self._table = tables.table(BurstCubeTOOModel.__tablename__) + # return self._table + + @property + def table(self): + """Return the table for the BurstCubeTOO.""" + if not hasattr(self, "_table"): + self._table = dynamodb_table(BurstCubeTOOModel.__tablename__) + return self._table + + @table.setter + def table(self, value): + self._table = value + + @cached_property + def skycoord(self) -> Optional[SkyCoord]: + """Return the skycoord of the BurstCubeTOO.""" + if self.ra is not None and self.dec is not None: + return SkyCoord(self.ra, self.dec) + return None + + def get(self) -> bool: + """ + Fetch a BurstCubeTOO for a given id. + + Returns + ------- + Did this work? True | False + """ + + # Fetch BurstCubeTOO from database + try: + response = self.table.get_item(Key={"id": self.id}) + except botocore.exceptions.ClientError as e: + raise HTTPException(500, f"Error fetching BurstCubeTOO: {e}") + if "Item" not in response: + raise HTTPException(404, "BurstCubeTOO not found.") + + # Validate the response + too = BurstCubeTOOSchema.model_validate(response["Item"]) + + # Set the attributes of the BurstCubeTOO to the values from the database + for k, v in too: + setattr(self, k, v) + return True + + def delete(self) -> bool: + """ + Delete a given too, specified by id. created_by of BurstCubeTOO has to match yours. + + Returns + ------- + Did this work? True | False + """ + if self.validate_del(): + if self.get(): + # FIXME: Need proper authentication here + if self.created_by != self.username: + raise HTTPException(401, "BurstCubeTOO not owned by user.") + + self.status = TOOStatus.deleted + self.modified_by = self.username + self.modified_on = Time.now().datetime.isoformat() + # Write updated BurstCubeTOO to the database + too = BurstCubeTOOModel(**self.schema.model_dump(mode="json")) + self.table.put_item(Item=too.model_dump()) + return True + return False + + def put(self) -> bool: + """ + Alter existing BurstCube BurstCubeTOO using ACROSS API using POST + + Returns + ------- + Did this work? True | False + """ + # Make sure the PUT request validates + if not self.validate_put(): + return False + + # Check if this BurstCubeTOO exists + response = self.table.get_item(Key={"id": self.id}) + + # Check if the TOO exists + if "Item" not in response: + raise HTTPException(404, "BurstCubeTOO not found.") + + # Reconstruct the TOO as it exists in the database + old_too = BurstCubeTOO(**response["Item"]) + + # FIXME: Some validation as to whether the user is allowed to update + # this entry + + # Check if the coordinates are being changed, if yes, run the + # check_constraints again. If a healpix file is being uploaded, run + # the check_constraints again, as we don't record the healpix_loc for comparison. + if ( + self.ra != old_too.ra + or self.dec != old_too.dec + or self.error_radius != old_too.error_radius + or self.healpix_loc is not None + ): + # Check for various TOO constraints + if not self.check_constraints(): + self.status = TOOStatus.rejected + + # Write BurstCubeTOO to the database + self.modified_by = self.username + self.modified_on = Time.now().datetime.isoformat() + too = BurstCubeTOOModel(**self.schema.model_dump(mode="json")) + self.table.put_item(Item=too.model_dump()) + + return True + + def check_constraints(self): + """ + Check if BurstCubeTOO parameters are valid. + + Returns + ------- + Are BurstCubeTOO parameters valid? True | False + """ + # Reset too_info field + self.too_info = "" + # Check if the trigger time is in the future + # This is really just a sanity check, as the trigger time should be in the past + if self.trigger_time > Time.now(): + self.too_info += "Trigger time is in the future. " + self.reject_reason = TOOReason.other + return False + + # Reject if trigger is > 48 hours old + if self.trigger_time < Time.now() - 48 * u.hour: + self.reject_reason = TOOReason.too_old + self.too_info += "Trigger is too old. " + return False + + # Calculate Ephemeris for the requested dump time at one second time + # resolution + ephem = BurstCubeEphem( + begin=round_time(self.trigger_time + self.offset, 1 * u.s), + end=round_time(self.trigger_time + self.offset + self.exposure, 1 * u.s), + stepsize=1 * u.s, + ) + + # Check if the trigger time is in the SAA + saa = burstcube_saa_constraint( + time=round_time(self.trigger_time, 1 * u.s), ephem=ephem + ) + if saa is True: + self.too_info += "Trigger time inside SAA. " + self.reject_reason = TOOReason.saa + return False + + # Check if the trigger is inside FOV at T0 + if self.skycoord is not None or self.healpix_loc is not None: + fov = BurstCubeFOV(ephem=ephem, time=self.trigger_time) + infov = fov.probability_in_fov( + skycoord=self.skycoord, + error_radius=self.error_radius, + healpix_loc=self.healpix_loc, + healpix_scheme=self.healpix_scheme, + ) + + # Flag up if the required probability is not met + if infov < self.min_prob: + self.too_info += f"Probability inside FOV: {100*infov:.2f}%. Trigger was occulted at T0. " + self.reject_reason = TOOReason.earth_occult + return False + else: + self.too_info += f"Probability inside FOV: {100*infov:.2f}%. " + + # Check if any part of the dump time is inside the SAA, warn if so + if True in burstcube_saa_constraint(time=ephem.timestamp, ephem=ephem): + self.too_info += "Dump time partially inside SAA." + + # Strip excess whitespace + self.too_info = self.too_info.strip() + return True + + def post(self) -> bool: + """ + Upload BurstCubeTOO to ACROSS API using POST + + Returns + ------- + Did this work? True | False + """ + # Shouldn't run this unless a created_by is set + if self.username is None: + raise HTTPException(401, "Username not set.") + + # Validate supplied BurstCubeTOO values against the Schema + if not self.validate_post(): + return False + + # Set the created_on and created_by fields + self.created_on = Time.now() + self.created_by = self.username + + # Create a BurstCubeTOOModel from the BurstCubeTOO + too = BurstCubeTOOModel(**self.schema.model_dump(mode="json")) + + # Check if this BurstCubeTOO exists. As the id is just a hash of the + # trigger_time, exposure and offset, then repeated requests for values + # that match this will be caught. + try: + response = self.table.get_item(Key={"id": too.id}) + if response.get("Item"): + raise HTTPException(409, "BurstCubeTOO already exists.") + except botocore.exceptions.ClientError: + pass + + # Check for various TOO constraints + if not self.check_constraints(): + self.status = TOOStatus.rejected + + # Write BurstCubeTOO to the database + self.table.put_item(Item=too.model_dump()) + self.id = too.id + + return True diff --git a/python/requirements.in b/python/requirements.in index b36ffe91d7..1a9de6a92f 100644 --- a/python/requirements.in +++ b/python/requirements.in @@ -6,6 +6,7 @@ email-validator fastapi healpy mangum +python-multipart requests shapely sgp4 diff --git a/python/requirements.txt b/python/requirements.txt index 0fa5cd2909..e91e9d0d16 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -106,6 +106,8 @@ python-dateutil==2.8.2 # via matplotlib python-jose==3.3.0 # via architect-functions +python-multipart==0.0.9 + # via -r requirements.in pyyaml==6.0.1 # via astropy represent==2.0 diff --git a/python/tests/burstcubetoo/conftest.py b/python/tests/burstcubetoo/conftest.py new file mode 100644 index 0000000000..ab7e63d20c --- /dev/null +++ b/python/tests/burstcubetoo/conftest.py @@ -0,0 +1,118 @@ +import os +from unittest.mock import Mock + +import astropy.units as u # type: ignore[import] +import boto3 # type: ignore +import pytest +from across_api.base.schema import TLEEntry # type: ignore[import] +from across_api.burstcube.schema import BurstCubeTOOModel # type: ignore[import] +from across_api.burstcube.toorequest import BurstCubeTOO, BurstCubeTriggerInfo # type: ignore[import] +from astropy.time import Time # type: ignore +from astropy.time.core import TimeDelta # type: ignore[import] +from moto import mock_aws + + +@pytest.fixture +def burstcube_tle(): + return TLEEntry( + tle1="1 25544U 98067A 24059.70586912 .00019555 00000-0 35623-3 0 9995", + tle2="2 25544 51.6410 137.9505 0005676 302.8794 193.7648 15.49465684441577", + satname="ISS (ZARYA)", + ) + + +@pytest.fixture +def mock_read_tle_db(mocker, burstcube_tle): + """Mock TLEEntry find_tles_between_epochs to return burstcube_tle fixture, + so we don't spam space-track.org or CelesTrak.""" + mock = Mock() + mocker.patch.object( + TLEEntry, "find_tles_between_epochs", return_value=[burstcube_tle] + ) + return mock + + +@pytest.fixture(scope="function") +def aws_credentials(): + """Mocked AWS Credentials for moto.""" + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "us-east-1" + + +@pytest.fixture +def dynamodb(aws_credentials): + with mock_aws(): + yield boto3.client("dynamodb", region_name="us-east-1") + + +@pytest.fixture(scope="function") +def create_too_table(dynamodb): + """Create the TOO Table""" + dynamodb.create_table( + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + ], + TableName=BurstCubeTOOModel.__tablename__, + KeySchema=[ + {"AttributeName": "id", "KeyType": "HASH"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + + +@pytest.fixture(scope="function") +def create_tle_table(dynamodb): + """Create the TLE Table so we can use BurstCubeEphem in tests""" + dynamodb.create_table( + TableName="acrossapi_tle", + AttributeDefinitions=[ + {"AttributeName": "satname", "AttributeType": "S"}, + {"AttributeName": "epoch", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "satname", "KeyType": "HASH"}, + {"AttributeName": "epoch", "KeyType": "RANGE"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + + +@pytest.fixture +def now(): + yield Time.now() + + +@pytest.fixture +def username(): + yield "testuser" + + +@pytest.fixture +def trigger_info(): + yield BurstCubeTriggerInfo( + trigger_mission="BurstCube", + trigger_type="GRB", + ) + + +@pytest.fixture +def burstcube_too(username, now, trigger_info): + too = BurstCubeTOO( + trigger_time=now, + trigger_info=trigger_info, + username=username, + ) + yield too + + +@pytest.fixture +def burstcube_old_too(username, trigger_info): + too = BurstCubeTOO( + trigger_time=Time.now() - TimeDelta(48 * u.hr), + trigger_info=trigger_info, + username=username, + ) + yield too diff --git a/python/tests/burstcubetoo/test_burstcube_too.py b/python/tests/burstcubetoo/test_burstcube_too.py new file mode 100644 index 0000000000..9083b709d9 --- /dev/null +++ b/python/tests/burstcubetoo/test_burstcube_too.py @@ -0,0 +1,70 @@ +import astropy.units as u # type: ignore[import] +from across_api.burstcube.schema import TOOReason # type: ignore[import] +from across_api.burstcube.toorequest import BurstCubeTOO # type: ignore[import] +from astropy.time.core import Time, TimeDelta # type: ignore[import] +from moto import mock_aws + + +@mock_aws +def test_burstcube_too_crud( + username, burstcube_too, create_tle_table, create_too_table, mock_read_tle_db +): + assert burstcube_too.post() is True + assert burstcube_too.id is not None + + # Test fetching posted TOO + too = BurstCubeTOO(id=burstcube_too.id) + too.get() + assert too.id == burstcube_too.id + + assert too.trigger_time == burstcube_too.trigger_time + assert too.trigger_info == burstcube_too.trigger_info + assert too.created_by == burstcube_too.username + assert too.status == "Requested" or too.status == "Rejected" + assert too.trigger_info.trigger_mission == "BurstCube" + assert too.trigger_info.trigger_type == "GRB" + # If the TOO was rejected, it should only be rejected due to SAA as no + # coordinates were given, and trigger time is less than 48 hours old + assert ( + too.reject_reason == TOOReason.none + and too.status == "Requested" + or too.reject_reason == TOOReason.saa + and too.status == "Rejected" + ) + + # Test deleting posted TOO + too = BurstCubeTOO(id=burstcube_too.id, username=username) + too.delete() + + too = BurstCubeTOO(id=burstcube_too.id) + too.get() + assert too.status == "Deleted" + + # Test changing status to Approved + too.status = "Approved" + too.put() + + too = BurstCubeTOO(id=burstcube_too.id) + too.get() + assert too.status == "Approved" + + +def test_burstcube_old_too( + burstcube_old_too, create_tle_table, create_too_table, mock_read_tle_db +): + assert burstcube_old_too.post() is True + assert burstcube_old_too.trigger_time < Time.now() - TimeDelta(48 * u.hr) + assert burstcube_old_too.status == "Rejected" + assert "Trigger is too old." in burstcube_old_too.too_info + assert burstcube_old_too.reject_reason == TOOReason.too_old + + +def test_burstcube_too_double_post( + burstcube_too, create_tle_table, create_too_table, mock_read_tle_db +): + assert burstcube_too.post() is True + try: + burstcube_too.post() + except Exception as e: + assert e.status_code == 409 + assert e.detail == "BurstCubeTOO already exists." diff --git a/requirements.txt b/requirements.txt index e29250cc5f..8b8b28d493 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,9 @@ boto3 mypy ruff hypothesis +moto pytest +pytest-mock types-requests types-cachetools types-python-jose