Skip to content

Commit

Permalink
List api returns new uri
Browse files Browse the repository at this point in the history
  • Loading branch information
@thorbjoernl
thorbjoernl committed Aug 2, 2024
1 parent 59a76df commit 4836e13
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 73 deletions.
132 changes: 60 additions & 72 deletions src/aerovaldb/jsondb/jsonfiledb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
TemplateMapper,
DataVersionToTemplateMapper,
PriorityDataVersionToTemplateMapper,
ConstantTemplateMapper,
SkipMapper,
)
from .filter import filter_heatmap, filter_regional_stats
Expand Down Expand Up @@ -62,74 +63,52 @@ def __init__(self, basedir: str | Path, /, use_async: bool = False):

self.PATH_LOOKUP: dict[str, list[TemplateMapper]] = {
ROUTE_GLOB_STATS: [
DataVersionToTemplateMapper(
"./{project}/{experiment}/hm/glob_stats_{frequency}.json",
version_provider=self._get_version,
ConstantTemplateMapper(
"./{project}/{experiment}/hm/glob_stats_{frequency}.json"
)
],
ROUTE_REG_STATS: [
# Same as glob_stats
DataVersionToTemplateMapper(
"./{project}/{experiment}/hm/glob_stats_{frequency}.json",
version_provider=self._get_version,
ConstantTemplateMapper(
"./{project}/{experiment}/hm/glob_stats_{frequency}.json"
)
],
ROUTE_HEATMAP: [
# Same as glob_stats
DataVersionToTemplateMapper(
"./{project}/{experiment}/hm/glob_stats_{frequency}.json",
version_provider=self._get_version,
ConstantTemplateMapper(
"./{project}/{experiment}/hm/glob_stats_{frequency}.json"
)
],
ROUTE_CONTOUR: [
DataVersionToTemplateMapper(
"./{project}/{experiment}/contour/{obsvar}_{model}.geojson",
version_provider=self._get_version,
ConstantTemplateMapper(
"./{project}/{experiment}/contour/{obsvar}_{model}.geojson"
)
],
ROUTE_TIMESERIES: [
DataVersionToTemplateMapper(
"./{project}/{experiment}/ts/{location}_{network}-{obsvar}_{layer}.json",
version_provider=self._get_version,
ConstantTemplateMapper(
"./{project}/{experiment}/ts/{location}_{network}-{obsvar}_{layer}.json"
)
],
ROUTE_TIMESERIES_WEEKLY: [
DataVersionToTemplateMapper(
"./{project}/{experiment}/ts/diurnal/{location}_{network}-{obsvar}_{layer}.json",
version_provider=self._get_version,
ConstantTemplateMapper(
"./{project}/{experiment}/ts/diurnal/{location}_{network}-{obsvar}_{layer}.json"
)
],
ROUTE_EXPERIMENTS: [
PriorityDataVersionToTemplateMapper(["./{project}/experiments.json"])
],
ROUTE_EXPERIMENTS: [ConstantTemplateMapper("./{project}/experiments.json")],
ROUTE_CONFIG: [
PriorityDataVersionToTemplateMapper(
["./{project}/{experiment}/cfg_{project}_{experiment}.json"]
)
],
ROUTE_MENU: [
DataVersionToTemplateMapper(
"./{project}/{experiment}/menu.json",
version_provider=self._get_version,
ConstantTemplateMapper(
"./{project}/{experiment}/cfg_{project}_{experiment}.json"
)
],
ROUTE_MENU: [ConstantTemplateMapper("./{project}/{experiment}/menu.json")],
ROUTE_STATISTICS: [
DataVersionToTemplateMapper(
"./{project}/{experiment}/statistics.json",
version_provider=self._get_version,
)
ConstantTemplateMapper("./{project}/{experiment}/statistics.json")
],
ROUTE_RANGES: [
DataVersionToTemplateMapper(
"./{project}/{experiment}/ranges.json",
version_provider=self._get_version,
)
ConstantTemplateMapper("./{project}/{experiment}/ranges.json")
],
ROUTE_REGIONS: [
DataVersionToTemplateMapper(
"./{project}/{experiment}/regions.json",
version_provider=self._get_version,
)
ConstantTemplateMapper("./{project}/{experiment}/regions.json")
],
ROUTE_MODELS_STYLE: [
PriorityDataVersionToTemplateMapper(
Expand Down Expand Up @@ -164,9 +143,8 @@ def __init__(self, basedir: str | Path, /, use_async: bool = False):
),
],
ROUTE_PROFILES: [
DataVersionToTemplateMapper(
"./{project}/{experiment}/profiles/{location}_{network}-{obsvar}.json",
version_provider=self._get_version,
ConstantTemplateMapper(
"./{project}/{experiment}/profiles/{location}_{network}-{obsvar}.json"
)
],
ROUTE_HEATMAP_TIMESERIES: [
Expand All @@ -188,22 +166,17 @@ def __init__(self, basedir: str | Path, /, use_async: bool = False):
),
],
ROUTE_FORECAST: [
DataVersionToTemplateMapper(
"./{project}/{experiment}/forecast/{region}_{network}-{obsvar}_{layer}.json",
version_provider=self._get_version,
ConstantTemplateMapper(
"./{project}/{experiment}/forecast/{region}_{network}-{obsvar}_{layer}.json"
)
],
ROUTE_GRIDDED_MAP: [
DataVersionToTemplateMapper(
"./{project}/{experiment}/contour/{obsvar}_{model}.json",
version_provider=self._get_version,
ConstantTemplateMapper(
"./{project}/{experiment}/contour/{obsvar}_{model}.json"
)
],
ROUTE_REPORT: [
DataVersionToTemplateMapper(
"./reports/{project}/{experiment}/{title}.json",
version_provider=self._get_version,
)
ConstantTemplateMapper("./reports/{project}/{experiment}/{title}.json")
],
}

Expand Down Expand Up @@ -237,7 +210,7 @@ async def _get_version(self, project: str, experiment: str) -> Version:
version = Version("0.0.1")
finally:
return version
except simplejson.errors.JSONDecodeError:
except simplejson.JSONDecodeError:
# Work around for https://github.com/metno/aerovaldb/issues/28
return Version("0.14.0")

Expand Down Expand Up @@ -278,6 +251,16 @@ async def _get_template(self, route: str, substitutions: dict) -> str:

return file_path_template

def _get_templates(self, route: str) -> list[str]:
templates = list()

for f in self.PATH_LOOKUP[route]:
templates.extend(f.get_templates_without_constraints())
if isinstance(f, ConstantTemplateMapper):
break

return templates

async def _get(
self,
route,
Expand Down Expand Up @@ -484,8 +467,7 @@ async def get_heatmap(
cache=True,
)

@async_and_sync
async def _get_uri_for_file(self, file_path: str) -> str:
def _get_uri_for_file(self, file_path: str) -> str:
"""
For the provided data file path, returns the corresponding
URI.
Expand All @@ -495,20 +477,26 @@ async def _get_uri_for_file(self, file_path: str) -> str:
file_path = os.path.relpath(file_path, start=self._basedir)

for route in self.PATH_LOOKUP:
template = await self._get_template(route, {})
route_arg_names = extract_substitutions(template)
templates = self._get_templates(route)

try:
all_args = parse_formatted_string(f"./{template}", file_path)
for t in templates:
route_arg_names = extract_substitutions(t)

route_args = {k: v for k, v in all_args.items() if k in route_arg_names}
kwargs = {
k: v for k, v in all_args.items() if not (k in route_arg_names)
}
except:
continue
else:
return build_uri(route, route_args, kwargs)
try:
all_args = parse_formatted_string(t, f"./{file_path}")

route_args = {
k: v for k, v in all_args.items() if k in route_arg_names
}
kwargs = {
k: v for k, v in all_args.items() if not (k in route_arg_names)
}
except:
continue
else:
return build_uri(route, route_args, kwargs)

raise ValueError(f"Unable to build URI for file path {file_path}")

def list_glob_stats(
self, project: str, experiment: str
Expand All @@ -527,7 +515,7 @@ def list_glob_stats(

glb = glb.format(project=project, experiment=experiment)
for f in glob.glob(glb):
yield f
yield self._get_uri_for_file(f)

def list_timeseries(
self, project: str, experiment: str
Expand All @@ -552,7 +540,7 @@ def list_timeseries(
glb = glb.format(project=project, experiment=experiment)

for f in glob.glob(glb):
yield f
yield self._get_uri_for_file(f)

def list_map(self, project: str, experiment: str) -> Generator[str, None, None]:
template = str(
Expand All @@ -577,7 +565,7 @@ def list_map(self, project: str, experiment: str) -> Generator[str, None, None]:
glb = glb.format(project=project, experiment=experiment)

for f in glob.glob(glb):
yield f
yield self._get_uri_for_file(f)

def _list_experiments(
self, project: str, /, has_results: bool = False
Expand Down
21 changes: 21 additions & 0 deletions src/aerovaldb/jsondb/templatemapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ class TemplateMapper(abc.ABC):
async def __call__(self, *args, version_provider: VersionProvider, **kwargs) -> str:
raise NotImplementedError

def get_templates_without_constraints(self) -> list[str]:
raise NotImplementedError


class DataVersionToTemplateMapper(TemplateMapper):
"""
Expand Down Expand Up @@ -69,6 +72,9 @@ def __init__(

self.version_provider = version_provider

def get_templates_without_constraints(self):
return [self.template]

@async_and_sync
async def __call__(self, *args, **kwargs) -> str:
logger.debug(f"Trying template string {self.template}")
Expand Down Expand Up @@ -112,3 +118,18 @@ async def __call__(self, *args, **kwargs) -> str:
raise SkipMapper

return selected_template

def get_templates_without_constraints(self):
return self.templates


class ConstantTemplateMapper(TemplateMapper):
def __init__(self, template: str):
self.template = template

@async_and_sync
async def __call__(self, *args, **kwargs) -> str:
return self.template

def get_templates_without_constraints(self):
return [self.template]
2 changes: 1 addition & 1 deletion src/aerovaldb/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ def parse_uri(uri: str) -> tuple[str, dict[str, str], dict[str, str]]:

def build_uri(route: str, route_args: dict, kwargs: dict = {}) -> str:
uri = route.format(**route_args)
if not kwargs:
if kwargs:
queries = "&".join([f"{k}={v}" for k, v in kwargs.items()])
uri = f"{uri}?{queries}"

Expand Down

0 comments on commit 4836e13

Please sign in to comment.