From 89288d6e6d88f60aba05682ed3f47d634b3e8da4 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 22 Jan 2024 11:16:41 +0000 Subject: [PATCH 1/5] update requirments --- requirements.txt | 5 ++++- setup.cfg | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2d9a4e4..abd188f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,7 @@ lxml pika rich - pe \ No newline at end of file + pe + findlibs + pyfdb @ git+https://github.com/ecmwf/pyfdb@develop + aviso @ git+https://github.com/ecmwf/pyfdb@develop \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index f01dc13..fa8b3e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,9 @@ install_requires = pika rich pe + findlibs + # aviso @ git+https://github.com/ecmwf/pyfdb@develop + pyfdb @ git+https://github.com/ecmwf/pyfdb@develop [options.extras_require] From 36809b1a0c3918a9869bcf1619ef8d981a2ef456 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 22 Jan 2024 11:16:53 +0000 Subject: [PATCH 2/5] add new fdb schema format --- src/ionbeam/writers/fdb_writer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ionbeam/writers/fdb_writer.py b/src/ionbeam/writers/fdb_writer.py index 2e36869..598c6dd 100644 --- a/src/ionbeam/writers/fdb_writer.py +++ b/src/ionbeam/writers/fdb_writer.py @@ -30,10 +30,10 @@ class FB5Config: type: str = "remote" host: str = "localhost" - port: int = 7654 - dataPortStart: int = 40000 - dataPortCount: int = 100 + port: int = 8000 schema: Path | None = None + engine: str = "remote" + store: str = "remote" def asdict(self): d = dataclasses.asdict(self) From fcc8c7b08821381371b60afda7f647116d1ff742 Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 24 Jan 2024 12:12:38 +0000 Subject: [PATCH 3/5] make mars_request work --- src/ionbeam/writers/construct_mars_request.py | 8 +++-- src/ionbeam/writers/mars_client_writer.py | 34 ++++++++++--------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/ionbeam/writers/construct_mars_request.py b/src/ionbeam/writers/construct_mars_request.py index 9009ed6..5520695 100644 --- a/src/ionbeam/writers/construct_mars_request.py +++ b/src/ionbeam/writers/construct_mars_request.py @@ -13,7 +13,9 @@ } -def construct_mars_request(message): - request = {"database": "fdbdev", "class": "rd", "source": str(message.metadata.filepath)} - request |= message.metadata.mars_request.as_strings() +def construct_mars_request(message, override=dict(database="fdbdev")): + source = {"source": message.metadata.filepath} + odb_keys = {k.key: k.value for k in message.metadata.mars_keys if not k.reason == "Skipped"} + request = odb_keys | source | override # rightmost dict takes precedence here + request = {k: mars_value_formatters.get(k, str)(v) for k, v in request.items()} return request diff --git a/src/ionbeam/writers/mars_client_writer.py b/src/ionbeam/writers/mars_client_writer.py index b236f2b..2cad2e1 100644 --- a/src/ionbeam/writers/mars_client_writer.py +++ b/src/ionbeam/writers/mars_client_writer.py @@ -16,6 +16,7 @@ from ..core.bases import Writer, Message, FileMessage, FinishMessage from ..core.aviso import send_aviso_notification +from .construct_mars_request import construct_mars_request import logging @@ -33,14 +34,14 @@ def write_temp_mars_request( request, file, verb="archive", - quote_keys = {"source", "target", "filter"} + # quote_keys = {"source", "target", "filter"} ): keys = [] for key, value in request.items(): - if key in quote_keys: - keys.append(f'{key}="{value}"') - else: - keys.append(f'{key}={value}') + # if key in quote_keys: + keys.append(f'{key}="{value}"') + # else: + # keys.append(f'{key}={value}') keys = ",\n ".join(keys) rendered = f"""{verb}, @@ -58,8 +59,8 @@ def run_temp_mars_request(file): except sb.CalledProcessError as exc: print("Status : FAIL", exc.returncode, exc.output.decode()) raise exc - else: - logger.debug("MARS Archive Output: \n{}\n".format(output)) + # else: + # print("Output: \n{}\n".format(output)) @dataclasses.dataclass @@ -75,22 +76,23 @@ def process(self, message: FileMessage | FinishMessage) -> Iterable[Message]: if isinstance(message, FinishMessage): return - assert message.metadata.mars_request is not None + assert message.metadata.mars_keys is not None assert message.metadata.filepath is not None - - request = {"database": "fdbdev", "class": "rd", "source": str(message.metadata.filepath)} - mars_request = request | message.metadata.mars_request.as_strings() - - logger.info(f"mars_request: {mars_request}") + + mars_request = construct_mars_request(message) with tempfile.NamedTemporaryFile() as fp: mars_request_string = write_temp_mars_request(mars_request, file=fp.name) - logger.debug(f"mars_request_string: {mars_request_string}") + logger.debug(mars_request_string) run_temp_mars_request(file=fp.name) + # Send a notification to AVISO that we put this data into the DB + response = send_aviso_notification(mars_request) + # logger.debug("Aviso respose {response}") + # TODO: the explicit mars_keys should not be necessary here. - metadata = self.generate_metadata(message, mars_request=message.metadata.mars_request) + metadata = self.generate_metadata(message, mars_keys=message.metadata.mars_keys) output_msg = FileMessage(metadata=metadata) - assert output_msg.metadata.mars_request is not None + assert output_msg.metadata.mars_keys is not None yield self.tag_message(output_msg, message) From 223b3d21d82646cdbdc32684b971586d03140adc Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 30 Jan 2024 12:42:26 +0000 Subject: [PATCH 4/5] bugfixes --- src/ionbeam/__main__.py | 13 +++++++------ src/ionbeam/writers/mars_client_writer.py | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/ionbeam/__main__.py b/src/ionbeam/__main__.py index 88efc93..aea63db 100755 --- a/src/ionbeam/__main__.py +++ b/src/ionbeam/__main__.py @@ -91,7 +91,6 @@ handlers = [RichHandler(markup=True, rich_tracebacks=True)] if args.logfile: handlers.append(logging.FileHandler(args.logfile)) - # Set the log level, default is warnings, -v gives info, -vv for debug logging.basicConfig( @@ -120,11 +119,13 @@ logger.info(f" Data Path: {globals.globals.data_path}") logger.info(f" Offline: {globals.globals.offline}") logger.info(f" Overwrite: {globals.globals.overwrite}") - logger.info(f" Ingestion Time Constants:") - logger.info(f" Query Timespan: {tuple(d.isoformat() for d in globals.globals.ingestion_time_constants.query_timespan)}") - logger.info(f" Emit After (Hours): {globals.globals.ingestion_time_constants.emit_after_hours}") - logger.info(f" Granularity: {globals.globals.ingestion_time_constants.granularity}") - + if globals.globals.ingestion_time_constants is not None: + logger.info(f" Ingestion Time Constants:") + logger.info( + f" Query Timespan: {tuple(d.isoformat() for d in globals.globals.ingestion_time_constants.query_timespan)}" + ) + logger.info(f" Emit After (Hours): {globals.globals.ingestion_time_constants.emit_after_hours}") + logger.info(f" Granularity: {globals.globals.ingestion_time_constants.granularity}") logger.info("Sources") for i, a in enumerate(sources): diff --git a/src/ionbeam/writers/mars_client_writer.py b/src/ionbeam/writers/mars_client_writer.py index 33227a7..0d484ca 100644 --- a/src/ionbeam/writers/mars_client_writer.py +++ b/src/ionbeam/writers/mars_client_writer.py @@ -92,7 +92,7 @@ def process(self, message: FileMessage | FinishMessage) -> Iterable[Message]: if isinstance(message, FinishMessage): return - assert message.metadata.mars_keys is not None + assert message.metadata.mars_request is not None assert message.metadata.filepath is not None mars_request = message.metadata.mars_request.as_strings() @@ -114,8 +114,8 @@ def process(self, message: FileMessage | FinishMessage) -> Iterable[Message]: # logger.debug("Aviso respose {response}") # TODO: the explicit mars_keys should not be necessary here. - metadata = self.generate_metadata(message, mars_keys=message.metadata.mars_keys) + metadata = self.generate_metadata(message, mars_request=message.metadata.mars_request) output_msg = FileMessage(metadata=metadata) - assert output_msg.metadata.mars_keys is not None + assert output_msg.metadata.mars_request is not None yield self.tag_message(output_msg, message) From e4840bdac1edfe3b71b161bd2f5d966dac179da7 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 15 Feb 2024 11:25:34 +0100 Subject: [PATCH 5/5] Fix git links in requirments --- requirements.txt | 2 +- setup.cfg | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index abd188f..d374c1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,4 +19,4 @@ pe findlibs pyfdb @ git+https://github.com/ecmwf/pyfdb@develop - aviso @ git+https://github.com/ecmwf/pyfdb@develop \ No newline at end of file + aviso @ git+https://github.com/ecmwf/aviso@develop \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index c20bf67..b736e34 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,8 +20,9 @@ python_requires = >=3.10 install_requires = pandas numexpr - pyodc @ git+https://github.com/TomHodson/pyodc@test/string_hack - pyaviso @ git+https://github.com/ecmwf/aviso@develop + pyodc @ git+https://github.com/TomHodson/pyodc.git@test/string_hack + pyaviso @ git+https://github.com/ecmwf/aviso.git@develop + pyfdb @ git+https://github.com/ecmwf/pyfdb.git@develop requests oauthlib requests-oauthlib @@ -39,8 +40,6 @@ install_requires = rich pe findlibs - # aviso @ git+https://github.com/ecmwf/pyfdb@develop - pyfdb @ git+https://github.com/ecmwf/pyfdb@develop [options.extras_require]