diff --git a/CHANGES.md b/CHANGES.md index 7b2d707..030f7b8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,11 @@ * Add tests for CLI and implementations when invoked through the CLI * Refactored code dealing with requests and authentication to the `STACpopulator/requests.py` file * Add `--log-file` command line option to specify a non-default location to write log files to +* fix incorrect example in README +* move argument parsing for logging options to the implementation code +* fix bug where logging options were being set incorrectly +* rename files to avoid potential naming conflicts with other packages (`logging` and `requests`) + ## [0.6.0](https://github.com/crim-ca/stac-populator/tree/0.6.0) (2024-02-22) diff --git a/README.md b/README.md index 8cad8a9..06563c7 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ For example, using the [CMIP6_UofT][CMIP6_UofT] implementation, the script can b python STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py \ "http://localhost:8880/stac/" \ "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html" \ - "STACpopulator/implementations/CMIP6_UofT/collection_config.yml" + --config "STACpopulator/implementations/CMIP6_UofT/collection_config.yml" ``` *Note*:
diff --git a/STACpopulator/cli.py b/STACpopulator/cli.py index 11ed00f..00d8522 100644 --- a/STACpopulator/cli.py +++ b/STACpopulator/cli.py @@ -5,15 +5,13 @@ import sys from types import ModuleType import warnings -from datetime import datetime, timezone -from typing import Callable from STACpopulator import __version__, implementations from STACpopulator.exceptions import STACPopulatorError -from STACpopulator.logging import setup_logging +from STACpopulator.log import setup_logging -def add_parser_args(parser: argparse.ArgumentParser) -> dict[str, Callable]: +def add_parser_args(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--version", "-V", @@ -21,10 +19,6 @@ def add_parser_args(parser: argparse.ArgumentParser) -> dict[str, Callable]: version=f"%(prog)s {__version__}", help="prints the version of the library and exits", ) - parser.add_argument("--debug", action="store_const", const=logging.DEBUG, help="set logger level to debug") - parser.add_argument( - "--log-file", help="file to write log output to. By default logs will be written to the current directory." - ) commands_subparser = parser.add_subparsers( title="command", dest="command", description="STAC populator command to execute.", required=True ) @@ -52,8 +46,7 @@ def implementation_modules() -> dict[str, ModuleType]: def run(ns: argparse.Namespace) -> int: if ns.command == "run": - logfile_name = ns.log_file or f"{ns.populator}_log_{datetime.now(timezone.utc).isoformat() + 'Z'}.jsonl" - setup_logging(logfile_name, ns.debug or logging.INFO) + setup_logging(ns.log_file, ns.debug or logging.INFO) return implementation_modules()[ns.populator].runner(ns) or 0 diff --git a/STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py b/STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py index 7db2d53..336cd23 100644 --- a/STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py +++ b/STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py @@ -9,7 +9,9 @@ from pystac.extensions.datacube import DatacubeExtension from requests.sessions import Session -from STACpopulator.requests import add_request_options, apply_request_options +from STACpopulator import cli +from STACpopulator.log import add_logging_options +from STACpopulator.request_utils import add_request_options, apply_request_options from STACpopulator.extensions.cmip6 import CMIP6Helper, CMIP6Properties from STACpopulator.extensions.datacube import DataCubeHelper from STACpopulator.extensions.thredds import THREDDSExtension, THREDDSHelper @@ -31,7 +33,6 @@ def __init__( update: Optional[bool] = False, session: Optional[Session] = None, config_file: Optional[Union[os.PathLike[str], str]] = None, - log_debug: Optional[bool] = False, ) -> None: """Constructor @@ -40,7 +41,7 @@ def __init__( :param data_loader: loader to iterate over ingestion data. """ super().__init__( - stac_host, data_loader, update=update, session=session, config_file=config_file, log_debug=log_debug + stac_host, data_loader, update=update, session=session, config_file=config_file ) def create_stac_item( @@ -106,6 +107,7 @@ def add_parser_args(parser: argparse.ArgumentParser) -> None: ), ) add_request_options(parser) + add_logging_options(parser) def runner(ns: argparse.Namespace) -> int: @@ -120,7 +122,7 @@ def runner(ns: argparse.Namespace) -> int: data_loader = ErrorLoader() c = CMIP6populator( - ns.stac_host, data_loader, update=ns.update, session=session, config_file=ns.config, log_debug=ns.debug + ns.stac_host, data_loader, update=ns.update, session=session, config_file=ns.config ) c.ingest() return 0 @@ -130,8 +132,8 @@ def main(*args: str) -> int: parser = argparse.ArgumentParser() add_parser_args(parser) ns = parser.parse_args(args or None) - return runner(ns) - + ns.populator = os.path.basename(os.path.dirname(__file__)) + return cli.run(ns) if __name__ == "__main__": sys.exit(main()) diff --git a/STACpopulator/implementations/DirectoryLoader/crawl_directory.py b/STACpopulator/implementations/DirectoryLoader/crawl_directory.py index 9b4cf01..cb6cd9b 100644 --- a/STACpopulator/implementations/DirectoryLoader/crawl_directory.py +++ b/STACpopulator/implementations/DirectoryLoader/crawl_directory.py @@ -6,7 +6,9 @@ from requests.sessions import Session -from STACpopulator.requests import add_request_options, apply_request_options +from STACpopulator import cli +from STACpopulator.log import add_logging_options +from STACpopulator.request_utils import add_request_options, apply_request_options from STACpopulator.input import STACDirectoryLoader from STACpopulator.models import GeoJSONPolygon from STACpopulator.populator_base import STACpopulatorBase @@ -51,6 +53,7 @@ def add_parser_args(parser: argparse.ArgumentParser) -> None: help="Limit search of STAC Collections only to first top-most matches in the crawled directory structure.", ) add_request_options(parser) + add_logging_options(parser) def runner(ns: argparse.Namespace) -> int: @@ -70,7 +73,8 @@ def main(*args: str) -> int: parser = argparse.ArgumentParser() add_parser_args(parser) ns = parser.parse_args(args or None) - return runner(ns) + ns.populator = os.path.basename(os.path.dirname(__file__)) + return cli.run(ns) if __name__ == "__main__": diff --git a/STACpopulator/logging.py b/STACpopulator/log.py similarity index 87% rename from STACpopulator/logging.py rename to STACpopulator/log.py index e278360..f195b47 100644 --- a/STACpopulator/logging.py +++ b/STACpopulator/log.py @@ -1,3 +1,4 @@ +import argparse import datetime as dt import json import logging.config @@ -126,3 +127,14 @@ def filter(self, record: logging.LogRecord) -> bool | logging.LogRecord: }, "loggers": {"root": {"level": "DEBUG", "handlers": ["stderr", "file"]}}, } + +def add_logging_options(parser: argparse.ArgumentParser) -> None: + """ + Adds arguments to a parser to configure logging options. + """ + parser.add_argument("--debug", action="store_const", const=logging.DEBUG, help="set logger level to debug") + parser.add_argument( + "--log-file", + default=f"stac_populator_log_{dt.datetime.now(dt.timezone.utc).isoformat() + 'Z'}.jsonl", + help="file to write log output to. By default logs will be written to the current directory." + ) diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 44341f3..8305cf9 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -27,10 +27,9 @@ def __init__( self, stac_host: str, data_loader: GenericLoader, - update: Optional[bool] = False, + update: bool = False, session: Optional[Session] = None, config_file: Optional[Union[os.PathLike[str], str]] = "collection_config.yml", - log_debug: Optional[bool] = False, ) -> None: """Constructor @@ -41,7 +40,6 @@ def __init__( :raises RuntimeError: Raised if one of the required definitions is not found in the collection info filename """ - super().__init__() self._collection_config_path = config_file self._collection_info: MutableMapping[str, Any] = None self._session = session diff --git a/STACpopulator/requests.py b/STACpopulator/request_utils.py similarity index 100% rename from STACpopulator/requests.py rename to STACpopulator/request_utils.py