From 3fbd636df733ba87da61baf827b12d75aa69bd9c Mon Sep 17 00:00:00 2001 From: ejseqera Date: Wed, 28 Feb 2024 16:03:22 -0500 Subject: [PATCH 1/5] fix: type parsing for ces,actions,credentials Re #117 --- seqerakit/helper.py | 51 ++++++++++++------------------ tests/unit/test_helper.py | 65 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 80 insertions(+), 36 deletions(-) diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 2b47599..0b71311 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -113,10 +113,10 @@ def parse_all_yaml(file_paths, destroy=False): def parse_block(block_name, item): # Define the mapping from block names to functions. block_to_function = { - "credentials": parse_credentials_block, - "compute-envs": parse_compute_envs_block, + "credentials": parse_type_block, + "compute-envs": parse_type_block, + "actions": parse_type_block, "teams": parse_teams_block, - "actions": parse_actions_block, "datasets": parse_datasets_block, "pipelines": parse_pipelines_block, "launch": parse_launch_block, @@ -141,27 +141,28 @@ def parse_generic_block(item): return cmd_args -def parse_credentials_block(item): +def parse_type_block(item, priority_keys=["type", "config-mode", "file-path"]): cmd_args = [] - for key, value in item.items(): - if key == "type": - cmd_args.append(str(value)) - elif isinstance(value, bool): - if value: - cmd_args.append(f"--{key}") - else: - cmd_args.extend([f"--{key}", str(value)]) - return cmd_args + # Ensure at least one of 'type' or 'file-path' is present + if not any(key in item for key in ["type", "file-path"]): + raise ValueError( + "Please specify at least 'type' or 'file-path' for creating the resource." + ) + + # Process priority keys first + for key in priority_keys: + if key in item: + cmd_args.append(str(item[key])) + del item[key] # Remove the key to avoid repeating in args -def parse_compute_envs_block(item): - cmd_args = [] for key, value in item.items(): - if key == "file-path" or key == "type" or key == "config-mode": - cmd_args.append(str(value)) - elif isinstance(value, bool): + if isinstance(value, bool): if value: cmd_args.append(f"--{key}") + elif key == "params": + temp_file_name = utils.create_temp_yaml(value) + cmd_args.extend(["--params-file", temp_file_name]) else: cmd_args.extend([f"--{key}", str(value)]) return cmd_args @@ -194,20 +195,6 @@ def parse_teams_block(item): return (cmd_args, members_cmd_args) -def parse_actions_block(item): - cmd_args = [] - temp_file_name = None - for key, value in item.items(): - if key == "type": - cmd_args.append(str(value)) - elif key == "params": - temp_file_name = utils.create_temp_yaml(value) - cmd_args.extend(["--params-file", temp_file_name]) - else: - cmd_args.extend([f"--{key}", str(value)]) - return cmd_args - - def parse_datasets_block(item): cmd_args = [] for key, value in item.items(): diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index ab7158c..66b1ec7 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -130,14 +130,14 @@ def test_create_mock_dataset_yaml(mock_yaml_file): assert result["datasets"] == expected_block_output -def test_create_mock_computeevs_yaml(mock_yaml_file): +def test_create_mock_computeevs_source_yaml(mock_yaml_file): test_data = { "compute-envs": [ { "name": "test_computeenv", "workspace": "my_organization/my_workspace", "credentials": "my_credentials", - "file-path": "./examples/yaml/computeenvs/computeenvs.yaml", + "file-path": "./computeenvs/computeenv.json", "wait": "AVAILABLE", "fusion-v2": True, "fargate": False, @@ -149,9 +149,9 @@ def test_create_mock_computeevs_yaml(mock_yaml_file): expected_block_output = [ { "cmd_args": [ + "./computeenvs/computeenv.json", "--credentials", "my_credentials", - "./examples/yaml/computeenvs/computeenvs.yaml", "--fusion-v2", "--name", "test_computeenv", @@ -171,6 +171,43 @@ def test_create_mock_computeevs_yaml(mock_yaml_file): assert result["compute-envs"] == expected_block_output +def test_create_mock_computeevs_cli_yaml(mock_yaml_file): + test_data = { + "compute-envs": [ + { + "name": "test_computeenv", + "workspace": "my_organization/my_workspace", + "credentials": "my_credentials", + "type": "aws-batch", + "config-mode": "forge", + "wait": "AVAILABLE", + } + ], + } + + expected_block_output = [ + { + "cmd_args": [ + "aws-batch", + "forge", + "--credentials", + "my_credentials", + "--name", + "test_computeenv", + "--wait", + "AVAILABLE", + "--workspace", + "my_organization/my_workspace", + ], + "overwrite": False, + } + ] + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + assert "compute-envs" in result + assert result["compute-envs"] == expected_block_output + + def test_create_mock_pipeline_add_yaml(mock_yaml_file): test_data = { "pipelines": [ @@ -191,7 +228,6 @@ def test_create_mock_pipeline_add_yaml(mock_yaml_file): } ] } - # params file cmds parsed separately expected_block_output = [ { @@ -239,3 +275,24 @@ def test_empty_yaml_file(mock_yaml_file): assert f"The file '{file_path}' is empty or does not contain valid data." in str( e.value ) + + +def test_error_type_yaml_file(mock_yaml_file): + test_data = { + "compute-envs": [ + { + "name": "test_computeenv", + "workspace": "my_organization/my_workspace", + "credentials": "my_credentials", + "wait": "AVAILABLE", + } + ], + } + file_path = mock_yaml_file(test_data) + + with pytest.raises(ValueError) as e: + helper.parse_all_yaml([file_path]) + assert ( + "Please specify at least 'type' or 'file-path' for creating the resource." + in str(e.value) + ) From 09f04bf3facda41ad2dc7ad1fbc30a4c07e989a1 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Sun, 3 Mar 2024 22:07:03 -0500 Subject: [PATCH 2/5] fix: fix delete/overwrite bug with workspaces Re #115 --- seqerakit/overwrite.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/seqerakit/overwrite.py b/seqerakit/overwrite.py index dd6f510..7a18562 100644 --- a/seqerakit/overwrite.py +++ b/seqerakit/overwrite.py @@ -73,7 +73,7 @@ def __init__(self, sp): "workspaces": { "keys": ["name", "organization"], "method_args": self._get_workspace_args, - "name_key": "workspaceId", + "name_key": "workspaceName", }, } @@ -101,7 +101,6 @@ def handle_overwrite(self, block, args, overwrite=False, destroy=False): self.block_operations["participants"]["name_key"] = "teamName" else: self.block_operations["participants"]["name_key"] = "email" - if self.check_resource_exists(operation["name_key"], sp_args): # if resource exists and overwrite is true, delete if overwrite: @@ -169,11 +168,7 @@ def _get_workspace_args(self, args): workspaceId used to delete will be retrieved using the _find_workspace_id() method. """ - workspace_id = self._find_workspace_id( - json.loads(self.sp.workspaces.list("-o", "json")), - args["organization"], - args["name"], - ) + workspace_id = self._find_workspace_id(args["organization"], args["name"]) return ("delete", "--id", str(workspace_id)) def _get_generic_deletion_args(self, args): @@ -267,12 +262,12 @@ def _find_workspace_id(self, organization, workspace_name): organization name and workspace name. This ID will be used to delete the workspace. """ - if "workspaces" in self.cached_jsondata: - workspaces = self.cached_jsondata["workspaces"] - for workspace in workspaces: - if ( - workspace.get("orgName") == organization - and workspace.get("workspaceName") == workspace_name - ): - return workspace.get("workspaceId") + jsondata = json.loads(self.cached_jsondata) + workspaces = jsondata["workspaces"] + for workspace in workspaces: + if ( + workspace.get("orgName") == organization + and workspace.get("workspaceName") == workspace_name + ): + return workspace.get("workspaceId") return None From f646f726167cfd336c391bb88ec3f78e7b6130d7 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Sun, 3 Mar 2024 22:57:36 -0500 Subject: [PATCH 3/5] feat: add flag for emitting version info, refactor parseargs --- README.md | 12 +++++++---- seqerakit/__init__.py | 3 +++ seqerakit/cli.py | 49 ++++++++++++++++++++++++++++--------------- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 6817c6e..4e5dcbe 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ You will need to have an account on Seqera Platform (see [Plans and pricing](htt `seqerakit` requires the following dependencies: -1. [Seqera Platform CLI](https://github.com/seqeralabs/tower-cli#1-installation) +1. [Seqera Platform CLI (`>=0.9.2`)](https://github.com/seqeralabs/tower-cli#1-installation) 2. [Python (`>=3.8`)](https://www.python.org/downloads/) @@ -87,18 +87,22 @@ export TOWER_ACCESS_TOKEN= ## Usage -To confirm the installation of `seqerakit`, configuration of the Seqera Platform CLI and connection is working as expected: +To confirm the installation of `seqerakit`, configuration of the Seqera Platform CLI and connection to the Platform is working as expected. This will run the `tw info` command under the hood: ```bash seqerakit --info ``` -Use the `-h` or `--help `parameter to list the available commands and their associated options: - +Use the `--help` or `-h` parameter to list the available commands and their associated options: ```bash seqerakit --help ``` +Use `--version` or `-v` to retrieve the current version of your seqerakit installation: +```bash +seqerakit --version +``` + ### Dryrun To print the commands that would executed with `tw` when using a YAML file, you can run `seqerakit` with the `--dryrun` flag: diff --git a/seqerakit/__init__.py b/seqerakit/__init__.py index e69de29..fa5f80d 100644 --- a/seqerakit/__init__.py +++ b/seqerakit/__init__.py @@ -0,0 +1,3 @@ +import importlib.metadata as importlib_metadata + +__version__ = importlib_metadata.version(__name__) diff --git a/seqerakit/cli.py b/seqerakit/cli.py index 79605ed..f3e1b80 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -24,48 +24,63 @@ from pathlib import Path from seqerakit import seqeraplatform, helper, overwrite from seqerakit.seqeraplatform import ResourceExistsError, ResourceCreationError - +from seqerakit import __version__ logger = logging.getLogger(__name__) def parse_args(args=None): - parser = argparse.ArgumentParser() - parser.add_argument( + parser = argparse.ArgumentParser( + description="Seqerakit: Python wrapper for the Seqera Platform CLI" + ) + # General options + general = parser.add_argument_group("General Options") + general.add_argument( "-l", "--log_level", default="INFO", choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"), - help="The desired log level (default: INFO).", - type=str.upper, + help="Set the logging level.", ) - parser.add_argument( + general.add_argument( "--info", + "-i", action="store_true", - help="Display information about the Seqera Platform and exit", + help="Display Seqera Platform information and exit.", ) - parser.add_argument( + general.add_argument( "--dryrun", + "-d", action="store_true", - help="Print the commands that would be executed without running them.", + help="Print the commands that would be executed.", ) - parser.add_argument( + general.add_argument( + "--version", + "-v", + action="version", + version=f"%(prog)s {__version__}", + help="Show version number and exit.", + ) + + # YAML processing options + yaml_processing = parser.add_argument_group("YAML Processing Options") + yaml_processing.add_argument( "yaml", type=Path, - nargs="*", # allow multiple YAML paths - help="One or more YAML files with Seqera Platform resources to create", + nargs="*", + help="One or more YAML files with Seqera Platform resource definitions.", ) - parser.add_argument( + yaml_processing.add_argument( "--delete", action="store_true", - help="Recursively delete all resources defined in the YAML file(s)", + help="Recursively delete resources defined in the YAML files.", ) - parser.add_argument( + yaml_processing.add_argument( "--cli", dest="cli_args", type=str, - help="Additional arguments to pass to Seqera Platform" - " CLI enclosed in double quotes (e.g. '--cli=\"--insecure\"')", + help="Additional Seqera Platform CLI specific options to be passed," + " enclosed in double quotes (e.g. '--cli=\"--insecure\"').", ) return parser.parse_args(args) From 349b3141dbb32d262df911247076de82a3e4dd65 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 4 Mar 2024 01:47:23 -0500 Subject: [PATCH 4/5] docs: api endpoint detail in config --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e5dcbe..3e6e3de 100644 --- a/README.md +++ b/README.md @@ -82,9 +82,16 @@ Create a Seqera Platform access token using the [Seqera Platform](https://tower. `seqerakit` reads this token from the environment variable `TOWER_ACCESS_TOKEN`. Please export it into your terminal as shown below: ```bash -export TOWER_ACCESS_TOKEN= +export TOWER_ACCESS_TOKEN= ``` +For Enterprise installations of Seqera Platform, you will also need to configure the API endpoint that will be used to connect to the Platform. You can do so by exporting the following environment variable: +```bash +export TOWER_API_ENDPOINT= +``` +By default, this is set to `https://api.tower.nf` to connect to Seqera Platform Cloud. + + ## Usage To confirm the installation of `seqerakit`, configuration of the Seqera Platform CLI and connection to the Platform is working as expected. This will run the `tw info` command under the hood: From bc32a9b0398092c6eb0387f08dac91760f4527e4 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 4 Mar 2024 01:57:59 -0500 Subject: [PATCH 5/5] build: bump version in setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f5bf293..268e0f1 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -VERSION = "0.4.5" +VERSION = "0.4.6" with open("README.md") as f: readme = f.read()