diff --git a/.github/workflows/teardown.yml b/.github/workflows/teardown.yml index a44b93c..12a8a9a 100644 --- a/.github/workflows/teardown.yml +++ b/.github/workflows/teardown.yml @@ -36,7 +36,7 @@ jobs: environment-file: environment.yml python-version: '3.12' mamba-version: '*' - channels: conda-forge,bioconda,defaults + channels: conda-forge,bioconda activate-environment: seqerakit use-mamba: true diff --git a/README.md b/README.md index ad31f84..866b020 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ You will need to have an account on Seqera Platform (see [Plans and pricing](htt You can install `seqerakit` and its dependencies via Conda. Ensure that you have the correct channels configured: ```console -conda config --add channels defaults conda config --add channels bioconda conda config --add channels conda-forge conda config --set channel_priority strict @@ -170,6 +169,43 @@ seqerakit hello-world-config.yml --cli="-Djavax.net.ssl.trustStore=/absolute/pat Note: Use of `--verbose` option for the `tw` CLI is currently not supported by `seqerakit`. Supplying `--cli="--verbose"` will raise an error. +## Specify targets +When using a YAML file as input that defines multiple resources, you can use the `--targets` flag to specify which resources to create. This flag takes a comma-separated list of resource names. + +For example, given a YAML file that defines the following resources: + +```yaml +workspaces: + - name: 'showcase' + organization: 'seqerakit_automation' +... +compute-envs: + - name: 'compute-env' + type: 'aws-batch forge' + workspace: 'seqerakit/test' +... +pipelines: + - name: "hello-world-test-seqerakit" + url: "https://github.com/nextflow-io/hello" + workspace: 'seqerakit/test' + compute-env: "compute-env" +... +``` + +You can target the creation of `pipelines` only by running: + +```bash +seqerakit test.yml --targets pipelines +``` +This will process only the pipelines block from the YAML file and ignore other blocks such as `workspaces` and `compute-envs`. + +### Multiple Targets +You can also specify multiple resources to create by separating them with commas. For example, to create both workspaces and pipelines, run: + +```bash +seqerakit test.yml --targets workspaces,pipelines +``` + ## YAML Configuration Options There are several options that can be provided in your YAML configuration file, that are handled specially by seqerakit and/or are not exposed as `tw` CLI options. diff --git a/environment.yaml b/environment.yaml new file mode 100644 index 0000000..0de4e09 --- /dev/null +++ b/environment.yaml @@ -0,0 +1,8 @@ +name: seqerakitdev +channels: + - bioconda + - conda-forge +dependencies: + - conda-forge::python=3.10.9 + - conda-forge::pyyaml=6.0 + - bioconda::tower-cli=0.9.2 diff --git a/seqerakit/cli.py b/seqerakit/cli.py index 097f5e8..893066c 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -83,6 +83,13 @@ def parse_args(args=None): help="Additional Seqera Platform CLI specific options to be passed," " enclosed in double quotes (e.g. '--cli=\"--insecure\"').", ) + yaml_processing.add_argument( + "--targets", + dest="targets", + type=str, + help="Specify the resources to be targeted for creation in a YAML file through " + "a comma-separated list (e.g. '--targets=teams,participants').", + ) return parser.parse_args(args) @@ -146,12 +153,20 @@ def handle_block(self, block, args, destroy=False, dryrun=False): def main(args=None): options = parse_args(args if args is not None else sys.argv[1:]) - logging.basicConfig(level=options.log_level) + logging.basicConfig(level=getattr(logging, options.log_level.upper())) + + # Parse CLI arguments into a list + cli_args_list = options.cli_args.split() if options.cli_args else [] + + sp = seqeraplatform.SeqeraPlatform( + cli_args=cli_args_list, dryrun=options.dryrun, json=options.json + ) # If the info flag is set, run 'tw info' if options.info: - sp = seqeraplatform.SeqeraPlatform() - print(sp.info()) + result = sp.info() + if not options.dryrun: + print(result) return if not options.yaml: @@ -164,13 +179,6 @@ def main(args=None): else: options.yaml = [sys.stdin] - # Parse CLI arguments into a list - cli_args_list = options.cli_args.split() if options.cli_args else [] - - sp = seqeraplatform.SeqeraPlatform( - cli_args=cli_args_list, dryrun=options.dryrun, json=options.json - ) - block_manager = BlockParser( sp, [ @@ -188,18 +196,15 @@ def main(args=None): # Parse the YAML file(s) by blocks # and get a dictionary of command line arguments try: - cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete) + cmd_args_dict = helper.parse_all_yaml( + options.yaml, destroy=options.delete, targets=options.targets + ) for block, args_list in cmd_args_dict.items(): for args in args_list: - try: - # Run the 'tw' methods for each block - block_manager.handle_block( - block, args, destroy=options.delete, dryrun=options.dryrun - ) - except (ResourceExistsError, ResourceCreationError) as e: - logging.error(e) - sys.exit(1) - except ValueError as e: + block_manager.handle_block( + block, args, destroy=options.delete, dryrun=options.dryrun + ) + except (ResourceExistsError, ResourceCreationError, ValueError) as e: logging.error(e) sys.exit(1) diff --git a/seqerakit/helper.py b/seqerakit/helper.py index c4792af..86a71a1 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -55,7 +55,7 @@ def parse_yaml_block(yaml_data, block_name): return block_name, cmd_args_list -def parse_all_yaml(file_paths, destroy=False): +def parse_all_yaml(file_paths, destroy=False, targets=None): # If multiple yamls, merge them into one dictionary merged_data = {} @@ -108,6 +108,11 @@ def parse_all_yaml(file_paths, destroy=False): block_names = list(merged_data.keys()) + # Filter blocks based on targets if provided + if targets: + target_blocks = set(targets.split(",")) + block_names = [block for block in block_names if block in target_blocks] + # Define the order in which the resources should be created. resource_order = [ "organizations", diff --git a/seqerakit/overwrite.py b/seqerakit/overwrite.py index 4be03a5..d98d916 100644 --- a/seqerakit/overwrite.py +++ b/seqerakit/overwrite.py @@ -117,13 +117,13 @@ def handle_overwrite(self, block, args, overwrite=False, destroy=False): if self.check_resource_exists(operation["name_key"], sp_args): # if resource exists and overwrite is true, delete if overwrite: - logging.debug( + logging.info( f" The attempted {block} resource already exists." " Overwriting.\n" ) self.delete_resource(block, operation, sp_args) elif destroy: - logging.debug(f" Deleting the {block} resource.") + logging.info(f" Deleting the {block} resource.") self.delete_resource(block, operation, sp_args) else: # return an error if resource exists, overwrite=False raise ResourceExistsError( @@ -147,7 +147,8 @@ def _get_team_args(self, args): if not jsondata: json_method = getattr(self.sp, "-o json") - json_out = json_method("teams", "list", "-o", args["organization"]) + with self.sp.suppress_output(): + json_out = json_method("teams", "list", "-o", args["organization"]) self.block_jsondata["teams"] = json_out else: json_out = jsondata @@ -244,27 +245,30 @@ def _get_json_data(self, block, args, keys_to_get): # Fetch the data if it does not exist if block == "teams": sp_args = self._get_values_from_cmd_args(args[0], keys_to_get) - self.cached_jsondata = json_method( - block, "list", "-o", sp_args["organization"] - ) + with self.sp.suppress_output(): + self.cached_jsondata = json_method( + block, "list", "-o", sp_args["organization"] + ) elif block in Overwrite.generic_deletion or block in { "participants", "labels", }: sp_args = self._get_values_from_cmd_args(args, keys_to_get) - self.cached_jsondata = json_method( - block, "list", "-w", sp_args["workspace"] - ) - elif block == "members" or block == "workspaces": # TODO + with self.sp.suppress_output(): + self.cached_jsondata = json_method( + block, "list", "-w", sp_args["workspace"] + ) + elif block == "members" or block == "workspaces": sp_args = self._get_values_from_cmd_args(args, keys_to_get) - self.cached_jsondata = json_method( - block, "list", "-o", sp_args["organization"] - ) + with self.sp.suppress_output(): + self.cached_jsondata = json_method( + block, "list", "-o", sp_args["organization"] + ) else: sp_args = self._get_values_from_cmd_args(args, keys_to_get) - self.cached_jsondata = json_method(block, "list") + with self.sp.suppress_output(): + self.cached_jsondata = json_method(block, "list") - # Store this data in the block_jsondata dict for later use self.block_jsondata[block] = self.cached_jsondata return self.cached_jsondata, sp_args diff --git a/seqerakit/seqeraplatform.py b/seqerakit/seqeraplatform.py index 49265cd..d55f38a 100644 --- a/seqerakit/seqeraplatform.py +++ b/seqerakit/seqeraplatform.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from contextlib import contextmanager import os import shlex import logging @@ -19,8 +20,6 @@ import re import json -logging.basicConfig(level=logging.DEBUG) - class SeqeraPlatform: """ @@ -44,14 +43,16 @@ def __call__(self, *args, **kwargs): return self.tw_instance._tw_run(command, **kwargs) # Constructs a new SeqeraPlatform instance - def __init__(self, cli_args=None, dryrun=False, json=False): + def __init__(self, cli_args=None, dryrun=False, print_stdout=True, json=False): if cli_args and "--verbose" in cli_args: raise ValueError( "--verbose is not supported as a CLI argument to seqerakit." ) self.cli_args = cli_args or [] self.dryrun = dryrun + self.print_stdout = print_stdout self.json = json + self._suppress_output = False def _construct_command(self, cmd, *args, **kwargs): command = ["tw"] + self.cli_args @@ -68,8 +69,19 @@ def _construct_command(self, cmd, *args, **kwargs): if "params_file" in kwargs: command.append(f"--params-file={kwargs['params_file']}") + # Check for empty string arguments and handle them + self._check_empty_args(command) + return self._check_env_vars(command) + def _check_empty_args(self, command): + for current_arg, next_arg in zip(command, command[1:]): + if isinstance(next_arg, str) and next_arg.strip() == "": + raise ValueError( + f"Empty string argument found for parameter '{current_arg}'. " + "Please provide a valid value or remove the argument." + ) + # Checks environment variables to see that they are set accordingly def _check_env_vars(self, command): full_cmd_parts = [] @@ -89,16 +101,23 @@ def _check_env_vars(self, command): return " ".join(full_cmd_parts) # Executes a 'tw' command in a subprocess and returns the output. - def _execute_command(self, full_cmd): - logging.debug(f" Running command: {full_cmd}") + def _execute_command(self, full_cmd, to_json=False, print_stdout=True): + logging.info(f" Running command: {full_cmd}") process = subprocess.Popen( full_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True ) stdout, _ = process.communicate() stdout = stdout.decode("utf-8").strip() + should_print = ( + print_stdout if print_stdout is not None else self.print_stdout + ) and not self._suppress_output + + if should_print: + logging.info(f" Command output: {stdout}") + if "ERROR: " in stdout or process.returncode != 0: - self._handle_command_errors(str(stdout)) + self._handle_command_errors(stdout) if self.json: out = json.loads(stdout) @@ -109,14 +128,7 @@ def _execute_command(self, full_cmd): return out - def _execute_info_command(self): - # Directly execute 'tw info' command - command = "tw info" - return self._execute_command(command) - def _handle_command_errors(self, stdout): - logging.error(stdout) - # Check for specific tw cli error patterns and raise custom exceptions if re.search( r"ERROR: .*already (exists|a participant)", stdout, flags=re.IGNORECASE @@ -131,19 +143,33 @@ def _handle_command_errors(self, stdout): ) def _tw_run(self, cmd, *args, **kwargs): + print_stdout = kwargs.pop("print_stdout", None) full_cmd = self._construct_command(cmd, *args, **kwargs) if not full_cmd or self.dryrun: - logging.debug(f"DRYRUN: Running command {full_cmd}") + logging.info(f"DRYRUN: Running command {full_cmd}") return None - result = self._execute_command(full_cmd) - return result + return self._execute_command(full_cmd, kwargs.get("to_json"), print_stdout) + + @contextmanager + def suppress_output(self): + original_suppress = self._suppress_output + self._suppress_output = True + try: + yield + finally: + self._suppress_output = original_suppress # Allow any 'tw' subcommand to be called as a method. def __getattr__(self, cmd): if cmd == "info": - return self._execute_info_command - else: - return self.TwCommand(self, cmd.replace("_", "-")) + return lambda *args, **kwargs: self._tw_run( + ["info"], *args, **kwargs, print_stdout=False + ) + if cmd == "-o json": + return lambda *args, **kwargs: self._tw_run( + ["-o", "json"] + list(args), **kwargs + ) + return self.TwCommand(self, cmd.replace("_", "-")) class ResourceExistsError(Exception): diff --git a/setup.py b/setup.py index 0d94a03..ee30640 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -VERSION = "0.4.7" +VERSION = "0.4.9" with open("README.md") as f: readme = f.read() diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 76f5b55..73ce6d5 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -442,3 +442,73 @@ def test_error_duplicate_name_yaml_file(mock_yaml_file): "compute-envs: test_computeenv. Please specify " "a unique value." in str(e.value) ) + + +def test_targets_specified(): + # mock YAML data + yaml_data = """ +organizations: + - name: org1 + description: Organization 1 +workspaces: + - name: workspace1 + organization: org1 + description: Workspace 1 +pipelines: + - name: pipeline1 + workspace: workspace1 + description: Pipeline 1 +""" + with patch("builtins.open", lambda f, _: StringIO(yaml_data)): + result = helper.parse_all_yaml( + ["dummy_path.yaml"], targets="organizations,workspaces" + ) + + expected_organizations_output = [ + { + "cmd_args": ["--name", "org1", "--description", "Organization 1"], + "overwrite": False, + } + ] + expected_workspaces_output = [ + { + "cmd_args": [ + "--name", + "workspace1", + "--organization", + "org1", + "--description", + "Workspace 1", + ], + "overwrite": False, + } + ] + # Check that only 'organizations' and 'workspaces' are in the result + assert "organizations" in result + assert result["organizations"] == expected_organizations_output + assert "workspaces" in result + assert result["workspaces"] == expected_workspaces_output + assert "pipelines" not in result + + +def test_no_targets_specified(): + yaml_data = """ +organizations: + - name: org1 + description: Organization 1 +workspaces: + - name: workspace1 + organization: org1 + description: Workspace 1 +pipelines: + - name: pipeline1 + workspace: workspace1 + description: Pipeline 1 +""" + with patch("builtins.open", lambda f, _: StringIO(yaml_data)): + result = helper.parse_all_yaml(["dummy_path.yaml"]) + + # Check that all blocks are in the result + assert "organizations" in result + assert "workspaces" in result + assert "pipelines" in result diff --git a/tests/unit/test_seqeraplatform.py b/tests/unit/test_seqeraplatform.py index 2033a1b..7558790 100644 --- a/tests/unit/test_seqeraplatform.py +++ b/tests/unit/test_seqeraplatform.py @@ -4,6 +4,8 @@ import json import subprocess import os +import logging +from io import StringIO class TestSeqeraPlatform(unittest.TestCase): @@ -91,6 +93,22 @@ def test_resource_creation_error(self): with self.assertRaises(seqeraplatform.ResourceCreationError): command("import", "my_pipeline.json", "--name", "pipeline_name") + def test_empty_string_argument(self): + command = ["--profile", " ", "--config", "my_config"] + with self.assertRaises(ValueError) as context: + self.sp._check_empty_args(command) + self.assertIn( + "Empty string argument found for parameter '--profile'", + str(context.exception), + ) + + def test_no_empty_string_argument(self): + command = ["--profile", "test_profile", "--config", "my_config"] + try: + self.sp._check_empty_args(command) + except ValueError: + self.fail("_check_empty_args() raised ValueError unexpectedly!") + def test_json_parsing(self): with patch("subprocess.Popen") as mock_subprocess: # Mock the stdout of the Popen process to return JSON @@ -170,6 +188,32 @@ def test_cli_args_exclusion_of_verbose(self): # TODO: remove this test once fix "--verbose is not supported as a CLI argument to seqerakit.", ) + @patch("subprocess.Popen") + def test_info_command_construction(self, mock_subprocess): + # Mock the subprocess call to prevent actual execution + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = (b"", b"") + + self.sp.info() + called_command = mock_subprocess.call_args[0][0] + + # Check if the constructed command is correct + expected_command_part = "tw --url http://tower-api.com --insecure info" + self.assertIn(expected_command_part, called_command) + + # Check if the cli_args are included in the called command + for arg in self.cli_args: + self.assertIn(arg, called_command) + + @patch("subprocess.Popen") + def test_info_command_dryrun(self, mock_subprocess): + # Initialize SeqeraPlatform with dryrun enabled + self.sp.dryrun = True + self.sp.info() + + # Check that subprocess.Popen is not called + mock_subprocess.assert_not_called() + class TestKitOptions(unittest.TestCase): def setUp(self): @@ -226,5 +270,107 @@ def test_error_raised_for_unset_env_vars(self): ) +class TestSeqeraPlatformOutputHandling(unittest.TestCase): + def setUp(self): + self.sp = seqeraplatform.SeqeraPlatform() + # Set up logging to capture output + self.log_capture = StringIO() + self.log_handler = logging.StreamHandler(self.log_capture) + logging.getLogger().addHandler(self.log_handler) + logging.getLogger().setLevel(logging.INFO) + + def tearDown(self): + logging.getLogger().removeHandler(self.log_handler) + logging.getLogger().setLevel(logging.NOTSET) + + @patch("subprocess.Popen") + def test_suppress_output(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = ( + b'{"key": "value"}', + b"", + ) + + log_capture = StringIO() + logging.getLogger().addHandler(logging.StreamHandler(log_capture)) + + with self.sp.suppress_output(): + self.sp.pipelines("list") + + log_contents = log_capture.getvalue() + self.assertIn("Running command:", log_contents) + self.assertNotIn("Command output:", log_contents) + + @patch("subprocess.Popen") + def test_suppress_output_context(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = ( + b'{"key": "value"}', + b"", + ) + + # Test that stdout is suppressed within the context manager + with self.sp.suppress_output(): + result = self.sp._execute_command("tw pipelines list", to_json=True) + self.assertEqual(result, {"key": "value"}) + + # Test that stdout is not suppressed outside the context manager + result = self.sp._execute_command("tw pipelines list", to_json=True) + self.assertEqual(result, {"key": "value"}) + + @patch("subprocess.Popen") + def test_json_output_handling(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = ( + b'{"key": "value"}', + b"", + ) + + result = self.sp._execute_command("tw pipelines list", to_json=True) + self.assertEqual(result, {"key": "value"}) + + result = self.sp._execute_command("tw pipelines list", to_json=False) + self.assertEqual(result, '{"key": "value"}') + + @patch("subprocess.Popen") + def test_print_stdout_override(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = (b"output", b"") + + # Test with print_stdout=True + self.sp._execute_command("tw pipelines list", print_stdout=True) + log_output = self.log_capture.getvalue() + self.assertIn("Command output: output", log_output) + + # Clear the log capture + self.log_capture.truncate(0) + self.log_capture.seek(0) + + # Test with print_stdout=False + self.sp._execute_command("tw pipelines list", print_stdout=False) + log_output = self.log_capture.getvalue() + self.assertNotIn("Command output: output", log_output) + + @patch("subprocess.Popen") + def test_error_handling_with_suppressed_output(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=1) + mock_subprocess.return_value.communicate.return_value = ( + b"ERROR: Something went wrong", + b"", + ) + + with self.assertRaises(seqeraplatform.ResourceCreationError): + with self.sp.suppress_output(): + self.sp._execute_command("tw pipelines list") + + @patch("subprocess.Popen") + def test_json_parsing_error(self, mock_subprocess): + mock_subprocess.return_value = MagicMock(returncode=0) + mock_subprocess.return_value.communicate.return_value = (b"Invalid JSON", b"") + + with self.assertRaises(json.JSONDecodeError): + self.sp._execute_command("tw pipelines list", to_json=True) + + if __name__ == "__main__": unittest.main()