diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..5a44a45 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,46 @@ +# This workflow will install Python dependencies and run tests +name: Python application + +on: + push: + branches: [master, devel] + pull_request: + branches: [master, devel] + +jobs: + + test-packaging: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install Poetry + run: | + pipx install poetry + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + architecture: 'x64' + + - name: Run poetry install + run: | + poetry env use '3.11' + poetry install + + - name: Run pytest + run: | + poetry run pytest + + - name: Run import checker + run: | + poetry run isort --check . + + - name: Run code style checker + run: | + poetry run pflake8 + + - name: Run code formatter + run: | + poetry run black --check src tests diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b77136a --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +*~ +__pycache__ +*.egg-info +.vscode +.eggs +build +.pytest_cache +.vscode diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..88746e0 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Change Log for npg_porch Project + +The format is based on [Keep a Changelog](http://keepachangelog.com/). +This project adheres to [Semantic Versioning](http://semver.org/). + +## [Unreleased] + +## [0.1.0] - 2024-07-23 + +### Added + +# Initial project scaffold, code and tests diff --git a/README.md b/README.md index 370fb0d..97e6c2c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,67 @@ # npg_porch_cli -A command-line client to enable communication with the npg_porch API. + +A command-line client and a simple client library to enable communication +with the [npg_porch](https://github.com/wtsi-npg/npg_porch) JSON API. + +Provides a Python script, `npg_porch_client`, and a Python client API. + +NPG_PORCH_TOKEN environment variable should be set to the value of either +the admin or project-specific token. The token should be pre-registered in +the database that is used by the `porch` API server. + +The project can be deployed with pip or poetry in a standard way. + +Example of using a client API: + +``` python + from npg_porch_cli.api import PorchRequest + + pr = PorchRequest(porch_url="https://myporch.com") + response = pr.send(action="list_pipelines") + + pr = PorchRequest( + porch_url="https://myporch.com", + pipeline_name="Snakemake_Cardinal", + pipeline_url="https://github.com/wtsi-npg/snakemake_cardinal", + pipeline_version="1.0", + ) + response = pr.send( + action="update_task", + task_status="FAILED", + task_input={"id_run": 409, "sample": "Valxxxx", "id_study": "65"}, + ) +``` + +By default the client validates the certificate of the server's certification +authority (CA). If the server's certificate is signed by a custom CA, set the +`SSL_CERT_FILE` environment variable to the path of the CA's certificate. +Python versions starting from 3.11 seem to have increased security precautions +when validating certificates of custom CAs. It might be necessary to set the +`REQUESTS_CA_BUNDLE` environmental variable, see details +[here](https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification). + +``` bash + export NPG_PORCH_TOKEN='my_token' + export SSL_CERT_FILE=/path_to/my.pem + npg_porch_client list_pipelines --base_url https://myporch.com +``` + +It is possible, but not recommended, to disable this validation check. + +``` bash + export NPG_PORCH_TOKEN='my_token' + npg_porch_client list_pipelines --base_url https://myporch.com --no-validate_ca_cert +``` + +A valid JSON string is required for the `--task_json` script's argument, note +double quotes in the example below. + +``` bash + export NPG_PORCH_TOKEN='my_token' + npg_porch_client update_task --base_url https://myporch.com \ + --pipeline Snakemake_Cardinal \ + --pipeline_url 'https://github.com/wtsi-npg/snakemake_cardinal' \ + --pipeline_version 1.0 \ + --task_json '{"id_run": 409, "sample": "Valxxxx", "id_study": "65"}' \ + --status FAILED +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..049982b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,55 @@ +[tool.poetry] +name = "npg_porch_cli" +version = "0.1.0" +authors = [ + "Marina Gourtovaia", + "Kieron Taylor", + "Jennifer Liddle", +] +description = "CLI client for communicating with npg_porch JSON API" +readme = "README.md" +license = "GPL-3.0-or-later" + +[tool.poetry.scripts] +npg_porch_client = "npg_porch_cli.api_cli_user:run" + +[tool.poetry.dependencies] +python = "^3.10" +requests = "^2.31.0" + +[tool.poetry.dev-dependencies] +black = "^22.3.0" +pyproject-flake8 = "^7.0.0" +flake8-bugbear = "^24.4.0" +pytest = "^7.1.1" +isort = { version = "^5.10.1", extras = ["colors"] } + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.isort] +profile = "black" + +[tool.black] +line_length = 88 + +[tool.flake8] +max-line-length = 88 +extend-select = ["B950"] +extend-ignore = ["E501"] +exclude = [ + # No need to traverse our git directory + ".git", + # There's no value in checking cache directories + "__pycache__" +] +per-file-ignores = """ + # Disable 'imported but unused' + __init__.py: F401 + """ + +[tool.pytest.ini_options] +addopts = [ + "--import-mode=importlib", +] diff --git a/src/npg_porch_cli/__init__.py b/src/npg_porch_cli/__init__.py new file mode 100644 index 0000000..72d9c44 --- /dev/null +++ b/src/npg_porch_cli/__init__.py @@ -0,0 +1 @@ +from .api import send_request diff --git a/src/npg_porch_cli/api.py b/src/npg_porch_cli/api.py new file mode 100644 index 0000000..11f6dba --- /dev/null +++ b/src/npg_porch_cli/api.py @@ -0,0 +1,431 @@ +# Copyright (c) 2024 Genome Research Ltd. +# +# Authors: Marina Gourtovaia +# Jennifer Liddle +# +# This file is part of npg_porch_cli project. +# +# npg_porch_cli is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program. If not, see . + +import json +import os +from dataclasses import InitVar, asdict, dataclass, field +from urllib.parse import urljoin + +import requests + +PORCH_OPENAPI_SCHEMA_URL = "api/v1/openapi.json" +PORCH_TASK_STATUS_ENUM_NAME = "TaskStateEnum" + +INITIAL_PORCH_STATUS = "PENDING" +PORCH_STATUSES = [ + INITIAL_PORCH_STATUS, + "CLAIMED", + "RUNNING", + "DONE", + "FAILED", + "CANCELLED", +] + +CLIENT_TIMEOUT = (10, 60) + +NPG_PORCH_TOKEN_ENV_VAR = "NPG_PORCH_TOKEN" + + +class AuthException(Exception): + pass + + +class ServerErrorException(Exception): + pass + + +@dataclass(kw_only=True) +class Pipeline: + name: str + uri: str + version: str + + def __post_init__(self): + "Post-constructor hook. Ensures all fields are defined." + if not (self.name and self.uri and self.version): + raise TypeError("Pipeline name, uri and version should be defined") + + +@dataclass(kw_only=True) +class PorchAction: + porch_url: str + action: str + validate_ca_cert: bool = field(default=True) + task_json: InitVar[str | None] = field(default=None, repr=False) + task_input: dict = field(default=None) + task_status: str | None = field(default=None) + + def __post_init__(self, task_json): + "Post-constructor hook. Ensures integrity and validity of attributes." + + if self.porch_url is None: + raise TypeError("'porch_url' attribute cannot be None") + + if task_json is not None: + if self.task_input is not None: + raise ValueError("task_json and task_input cannot be both set") + self.task_input = json.loads(task_json) + + self._validate_action_name() + self.task_status = self._validate_status() + + def _validate_action_name(self): + if self.action is None: + raise TypeError("'action' attribute cannot be None") + if self.action not in _PORCH_CLIENT_ACTIONS: + raise ValueError( + f"Action '{self.action}' is not valid. " + "Valid actions: " + ", ".join(list_client_actions()) + ) + + def _validate_status(self) -> str | None: + """ + Retrieves OpenAPI schema for the porch server and validates the given + task status value against the values listed in the schema document. + + Returns a validated task status value. The case of this string can be + different from the input string. + """ + + if self.task_status is None: + return None + + url = urljoin(self.porch_url, PORCH_OPENAPI_SCHEMA_URL) + response = requests.request("GET", url, verify=self.validate_ca_cert) + if not response.ok: + raise ServerErrorException( + f"Failed to get OpenAPI Schema. " + f'Status code {response.status_code} "{response.reason}" ' + f"received from {response.url}" + ) + + status = self.task_status.upper() + valid_statuses = [] + error_message = f"Failed to get enumeration of valid statuses from {url}" + try: + valid_statuses = response.json()["components"]["schemas"][ + PORCH_TASK_STATUS_ENUM_NAME + ]["enum"] + except Exception as e: + raise Exception(f"{error_message}: " + e.__str__()) + + if len(valid_statuses) == 0: + raise Exception(error_message) + + if status not in valid_statuses: + raise ValueError( + f"Task status '{self.task_status}' is not valid. " + "Valid statuses: " + ", ".join(sorted(valid_statuses)) + ) + + return status + + +def get_token() -> str: + """Gets the value of the porch token from the environment variable. + + If the NPG_PORCH_TOKEN is not defined or assigned to am empty string, + raises AuthException. + + Returns: + The token. + """ + if NPG_PORCH_TOKEN_ENV_VAR not in os.environ: + raise AuthException("Authorization token is needed") + token = os.environ[NPG_PORCH_TOKEN_ENV_VAR] + if token == "": + raise AuthException("Authorization token is needed") + + return token + + +def list_client_actions() -> list[str]: + """Returns a sorted list of currently implemented client actions.""" + + return sorted(_PORCH_CLIENT_ACTIONS.keys()) + + +def send(action: PorchAction, pipeline: Pipeline = None) -> dict | list: + """Sends a request to the porch API server. + + Sends a request to the porch API server to perform an action defined + by the `action` attribute of the `action` argument. The context of the + query is defined by the pipeline argument. + + Args: + action: + npg_porch_cli.api.PorchAction object + pipeline: + npg_porch_cli.api.Pipeline object + + Returns: + The server's response is returned as a Python data structure. + """ + + # Get function's definition and then call the function. + function = _PORCH_CLIENT_ACTIONS[action.action] + if action.action == "list_pipelines": + return function(action=action) + return function(action=action, pipeline=pipeline) + + +def list_pipelines(action: PorchAction) -> list: + """Lists all pipelines registered with the porch server. + + Args: + action: + npg_porch_cli.api.PorchAction object + + Returns: + A list of dictionaries representing npg_porch_cli.api.Pipeline objects + + """ + + return send_request( + validate_ca_cert=action.validate_ca_cert, + url=urljoin(action.porch_url, "pipelines"), + method="GET", + ) + + +def list_tasks(action: PorchAction, pipeline: Pipeline = None) -> list: + """Lists tasks. + + Args: + action: + npg_porch_cli.api.PorchAction object + pipeline: + npg_porch_cli.api.Pipeline object, optional + + Returns: + A list of Python objects, most likely dictionaries, representing registered + tasks. + + If the pipeline argument is defined, only tasks belonging to this pipeline + are listed. Otherwise the list contains all tasks registered with the + porch server. + """ + + response_obj = send_request( + validate_ca_cert=action.validate_ca_cert, + url=urljoin(action.porch_url, "tasks"), + method="GET", + ) + if pipeline is not None: + pipeline_dict = asdict(pipeline) + response_obj = [o for o in response_obj if o["pipeline"] == pipeline_dict] + return response_obj + + +def add_pipeline(action: PorchAction, pipeline: Pipeline) -> dict: + """Registers a new pipeline with the porch server. + + Args: + action: + npg_porch_cli.api.PorchAction object + pipeline: + npg_porch_cli.api.Pipeline object + + Returns: + A dictionary representing npg_porch_cli.api.Pipeline object + """ + + return send_request( + validate_ca_cert=action.validate_ca_cert, + method="POST", + url=urljoin(action.porch_url, "pipelines"), + data=asdict(pipeline), + ) + + +def add_task(action: PorchAction, pipeline: Pipeline): + """Registers a new task with the porch server. + + Args: + action: + npg_porch_cli.api.PorchAction object + pipeline: + npg_porch_cli.api.Pipeline object + + Returns: + A dictionary representing the new task. The status of the new task is + 'PENDING'. + """ + + if action.task_input is None: + raise TypeError(f"task_input cannot be None for action '{action.action}'") + return send_request( + validate_ca_cert=action.validate_ca_cert, + url=urljoin(action.porch_url, "tasks"), + method="POST", + data={ + "pipeline": asdict(pipeline), + "task_input": action.task_input, + "status": INITIAL_PORCH_STATUS, + }, + ) + + +def claim_task(action: PorchAction, pipeline: Pipeline): + """Claims a task that belongs to the pipeline. + + Args: + action: + npg_porch_cli.api.PorchAction object + pipeline: + npg_porch_cli.api.Pipeline object + + Returns: + A dictionary representing the claimed task. + """ + + return send_request( + validate_ca_cert=action.validate_ca_cert, + url=urljoin(action.porch_url, "tasks/claim"), + method="POST", + data=asdict(pipeline), + ) + + +def update_task(action: PorchAction, pipeline: Pipeline): + """Updates the status of an existing task. + + Args: + action: + npg_porch_cli.api.PorchAction object + pipeline: + npg_porch_cli.api.Pipeline object + + Returns: + A dictionary representing the updated task. + """ + + if action.task_input is None: + raise TypeError(f"task_input cannot be None for action '{action.action}'") + if action.task_status is None: + raise TypeError(f"task_status cannot be None for action '{action.action}'") + return send_request( + validate_ca_cert=action.validate_ca_cert, + url=urljoin(action.porch_url, "tasks/"), + method="PUT", + data={ + "pipeline": asdict(pipeline), + "task_input": action.task_input, + "status": action.task_status, + }, + ) + + +_PORCH_CLIENT_ACTIONS = { + "list_tasks": list_tasks, + "list_pipelines": list_pipelines, + "add_pipeline": add_pipeline, + "add_task": add_task, + "claim_task": claim_task, + "update_task": update_task, +} + + +def send_request( + validate_ca_cert: bool, + url: str, + method: str, + data: dict | None = None, + auth_type: str | None = "token", +): + """Sends an HTTP request to a JSON API web service. + + Raises ServerErrorException if the status code of the response is not + in the 200 – 299 range. + + Args: + validate_ca_cert: + A boolean flag defining whether the server CA certificate + will be validated. If set to True, SSL_CERT_FILE environment + variable should be set. + url: + A URL to send the request to. + method: + The HTTP method to use (GET, POST, etc.) + data: + Optional payload for the request as a Python object. + auth_type: + Authorization type, defaults to 'token'. If no authorization + is required, set the value explicitly to None. Only the token + type authorization is implemented at the moment. For this type + of authorization to work, set NPG_PORCH_TOKEN environment + variable. + + Example: + + from npg_porch_cli import send_request + + url = "https://some.com/api/fruit_types" + fruit_types = send_request(validate_ca_cert=True, url=url, method="GET", auth_type=None) + + url = "https://some.com/api/add_fruit_type" + new_type = send_request( + validate_ca_cert=True, + url=url, + method="PUT", + data={"banana": {"taste": "sweet"}}, + ) + + Returns: + Server's decoded reply. + """ + + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + } + if auth_type is not None: + if auth_type == "token": + headers["Authorization"] = "Bearer " + get_token() + else: + raise ValueError(f"Authorization type {auth_type} is not implemented") + + request_args = { + "headers": headers, + "timeout": CLIENT_TIMEOUT, + "verify": validate_ca_cert, + } + if data is not None: + request_args["json"] = data + + response = requests.request(method, url, **request_args) + if not response.ok: + detail = "" + try: + data = response.json() + if "detail" in data: + detail = data["detail"] + except Exception: + pass + + message = ( + f'Status code {response.status_code} "{response.reason}" ' + f"received from {response.url}" + ) + if detail: + message += f".\nDetail: {detail}" + raise ServerErrorException(message) + + return response.json() diff --git a/src/npg_porch_cli/api_cli_user.py b/src/npg_porch_cli/api_cli_user.py new file mode 100755 index 0000000..cd94ddc --- /dev/null +++ b/src/npg_porch_cli/api_cli_user.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2022, 2024 Genome Research Ltd. +# +# Author: Jennifer Liddle +# +# This file is part of npg_porch_cli project. +# +# npg_porch_cli is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program. If not, see . + +import argparse +import json + +from npg_porch_cli.api import Pipeline, PorchAction, list_client_actions, send + + +def run(): + """ + Parses command line arguments, send the request to porch server API, + prints out the server's response to STDOUT. + + The text below assumes that this code is deployed as a script called + `npg_porch_client`. + + One positional argument, the name of the action to be performed, is required. + One named argument `--base_url`, teh URL of the `porch` server, is also + required. + + npg_porch_client list_pipeline --base_url SOME_URL + + A full list of actions: + list_tasks + list_pipelines + add_pipeline + add_task + claim_task + update_task + + Though most of named arguments are optional, some actions require + certain combinations of arguments to be defined. + + All list actions do not require any optional arguments defined. If + `--pipeline_name` is defined, `list_tasks` returns a list of tasks for + this pipeline, otherwise all registered tasks are returned. + + All non-list actions require all `--pipeline`, `pipeline_url` and + `--pipeline_version` defined. + + The `add_task` and `update_task` actions require the `--task_json` to + be defined. In addition to this, for the `update_task` action `--status` + should be defined. + + NPG_PORCH_TOKEN environment variable should be set to the value of + either an admin or project-specific token. + + """ + parser = argparse.ArgumentParser( + prog="npg_porch_client", + description="npg_porch (Pipeline Orchestration)API Client", + epilog="The server JSON reply is printed to STDOUT", + ) + + parser.add_argument( + "action", + type=str, + help="Action to send to npg_porch server API", + choices=list_client_actions(), + ) + parser.add_argument("--base_url", type=str, required=True, help="Base URL") + parser.add_argument( + "--validate_ca_cert", + action=argparse.BooleanOptionalAction, + type=bool, + help="A flag instructing to validate server's CA SSL certificate, true by default", + default=True, + ) + parser.add_argument( + "--pipeline_url", type=str, help="Pipeline git project URL, optional" + ) + parser.add_argument( + "--pipeline_version", type=str, help="Pipeline version, optional" + ) + parser.add_argument("--pipeline", type=str, help="Pipeline name, optional") + parser.add_argument("--task_json", type=str, help="Task as JSON, optional") + parser.add_argument("--status", type=str, help="New status to set, optional") + + args = parser.parse_args() + + action = PorchAction( + porch_url=args.base_url, + validate_ca_cert=args.validate_ca_cert, + action=args.action, + task_json=args.task_json, + task_status=args.status, + ) + pipeline = None + if args.pipeline is not None: + pipeline = Pipeline( + name=args.pipeline, uri=args.pipeline_url, version=args.pipeline_version + ) + + print(json.dumps(send(action=action, pipeline=pipeline), indent=2)) diff --git a/tests/data/porch_openapi.json b/tests/data/porch_openapi.json new file mode 100644 index 0000000..ae11577 --- /dev/null +++ b/tests/data/porch_openapi.json @@ -0,0 +1 @@ +{"openapi":"3.1.0","info":{"title":"Pipeline Orchestration (POrch)","version":"0.1.0"},"paths":{"/pipelines/":{"get":{"tags":["pipelines"],"summary":"Get information about all pipelines.","description":"Returns a list of pydantic Pipeline models.\n A uri and/or version filter can be used.\n A valid token issued for any pipeline is required for authorisation.","operationId":"get_pipelines_pipelines__get","security":[{"HTTPBearer":[]}],"parameters":[{"name":"uri","in":"query","required":false,"schema":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Uri"}},{"name":"version","in":"query","required":false,"schema":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Version"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/Pipeline"},"title":"Response Get Pipelines Pipelines Get"}}}},"403":{"description":"Not authorised"},"500":{"description":"Unexpected error"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"post":{"tags":["pipelines"],"summary":"Create one pipeline record.","description":"Using JSON data in the request, creates a new pipeline record.\n A valid special power user token is required for authorisation.","operationId":"create_pipeline_pipelines__post","security":[{"HTTPBearer":[]}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Pipeline"}}}},"responses":{"201":{"description":"Pipeline was created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Pipeline"}}}},"403":{"description":"Not authorised"},"500":{"description":"Unexpected error"},"400":{"description":"Insufficient pipeline properties provided"},"409":{"description":"Pipeline already exists"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/pipelines/{pipeline_name}":{"get":{"tags":["pipelines"],"summary":"Get information about one pipeline.","description":"Returns a single pydantic Pipeline model if found.\n A valid token issued for any pipeline is required for authorisation.","operationId":"get_pipeline_pipelines__pipeline_name__get","security":[{"HTTPBearer":[]}],"parameters":[{"name":"pipeline_name","in":"path","required":true,"schema":{"type":"string","title":"Pipeline Name"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Pipeline"}}}},"403":{"description":"Not authorised"},"500":{"description":"Unexpected error"},"404":{"description":"Not found"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/tasks/":{"get":{"tags":["tasks"],"summary":"Returns all tasks, and can be filtered to task status or pipeline name","description":"Return all tasks. The list of tasks can be filtered by supplying a pipeline\n name and/or task status","operationId":"get_tasks_tasks__get","security":[{"HTTPBearer":[]}],"parameters":[{"name":"pipeline_name","in":"query","required":false,"schema":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Pipeline Name"}},{"name":"status","in":"query","required":false,"schema":{"anyOf":[{"$ref":"#/components/schemas/TaskStateEnum"},{"type":"null"}],"title":"Status"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/Task"},"title":"Response Get Tasks Tasks Get"}}}},"403":{"description":"Not authorised"},"500":{"description":"Unexpected error"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"post":{"tags":["tasks"],"summary":"Creates one task record.","description":"Given a Task object, creates a database record for it and returns\n the same object, the response HTTP status is 201 'Created'. The\n new task is assigned pending status, ie becomes available for claiming.\n\n The pipeline specified by the `pipeline` attribute of the Task object\n should exist. If it does not exist, return status 404 'Not found'.","operationId":"create_task_tasks__post","security":[{"HTTPBearer":[]}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"}}}},"responses":{"201":{"description":"Task creation was successful","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"}}}},"403":{"description":"Not authorised"},"500":{"description":"Unexpected error"},"409":{"description":"A task with the same signature already exists"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"put":{"tags":["tasks"],"summary":"Update one task.","description":"Given a Task object, updates the status of the task in the database\n to the value of the status in this Task object.\n\n If the task does not exist, status 404 'Not found' is returned.","operationId":"update_task_tasks__put","security":[{"HTTPBearer":[]}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"}}}},"responses":{"200":{"description":"Task was modified","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"}}}},"403":{"description":"Not authorised"},"500":{"description":"Unexpected error"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/tasks/claim":{"post":{"tags":["tasks"],"summary":"Claim tasks for a particular pipeline.","description":"Arguments - the Pipeline object and the maximum number of tasks\n to retrieve and claim, the latter defaults to 1 if not given.\n\n If no tasks that satisfy the given criteria and are unclaimed\n are found, returns status 200 and an empty array.\n\n If any tasks are claimed, return an array of these Task objects\n and status 200.\n\n The pipeline object returned within each of the tasks is consistent\n with the pipeline object in the payload, but has all possible\n attributes defined (uri, version).","operationId":"claim_task_tasks_claim_post","security":[{"HTTPBearer":[]}],"parameters":[{"name":"num_tasks","in":"query","required":false,"schema":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"null"}],"default":1,"title":"Num Tasks"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Pipeline"}}}},"responses":{"200":{"description":"Receive a list of tasks that have been claimed","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/Task"},"title":"Response Claim Task Tasks Claim Post"}}}},"403":{"description":"Not authorised"},"500":{"description":"Unexpected error"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/":{"get":{"tags":["index"],"summary":"Web page with links to OpenAPI documentation.","operationId":"root__get","responses":{"200":{"description":"Successful Response","content":{"text/html":{"schema":{"type":"string"}}}}}}}},"components":{"schemas":{"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"Pipeline":{"properties":{"name":{"type":"string","title":"Pipeline Name","description":"A user-controlled name for the pipeline"},"uri":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"URI","description":"URI to bootstrap the pipeline code"},"version":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Version","description":"Pipeline version to use with URI"}},"type":"object","title":"Pipeline"},"Task":{"properties":{"pipeline":{"$ref":"#/components/schemas/Pipeline"},"task_input_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Task Input ID","description":"A stringified unique identifier for a piece of work. Set by the npg_porch server, not the client"},"task_input":{"type":"object","title":"Task Input","description":"A structured parameter set that uniquely identifies a piece of work, and enables an iteration of a pipeline"},"status":{"anyOf":[{"$ref":"#/components/schemas/TaskStateEnum"},{"type":"null"}]}},"type":"object","required":["pipeline"],"title":"Task"},"TaskStateEnum":{"type":"string","enum":["PENDING","CLAIMED","RUNNING","DONE","FAILED","CANCELLED"],"title":"TaskStateEnum"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"}},"securitySchemes":{"HTTPBearer":{"type":"http","scheme":"bearer"}}},"tags":[{"name":"pipelines","description":"Manage pipelines."},{"name":"index","description":"Links to documentation."}]} \ No newline at end of file diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..685b47b --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,249 @@ +import json + +import pytest +import requests + +from npg_porch_cli.api import ( + AuthException, + Pipeline, + PorchAction, + ServerErrorException, + get_token, + list_client_actions, + send, +) + +url = "http://some.com" +var_name = "NPG_PORCH_TOKEN" + + +class MockPorchResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + self.reason = "Some reason" + self.url = url + self.ok = True if self.status_code == 200 else False + + def json(self): + return self.json_data + + +def test_retrieving_token(monkeypatch): + monkeypatch.delenv(var_name, raising=False) + with pytest.raises(AuthException) as e: + get_token() + assert e.value.args[0] == "Authorization token is needed" + + monkeypatch.setenv(var_name, "") + with pytest.raises(AuthException) as e: + get_token() + assert e.value.args[0] == "Authorization token is needed" + + monkeypatch.setenv(var_name, "token_xyz") + assert get_token() == "token_xyz" + + monkeypatch.undo() + + +def test_listing_actions(): + assert list_client_actions() == [ + "add_pipeline", + "add_task", + "claim_task", + "list_pipelines", + "list_tasks", + "update_task", + ] + + +def test_pipeline_class(): + with pytest.raises(TypeError) as e: + Pipeline(name=None, uri="http://some.come", version="1.0") + assert e.value.args[0] == "Pipeline name, uri and version should be defined" + + +def test_porch_action_class(monkeypatch): + with pytest.raises(TypeError) as e: + PorchAction(porch_url=None, action="list_tasks") + assert e.value.args[0] == "'porch_url' attribute cannot be None" + + with pytest.raises(TypeError) as e: + PorchAction(porch_url="http://some.come", action=None) + assert e.value.args[0] == "'action' attribute cannot be None" + + with pytest.raises(ValueError) as e: + PorchAction(porch_url=url, action="list_tools") + assert ( + e.value.args[0] == "Action 'list_tools' is not valid. " + "Valid actions: add_pipeline, add_task, claim_task, list_pipelines, " + "list_tasks, update_task" + ) + + pa = PorchAction(porch_url=url, action="list_tasks") + assert pa.validate_ca_cert is True + assert pa.task_input is None + assert pa.task_status is None + + with pytest.raises(ValueError) as e: + pa = PorchAction( + porch_url=url, + action="add_task", + task_json='{"id_run": 5}', + task_input={"id_run": 5}, + ) + assert e.value.args[0] == "task_json and task_input cannot be both set" + pa = PorchAction( + validate_ca_cert=False, + porch_url=url, + action="add_task", + task_json='{"id_run": 5}', + ) + assert pa.task_input == {"id_run": 5} + assert pa.validate_ca_cert is False + pa = PorchAction(porch_url=url, action="add_task", task_input={"id_run": 5}) + assert pa.task_input == {"id_run": 5} + + with monkeypatch.context() as m: + + def mock_get_200(*args, **kwargs): + f = open("tests/data/porch_openapi.json") + r = MockPorchResponse(json.load(f), 200) + f.close() + return r + + m.setattr(requests, "request", mock_get_200) + pa = PorchAction( + task_status="FAILED", + action="update_task", + task_input='{"id_run": 5}', + porch_url=url, + ) + assert pa.task_status == "FAILED" + pa = PorchAction( + task_status="FAILED", + action="update_task", + task_input='{"id_run": 5}', + porch_url=url, + ) + assert pa.task_status == "FAILED" + with pytest.raises(ValueError) as e: + PorchAction( + task_status="Swimming", + action="update_task", + task_input='{"id_run": 5}', + porch_url=url, + ) + assert ( + e.value.args[0] == "Task status 'Swimming' is not valid. " + "Valid statuses: CANCELLED, CLAIMED, DONE, FAILED, PENDING, RUNNING" + ) + + with monkeypatch.context() as mk: + + def mock_get_404(*args, **kwargs): + return MockPorchResponse({"Error": "Not found"}, 404) + + mk.setattr(requests, "request", mock_get_404) + with pytest.raises(ServerErrorException) as e: + PorchAction( + task_status="FAILED", + action="update_task", + task_input='{"id_run": 5}', + porch_url=url, + ) + assert ( + e.value.args[0] == "Failed to get OpenAPI Schema. Status code 404 " + '"Some reason" received from http://some.com' + ) + + with monkeypatch.context() as mkp: + + def mock_get_200(*args, **kwargs): + return MockPorchResponse( + {"openapi": "3.1.0", "info": {"title": "Pipeline", "version": "0.1.0"}}, + 200, + ) + + mkp.setattr(requests, "request", mock_get_200) + with pytest.raises(Exception) as e: + PorchAction( + task_status="FAILED", + action="update_task", + task_input='{"id_run": 5}', + porch_url=url, + ) + assert e.value.args[0].startswith( + f"Failed to get enumeration of valid statuses from {url}" + ) + + +def test_request_validation(): + p = Pipeline(uri=url, version="1.0", name="p1") + + pa = PorchAction(porch_url=url, action="add_task") + with pytest.raises(TypeError) as e: + send(action=pa, pipeline=p) + assert e.value.args[0] == "task_input cannot be None for action 'add_task'" + + pa = PorchAction(porch_url=url, action="update_task") + with pytest.raises(TypeError) as e: + send(action=pa, pipeline=p) + assert e.value.args[0] == "task_input cannot be None for action 'update_task'" + pa = PorchAction(porch_url=url, action="update_task", task_input={"id_run": 5}) + with pytest.raises(TypeError) as e: + send(action=pa, pipeline=p) + assert e.value.args[0] == "task_status cannot be None for action 'update_task'" + + +def test_sending_request(monkeypatch): + monkeypatch.delenv(var_name, raising=False) + monkeypatch.setenv(var_name, "MY_TOKEN") + + def all_valid(*args, **kwargs): + return "PENDING" + + monkeypatch.setattr(PorchAction, "_validate_status", all_valid) + + p = Pipeline(uri=url, version="0.1", name="p1") + task = {"id_run": 5} + response_data = { + "pipeline": p, + "task_input_id": "8d505b17b4f", + "task_input": task, + "status": "PENDING", + } + + with monkeypatch.context() as m: + + def mock_get_200(*args, **kwargs): + return MockPorchResponse(response_data, 200) + + m.setattr(requests, "request", mock_get_200) + + pa = PorchAction(porch_url=url, action="add_task", task_input=task) + assert send(action=pa, pipeline=p) == response_data + + with monkeypatch.context() as mk: + response_data["status"] = "CLAIMED" + + def mock_get_200(*args, **kwargs): + return MockPorchResponse(response_data, 200) + + mk.setattr(requests, "request", mock_get_200) + + pa = PorchAction(porch_url=url, action="claim_task") + assert send(action=pa, pipeline=p) == response_data + + with monkeypatch.context() as mkp: + response_data["status"] = "DONE" + + def mock_get_200(*args, **kwargs): + return MockPorchResponse(response_data, 200) + + mkp.setattr(requests, "request", mock_get_200) + + pa = PorchAction( + porch_url=url, action="update_task", task_input=task, task_status="DONE" + ) + assert send(action=pa, pipeline=p) == response_data diff --git a/tests/test_send_request.py b/tests/test_send_request.py new file mode 100644 index 0000000..188ceee --- /dev/null +++ b/tests/test_send_request.py @@ -0,0 +1,96 @@ +import pytest +import requests + +from npg_porch_cli import send_request +from npg_porch_cli.api import AuthException, ServerErrorException + +url = "http://some.com" +var_name = "NPG_PORCH_TOKEN" +json_data = {"some_data": "delivered"} + + +class MockResponseOK: + def __init__(self): + self.status_code = 200 + self.reason = "OK" + self.url = url + self.ok = True + + def json(self): + return json_data + + +class MockResponseNotFound: + def __init__(self): + self.status_code = 404 + self.reason = "NOT FOUND" + self.url = url + self.ok = False + + def json(self): + return {"detail": "Not found in our data"} + + +class MockResponseNotFoundShort: + def __init__(self): + self.status_code = 404 + self.reason = "NOT FOUND" + self.url = url + self.ok = False + + def json(self): + return {} + + +def mock_get_200(*args, **kwargs): + return MockResponseOK() + + +def mock_get_404(*args, **kwargs): + return MockResponseNotFound() + + +def mock_get_404_short(*args, **kwargs): + return MockResponseNotFoundShort() + + +def test_sending_request(monkeypatch): + monkeypatch.delenv(var_name, raising=False) + + with pytest.raises(ValueError) as e: + send_request(validate_ca_cert=True, url=url, method="GET", auth_type="unknown") + assert e.value.args[0] == "Authorization type unknown is not implemented" + + with pytest.raises(AuthException) as e: + send_request(validate_ca_cert=True, url=url, method="GET") + assert e.value.args[0] == "Authorization token is needed" + + with monkeypatch.context() as m: + m.setattr(requests, "request", mock_get_200) + assert ( + send_request(validate_ca_cert=True, url=url, method="GET", auth_type=None) + == json_data + ) + + monkeypatch.setenv(var_name, "token_xyz") + + with monkeypatch.context() as m: + m.setattr(requests, "request", mock_get_200) + assert send_request(validate_ca_cert=False, url=url, method="GET") == json_data + + with monkeypatch.context() as m: + m.setattr(requests, "request", mock_get_404) + with pytest.raises(ServerErrorException) as e: + send_request(validate_ca_cert=False, url=url, method="POST", data=json_data) + assert e.value.args[0] == ( + 'Status code 404 "NOT FOUND" received from ' + f"{url}.\nDetail: Not found in our data" + ) + + with monkeypatch.context() as m: + m.setattr(requests, "request", mock_get_404_short) + with pytest.raises(ServerErrorException) as e: + send_request(validate_ca_cert=False, url=url, method="POST", data=json_data) + assert e.value.args[0] == f'Status code 404 "NOT FOUND" received from {url}' + + monkeypatch.undo()