Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Removed task monitor and enabled background data save flow #251

Merged
merged 4 commits into from
Dec 31, 2022
Merged

Removed task monitor and enabled background data save flow #251

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
af2da9e
removed task monitor and enabled background data save flow
kushagra189 Nov 26, 2022
40d95f7
fixed docstrings
kushagra189 Dec 3, 2022
88292d3
fix review comments
kushagra189 Dec 31, 2022
e849652
remove app version
uniqueg Dec 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions cwl_wes/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from flask import current_app
from foca import Foca

from cwl_wes.tasks.register_celery import register_task_service
from cwl_wes.ga4gh.wes.service_info import ServiceInfo
from cwl_wes.exceptions import NotFound

Expand All @@ -25,7 +24,6 @@ def init_app() -> App:
service_info.set_service_info(
data=current_app.config.foca.custom.service_info.dict()
)
register_task_service()
return app


Expand Down
274 changes: 274 additions & 0 deletions cwl_wes/custom_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,107 @@
from typing import Dict, List, Optional
from foca.models.config import FOCABaseConfig


class StorageConfig(FOCABaseConfig):
"""Model for task run and storage configuration.

Args:
tmp_dir: Temporary run directory path
permanent_dir: Permanent working directory path
remote_storage_url: Remote file storage FTP endpoint

Attributes:
tmp_dir: Temporary run directory path
permanent_dir: Permanent working directory path
remote_storage_url: Remote file storage FTP endpoint

Raises:
pydantic.ValidationError: The class was instantianted with an illegal
data type.

Example:
>>> StorageConfig(
... tmp_dir='/data/tmp',
... permanent_dir='/data/output',
... remote_storage_url='ftp://ftp.private/upload'
... )
StorageConfig(tmp_dir='/data/tmp', permanent_dir='/data/output', remote_storage_url='ftp://ftp.private/upload')
"""
permanent_dir: str = '/data/output'
tmp_dir: str = '/data/tmp'
remote_storage_url: str = 'ftp://ftp-private.ebi.ac.uk/upload/foivos'


class MonitorConfig(FOCABaseConfig):
"""Model for monitoring celery.

Args:
timeout: Celery task timeout.

Attributes:
timeout: Celery task timeout.

Raises:
pydantic.ValidationError: The class was instantianted with an illegal
data type.

Example:
>>> MonitorConfig(
... timeout=15,
... )
MonitorConfig(timeout=15)
"""
timeout: float = 0.1


class CeleryConfig(FOCABaseConfig):
"""Model for celery configurations.

Args:
monitor: Celery monitoring configurations.
message_maxsize: Celery message max size.

Attributes:
monitor: Celery monitoring configurations.
message_maxsize: Celery message max size.

Raises:
pydantic.ValidationError: The class was instantianted with an illegal
data type.

Example:
>>> CeleryConfig(
... monitor=MonitorConfig(timeout=15),
... message_maxsize=1024
... )
CeleryConfig(monitor=MonitorConfig(timeout=15), message_maxsize=1024)
"""
monitor: MonitorConfig = MonitorConfig()
message_maxsize: int = 16777216


class EndpointConfig(FOCABaseConfig):
"""Model for celery configurations.

Args:
monitor: Celery monitoring configurations.
message_maxsize: Celery message max size.

Attributes:
monitor: Celery monitoring configurations.
message_maxsize: Celery message max size.

Raises:
pydantic.ValidationError: The class was instantianted with an illegal
data type.

Example:
>>> CeleryConfig(
... monitor=MonitorConfig(timeout=15),
... message_maxsize=1024
... )
CeleryConfig(monitor=MonitorConfig(timeout=15), message_maxsize=1024)
"""
default_page_size: int = 5
timeout_cancel_run: int = 60
timeout_run_workflow: Optional[int] = None
Expand All @@ -29,32 +114,140 @@ class WorkflowTypeVersionConfig(FOCABaseConfig):
Args:
workflow_type_version: List of one or more acceptable versions for the
workflow type.

Attributes:
workflow_type_version: List of one or more acceptable versions for the
workflow type.

Raises:
pydantic.ValidationError: The class was instantianted with an illegal
data type.

Example:
>>> WorkflowTypeVersionConfig(
... workflow_type_version=['v1.0']
... )
WorkflowTypeVersionConfig(workflow_type_version=['v1.0'])
"""
workflow_type_version: Optional[List[str]] = []


class DefaultWorkflowEngineParameterConfig(FOCABaseConfig):
"""Model for default workflow engine parameters.

Args:
name: Parameter name.
type: Parameter type.
default_value: Stringified version of default parameter.

Attributes:
name: Parameter name.
type: Parameter type.
default_value: Stringified version of default parameter.

Raises:
pydantic.ValidationError: The class was instantianted with an illegal
data type.

Example:
>>> DefaultWorkflowEngineParameterConfig(
... name='name',
... type='str',
... default_value='default'
... )
DefaultWorkflowEngineParameterConfig(name='name', type='str', default_value='default')
"""
name: Optional[str]
type: Optional[str]
default_value: Optional[str]


class TagsConfig(FOCABaseConfig):
"""Model for service info tag configuration.

Args:
known_tes_endpoints: Valid TES endpoints.
app_version: Application version.

Attributes:
known_tes_endpoints: Valid TES endpoints.
app_version: Application version.

Raises:
pydantic.ValidationError: The class was instantianted with an illegal
data type.

Example:
>>> TagsConfig(
... known_tes_endpoints='https://tes.endpoint',
... app_version='0.1.x',
... )
TagsConfig(known_tes_endpoints='https://tes.endpoint', app_version='0.1.x')
"""
known_tes_endpoints: str = 'https://tes.tsi.ebi.ac.uk/|https://tes-dev.tsi.ebi.ac.uk/|https://csc-tesk.c03.k8s-popup.csc.fi/|https://tesk.c01.k8s-popup.csc.fi/'
app_version: str = '0.15.0'


class ServiceInfoConfig(FOCABaseConfig):
"""Model for service info configurations.

Args:
contact_info: Email address/webpage URL with contact information.
auth_instructions_url: Web page URL with information about how to get an
authorization token necessary to use a specific endpoint.
supported_filesystem_protocols: Filesystem protocols supported by this
service.
supported_wes_versions: Version(s) of the WES schema supported by this
service.
workflow_type_versions: Map with keys as the workflow format type name and
value is a `WorkflowTypeVersionConfig` object which simply contains an
array of one or more version strings.
workflow_engine_versions: Workflow engine(s) used by this WES service.
default_workflow_engine_parameters: Each workflow engine can present additional
parameters that can be sent to the workflow engine.
tags: A key-value map of arbitrary, extended metadata outside the scope of the above but
useful to report back.

Attributes:
contact_info: Email address/webpage URL with contact information.
auth_instructions_url: Web page URL with information about how to get an
authorization token necessary to use a specific endpoint.
supported_filesystem_protocols: Filesystem protocols supported by this
service.
supported_wes_versions: Version(s) of the WES schema supported by this
service.
workflow_type_versions: Map with keys as the workflow format type name and
value is a `WorkflowTypeVersionConfig` object which simply contains an
array of one or more version strings.
workflow_engine_versions: Workflow engine(s) used by this WES service.
default_workflow_engine_parameters: Each workflow engine can present additional
parameters that can be sent to the workflow engine.
tags: A key-value map of arbitrary, extended metadata outside the scope of the above but
useful to report back.

Raises:
pydantic.ValidationError: The class was instantianted with an illegal
data type.

Example:
>>> ServiceInfoConfig(
... contact_info='https://contact.url',
... auth_instructions_url='https://auth.url',
... supported_filesystem_protocols=['ftp', 'https', 'local'],
... supported_wes_versions=['1.0.0'],
... workflow_type_versions={'CWL': WorkflowTypeVersionConfig(workflow_type_version=['v1.0'])},
... workflow_engine_versions={},
... default_workflow_engine_parameters=[],
... tags=TagsConfig(known_tes_endpoints='https://tes.endpoint/', app_version='0.1.0')
... )
ServiceInfoConfig(contact_info='https://github.com/elixir-cloud-aai/cwl-WES', auth_instruc\
tions_url='https://www.elixir-europe.org/services/compute/aai', supported_filesystem_proto\
cols=['ftp', 'https', 'local'], supported_wes_versions=['1.0.0'], workflow_type_versions={\
'CWL': WorkflowTypeVersionConfig(workflow_type_version=['v1.0'])}, workflow_engine_version\
s={}, default_workflow_engine_parameters=[], tags=TagsConfig(known_tes_endpoints='https://\
tes.tsi.ebi.ac.uk/|https://tes-dev.tsi.ebi.ac.uk/|https://csc-tesk.c03.k8s-popup.csc.fi/|h\
ttps://tesk.c01.k8s-popup.csc.fi/', app_version='0.15.0'))
"""
contact_info: str = 'https://github.com/elixir-cloud-aai/cwl-WES'
auth_instructions_url: str = 'https://www.elixir-europe.org/services/compute/aai'
supported_filesystem_protocols: List[str] = ['ftp', 'https', 'local']
Expand All @@ -70,12 +263,69 @@ class ServiceInfoConfig(FOCABaseConfig):


class TesServerConfig(FOCABaseConfig):
"""Model for tes server configuration.

Args:
url: TES Endpoint URL.
timeout: Request time out.
status_query_params: Request query parameters.

Attributes:
url: TES Endpoint URL.
timeout: Request time out.
status_query_params: Request query parameters.

Raises:
pydantic.ValidationError: The class was instantianted with an illegal
data type.

Example:
>>> TesServerConfig(
... url='https://tes.endpoint',
... timeout=5,
... status_query_params='FULL'
... )
TesServerConfig(url='https://tes.endpoint', timeout=5, status_query_params='FULL')
"""
url: str = 'https://csc-tesk.c03.k8s-popup.csc.fi/'
timeout: int = 5
status_query_params: str = 'FULL'


class DRSServerConfig(FOCABaseConfig):
"""Model for DRS server configuration.

Args:
port: Port for resolving DRS URIs;
set to `null` to use default (443).
base_path: Base path for resolving DRS URIs;
set to `null` to use default (`ga4gh/drs/v1`).
use_http: Use `http` for resolving DRS URIs;
set to `False` to use default (`https`).
file_types: Extensions of files to scan for DRS URI resolution.

Attributes:
port: Port for resolving DRS URIs;
set to `null` to use default (443).
base_path: Base path for resolving DRS URIs;
set to `null` to use default (`ga4gh/drs/v1`).
use_http: Use `http` for resolving DRS URIs;
set to `False` to use default (`https`).
file_types: Extensions of files to scan for DRS URI resolution.

Raises:
pydantic.ValidationError: The class was instantianted with an illegal
data type.

Example:
>>> DRSServerConfig(
... port=443,
... base_path='ga4gh/drs/v1',
... use_http=False,
... file_types=['cwl', 'yaml', 'yml']
... )
DRSServerConfig(port=443, base_path='ga4gh/drs/v1', use_http=False, file_types=['cwl', 'yaml', 'yml'])
"""
port: Optional[int] = None
base_path: Optional[str] = None
use_http: bool = False
Expand Down Expand Up @@ -111,6 +361,30 @@ class IdConfig(FOCABaseConfig):


class CustomConfig(FOCABaseConfig):
"""Model for custom configuration parameters.

Args:
storage: Storage config parameters.
celery: Celery config parameters.
endpoint_params: Endpoint config parameters.
service_info: Service Info config parameters.
tes_server: TES Server config parameters.
drs_server: DRS Server config parameters.
runs_id: Identifier config parameters.

Attributes:
storage: Storage config parameters.
celery: Celery config parameters.
endpoint_params: Endpoint config parameters.
service_info: Service Info config parameters.
tes_server: TES Server config parameters.
drs_server: DRS Server config parameters.
runs_id: Identifier config parameters.

Raises:
pydantic.ValidationError: The class was instantianted with an illegal
data type.
"""
storage: StorageConfig = StorageConfig()
celery: CeleryConfig = CeleryConfig()
endpoint_params: EndpointConfig = EndpointConfig()
Expand Down
Loading