diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 51148bebc8d7..be56f1ba7b3e 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -9035,7 +9035,7 @@ export interface components { * Type * @enum {string} */ - type: "ftp" | "posix" | "s3fs" | "azure" | "onedata"; + type: "ftp" | "posix" | "s3fs" | "azure" | "onedata" | "webdav"; /** Variables */ variables?: | ( @@ -17366,7 +17366,7 @@ export interface components { * Type * @enum {string} */ - type: "ftp" | "posix" | "s3fs" | "azure" | "onedata"; + type: "ftp" | "posix" | "s3fs" | "azure" | "onedata" | "webdav"; /** Uri Root */ uri_root: string; /** diff --git a/client/src/components/FileSources/FileSourceTypeSpan.vue b/client/src/components/FileSources/FileSourceTypeSpan.vue index e41b3b32858d..d98cdce9d622 100644 --- a/client/src/components/FileSources/FileSourceTypeSpan.vue +++ b/client/src/components/FileSources/FileSourceTypeSpan.vue @@ -9,6 +9,7 @@ const MESSAGES = { azure: "This is an remote file source plugin based on the Azure service.", onedata: "This is an remote file source plugin based on the Onedata service.", ftp: "This is an remote file source plugin based on the FTP protocol.", + webdav: "This is an remote file source plugin based on the WebDAV protocol.", }; interface Props { diff --git a/doc/source/admin/data.md b/doc/source/admin/data.md index 1e9a3ea90952..0ec463c8f6ee 100644 --- a/doc/source/admin/data.md +++ b/doc/source/admin/data.md @@ -350,6 +350,19 @@ configuration). ![](file_source_azure_configuration.png) +#### `webdav` + +The syntax for the `configuration` section of `webdav` templates looks like this. + +![](file_source_webdav_configuration_template.png) + +At runtime, after the `configuration` template is expanded, the resulting dictionary +passed to Galaxy's file source plugin infrastructure looks like this and should match a subset +of what you'd be able to add directly to `file_sources_conf.yml` (Galaxy's global file source +configuration). + +![](file_source_webdav_configuration.png) + ### YAML Syntax ![galaxy.files.templates.models](file_source_templates.png) @@ -398,6 +411,14 @@ and you are comfortable with it storing your user's secrets. :language: yaml ``` +#### Allow Users to Define WebDAV Servers as File Sources + +```{literalinclude} ../../../lib/galaxy/files/templates/examples/production_webdav.yml +:language: yaml +``` + +![Screenshot](user_file_source_form_full_webdav.png) + ## Playing Nicer with Ansible Many large instances of Galaxy are configured with Ansible and much of the existing administrator diff --git a/doc/source/admin/file_source_templates.png b/doc/source/admin/file_source_templates.png index 5d1556b69269..8e3c3fabe39e 100644 Binary files a/doc/source/admin/file_source_templates.png and b/doc/source/admin/file_source_templates.png differ diff --git a/doc/source/admin/file_source_webdav_configuration.png b/doc/source/admin/file_source_webdav_configuration.png new file mode 100644 index 000000000000..26ec57bd246d Binary files /dev/null and b/doc/source/admin/file_source_webdav_configuration.png differ diff --git a/doc/source/admin/file_source_webdav_configuration_template.png b/doc/source/admin/file_source_webdav_configuration_template.png new file mode 100644 index 000000000000..4684e8f05b8a Binary files /dev/null and b/doc/source/admin/file_source_webdav_configuration_template.png differ diff --git a/doc/source/admin/gen_diagrams.py b/doc/source/admin/gen_diagrams.py index 2627420177eb..108876b97faa 100644 --- a/doc/source/admin/gen_diagrams.py +++ b/doc/source/admin/gen_diagrams.py @@ -15,6 +15,8 @@ PosixFileSourceTemplateConfiguration, S3FSFileSourceConfiguration, S3FSFileSourceTemplateConfiguration, + WebdavFileSourceConfiguration, + WebdavFileSourceTemplateConfiguration, ) from galaxy.objectstore.templates.models import ( AwsS3ObjectStoreConfiguration, @@ -53,6 +55,8 @@ S3FSFileSourceConfiguration: "file_source_s3fs_configuration", FtpFileSourceTemplateConfiguration: "file_source_ftp_configuration_template", FtpFileSourceConfiguration: "file_source_ftp_configuration", + WebdavFileSourceTemplateConfiguration: "file_source_webdav_configuration_template", + WebdavFileSourceConfiguration: "file_source_webdav_configuration", } for clazz, diagram_name in class_to_diagram.items(): diff --git a/doc/source/admin/search_for_new_screenshots.py b/doc/source/admin/search_for_new_screenshots.py index 95f0ac2cbc69..ea09034da8d4 100644 --- a/doc/source/admin/search_for_new_screenshots.py +++ b/doc/source/admin/search_for_new_screenshots.py @@ -22,6 +22,7 @@ "user_file_source_form_full_aws_public.png", "user_file_source_form_full_azure.png", "user_file_source_form_full_ftp.png", + "user_file_source_form_full_webdav.png", ] diff --git a/doc/source/admin/user_file_source_form_full_webdav.png b/doc/source/admin/user_file_source_form_full_webdav.png new file mode 100755 index 000000000000..6e5ed85b5096 Binary files /dev/null and b/doc/source/admin/user_file_source_form_full_webdav.png differ diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 4b8519a6624f..3ed0c7455e74 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -3746,7 +3746,7 @@ mapping: Workflows launched with URI/URL inputs that are not marked as 'deferred' are "materialized" (or undeferred) by the workflow scheduler. This might be a lengthy process. Setting this to 'True' will place the invocation back in - the queue after materialization before scheduling the workflow so it is less + the queue after materialization before scheduling the workflow so it is less likely to starve other workflow scheduling. Ideally, Galaxy would allow more fine grain control of handlers but until then, this provides a way to tip the balance between "doing more work" and "being more fair". The default here is @@ -4110,7 +4110,7 @@ mapping: required: false desc: | Directory to store temporary files for file sources. This defaults to new_file_path if not set. - + file_source_webdav_use_temp_files: type: bool default: true diff --git a/lib/galaxy/files/templates/examples/production_webdav.yml b/lib/galaxy/files/templates/examples/production_webdav.yml new file mode 100644 index 000000000000..0767dc550867 --- /dev/null +++ b/lib/galaxy/files/templates/examples/production_webdav.yml @@ -0,0 +1,42 @@ +- id: webdav + version: 0 + name: WebDAV + description: | + The WebDAV protocol is a simple way to access files over the internet. This template + configuration allows you to connect to a WebDAV server. + variables: + url: + label: Server Domain (e.g. https://myowncloud.org) + type: string + help: | + The domain of the WebDAV server you are connecting to. This should be the full URL + including the protocol (http or https) and the domain name. + root: + label: WebDAV server Path (should end with /remote.php/webdav, e.g. /a/sub/path/remote.php/webdav) + type: string + help: | + The full server path to the WebDAV service. Ensure the path includes /remote.php/webdav. + login: + label: Username + type: string + help: | + The username to use to connect to the WebDAV server. This should be the username you use + to log in to the WebDAV server. + writable: + label: Writable? + type: boolean + default: false + help: Allow Galaxy to write data to this WebDAV server. + secrets: + password: + label: Password + help: | + The password to use to connect to the WebDAV server. This should be the password you use + to log in to the WebDAV server. + configuration: + type: webdav + url: '{{ variables.url }}' + root: '{{ variables.root }}' + login: '{{ variables.login }}' + writable: '{{ variables.writable }}' + password: '{{ secrets.password }}' diff --git a/lib/galaxy/files/templates/models.py b/lib/galaxy/files/templates/models.py index e4b7fcea49e8..bff512f130ca 100644 --- a/lib/galaxy/files/templates/models.py +++ b/lib/galaxy/files/templates/models.py @@ -25,7 +25,7 @@ UserDetailsDict, ) -FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure", "onedata"] +FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure", "onedata", "webdav"] class PosixFileSourceTemplateConfiguration(StrictModel): @@ -122,12 +122,33 @@ class OnedataFileSourceConfiguration(StrictModel): writable: bool = False +class WebdavFileSourceTemplateConfiguration(StrictModel): + type: Literal["webdav"] + url: Union[str, TemplateExpansion] + root: Union[str, TemplateExpansion] + login: Union[str, TemplateExpansion] + password: Union[str, TemplateExpansion] + writable: Union[bool, TemplateExpansion] = False + template_start: Optional[str] = None + template_end: Optional[str] = None + + +class WebdavFileSourceConfiguration(StrictModel): + type: Literal["webdav"] + url: str + root: str + login: str + password: str + writable: bool = False + + FileSourceTemplateConfiguration = Union[ PosixFileSourceTemplateConfiguration, S3FSFileSourceTemplateConfiguration, FtpFileSourceTemplateConfiguration, AzureFileSourceTemplateConfiguration, OnedataFileSourceTemplateConfiguration, + WebdavFileSourceTemplateConfiguration, ] FileSourceConfiguration = Union[ PosixFileSourceConfiguration, @@ -135,6 +156,7 @@ class OnedataFileSourceConfiguration(StrictModel): FtpFileSourceConfiguration, AzureFileSourceConfiguration, OnedataFileSourceConfiguration, + WebdavFileSourceConfiguration, ] @@ -196,6 +218,7 @@ def template_to_configuration( "s3fs": S3FSFileSourceConfiguration, "azure": AzureFileSourceConfiguration, "onedata": OnedataFileSourceConfiguration, + "webdav": WebdavFileSourceConfiguration, }