Skip to content

Commit

Permalink
Merge pull request #18457 from bwalkowi/add-onedata-templates
Browse files Browse the repository at this point in the history
Add onedata templates
  • Loading branch information
mvdbeek authored Sep 4, 2024
2 parents e85de5d + 6ab1f7c commit 55a9c22
Show file tree
Hide file tree
Showing 17 changed files with 234 additions and 33 deletions.
1 change: 1 addition & 0 deletions client/src/api/fileSources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ export type FileSourceTemplateSummary = components["schemas"]["FileSourceTemplat
export type FileSourceTemplateSummaries = FileSourceTemplateSummary[];

export type UserFileSourceModel = components["schemas"]["UserFileSourceModel"];
export type FileSourceTypes = UserFileSourceModel["type"];
8 changes: 4 additions & 4 deletions client/src/api/schema/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8785,7 +8785,7 @@ export interface components {
* Type
* @enum {string}
*/
type: "ftp" | "posix" | "s3fs" | "azure";
type: "ftp" | "posix" | "s3fs" | "azure" | "onedata";
/** Variables */
variables?:
| (
Expand Down Expand Up @@ -13748,7 +13748,7 @@ export interface components {
* Type
* @enum {string}
*/
type: "aws_s3" | "azure_blob" | "boto3" | "disk" | "generic_s3";
type: "aws_s3" | "azure_blob" | "boto3" | "disk" | "generic_s3" | "onedata";
/** Variables */
variables?:
| (
Expand Down Expand Up @@ -16503,7 +16503,7 @@ export interface components {
* Type
* @enum {string}
*/
type: "aws_s3" | "azure_blob" | "boto3" | "disk" | "generic_s3";
type: "aws_s3" | "azure_blob" | "boto3" | "disk" | "generic_s3" | "onedata";
/**
* Uuid
* Format: uuid4
Expand Down Expand Up @@ -16578,7 +16578,7 @@ export interface components {
* Type
* @enum {string}
*/
type: "ftp" | "posix" | "s3fs" | "azure";
type: "ftp" | "posix" | "s3fs" | "azure" | "onedata";
/** Uri Root */
uri_root: string;
/**
Expand Down
7 changes: 6 additions & 1 deletion client/src/components/FileSources/FileSourceTypeSpan.vue
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
<script setup lang="ts">
import { computed } from "vue";
import type { FileSourceTypes } from "@/api/fileSources";
const MESSAGES = {
posix: "This is a simple path based storage location that assumes the all the relevant paths are already mounted on the Galaxy server and target worker nodes.",
s3fs: "This is an remote file source plugin based on the Amazon Simple Storage Service (S3) interface. The AWS interface has become an industry standard and many storage vendors support it and use it to expose 'object' based storage.",
azure: "This is an remote file source plugin based on the Azure service.",
onedata: "This is an remote file source plugin based on the Onedata service.",
ftp: "This is an remote file source plugin based on the FTP protocol.",
};
interface Props {
type: "posix" | "s3fs";
type: FileSourceTypes;
}
const props = defineProps<Props>();
Expand Down
1 change: 1 addition & 0 deletions client/src/components/ObjectStore/ObjectStoreTypeSpan.vue
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const MESSAGES = {
boto3: "This is a storage location based on the Amazon Simple Storage Service (S3) interface, but likely not stored by Amazon. The AWS interface has become an industry standard and many storage vendors support it and use it to expose object based storage.",
generic_s3:
"This is a storage location based on the Amazon Simple Storage Service (S3) interface, but likely not stored by Amazon. The AWS interface has become an industry standard and many storage vendors support it and use it to expose object based storage.",
onedata: "This is a storage location based on the Onedata system.",
};
interface Props {
Expand Down
6 changes: 3 additions & 3 deletions lib/galaxy/config/sample/file_sources_conf.yml.sample
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,6 @@
id: onedata1
label: Onedata
doc: Your Onedata files - configure an access token via user preferences
accessToken: ${user.preferences['onedata|access_token']}
onezoneDomain: ${user.preferences['onedata|onezone_domain']}
disableTlsCertificateValidation: ${user.preferences['onedata|disable_tls_certificate_validation']}
access_token: ${user.preferences['onedata|access_token']}
onezone_domain: ${user.preferences['onedata|onezone_domain']}
disable_tls_certificate_validation: ${user.preferences['onedata|disable_tls_certificate_validation']}
15 changes: 9 additions & 6 deletions lib/galaxy/config/sample/object_store_conf.sample.yml
Original file line number Diff line number Diff line change
Expand Up @@ -220,21 +220,24 @@ extra_dirs:

type: onedata
auth:
# an access token suitable for data access (allowing calls to the Oneprovider REST API).
# An access token suitable for data access (allowing calls to the Oneprovider REST API).
access_token: ...
connection:
# the domain of the Onezone service (e.g. datahub.egi.eu), or its IP address for
# devel instances (see above).
# The domain of the Onezone service (e.g. datahub.egi.eu), or its IP address for
# devel instances (see above). The minimal supported Onezone version is 21.02.4.
onezone_domain: datahub.egi.eu
# Allows connection to Onedata servers that do not present trusted SSL certificates.
# SHOULD NOT be used unless you really know what you are doing.
disable_tls_certificate_validation: false
space:
# the name of the Onedata space where the Galaxy data will be stored.
# The name of the Onedata space where the Galaxy data will be stored.
# If there is more than one space with the same name, you can explicitly
# specify which one to select by using the format <space_name>@<space_id>
# (e.g. demo@7285220ecc636075ae5759aec7ad65d3cha8f9).
name: demo-space
# the relative directory path in the space under which the Galaxy data will be stored.
# The relative directory path in the space at which the Galaxy data will be stored.
# Optional, if not provided, the data will be stored in the root of space.
path: galaxy-data
galaxy_root_dir: galaxy-data
cache:
path: database/object_store_cache_s3
size: 1000
Expand Down
11 changes: 7 additions & 4 deletions lib/galaxy/config/sample/object_store_conf.xml.sample
Original file line number Diff line number Diff line change
Expand Up @@ -166,24 +166,27 @@
//connection/@onezone_domain -
the domain of the Onezone service (e.g. datahub.egi.eu), or its IP address for
devel instances (see above).
devel instances (see above). The minimal supported Onezone version is 21.02.4.
//connection/@disable_tls_certificate_validation -
Allows connection to Onedata servers that do not present trusted SSL certificates.
SHOULD NOT be used unless you really know what you are doing.
//space/@name -
the name of the Onedata space where the Galaxy data will be stored.
If there is more than one space with the same name, you can explicitly
specify which one to select by using the format <space_name>@<space_id>
(e.g. demo@7285220ecc636075ae5759aec7ad65d3cha8f9).
//space/@path -
the relative directory path in the space under which the Galaxy data will be stored.
//space/@galaxy_root_dir -
the relative directory path in the space at which the Galaxy data will be stored.
Optional, if not provided, the data will be stored in the root of space.
-->
<!--
<object_store type="onedata">
<auth access_token="..." />
<connection onezone_domain="datahub.egi.eu" disable_tls_certificate_validation="False"/>
<space name="demo-space" path="galaxy-data" />
<space name="demo-space" galaxy_root_dir="galaxy-data" />
<cache path="database/object_store_cache" size="1000" cache_updated_data="True" />
<extra_dir type="job_work" path="database/job_working_directory_onedata"/>
<extra_dir type="temp" path="database/tmp_onedata"/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ preferences:
description: Your Onedata account
inputs:
- name: onezone_domain
label: Domain of the Onezone service (e.g. datahub.egi.eu)
label: Domain of the Onezone service (e.g. datahub.egi.eu). The minimal supported Onezone version is 21.02.4.
type: text
required: False
- name: access_token
Expand Down
2 changes: 1 addition & 1 deletion lib/galaxy/dependencies/conditional-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ fs.googledrivefs # type: googledrive
fs-gcsfs # type: googlecloudstorage
# fs-gcsfs doesn't pin google-cloud-storage, and old versions log noisy exceptions and break test discovery
google-cloud-storage>=2.8.0 # type: googlecloudstorage
fs.onedatarestfs==21.2.5.1 # type: onedata, depends on onedatafilerestclient
fs.onedatarestfs==21.2.5.2 # type: onedata, depends on onedatafilerestclient
fs-basespace # type: basespace
fs-azureblob # type: azure

Expand Down
2 changes: 1 addition & 1 deletion lib/galaxy/dependencies/dev-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ myst-parser==3.0.1 ; python_version >= "3.8" and python_version < "3.13"
nh3==0.2.18 ; python_version >= "3.8" and python_version < "3.13"
numpy==1.24.4 ; python_version >= "3.8" and python_version < "3.9"
numpy==2.0.0 ; python_version >= "3.9" and python_version < "3.13"
onedatafilerestclient==21.2.5.1 ; python_version >= "3.8" and python_version < "3.13"
onedatafilerestclient==21.2.5.2 ; python_version >= "3.8" and python_version < "3.13"
outcome==1.3.0.post0 ; python_version >= "3.8" and python_version < "3.13"
packaging==24.1 ; python_version >= "3.8" and python_version < "3.13"
pathspec==0.12.1 ; python_version >= "3.8" and python_version < "3.13"
Expand Down
18 changes: 14 additions & 4 deletions lib/galaxy/files/sources/onedata.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from typing import Optional

from galaxy.util import mapped_chars
from . import FilesSourceOptions
from ._pyfilesystem2 import PyFilesystem2FilesSource

Expand All @@ -22,11 +23,20 @@ class OnedataFilesSource(PyFilesystem2FilesSource):

def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None):
props = self._serialization_props(user_context)
onezone_domain = props.pop("onezoneDomain", "") or ""

access_token = props.pop("access_token", "") or ""
onezone_domain = props.pop("onezone_domain", "") or ""
onezone_domain = remove_prefix("http://", remove_prefix("https://", onezone_domain))
access_token = props.pop("accessToken", "") or ""
disable_tls_certificate_validation = props.pop("disableTlsCertificateValidation", False) or False
handle = OnedataRESTFS(onezone_domain, access_token, verify_ssl=not disable_tls_certificate_validation)
disable_tls_certificate_validation = props.pop("disable_tls_certificate_validation", False) or False

alt_space_fqn_separators = [mapped_chars["@"]] if "@" in mapped_chars else None

handle = OnedataRESTFS(
onezone_domain,
access_token,
alt_space_fqn_separators=alt_space_fqn_separators,
verify_ssl=not disable_tls_certificate_validation,
)
return handle


Expand Down
44 changes: 44 additions & 0 deletions lib/galaxy/files/templates/examples/onedata.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
- id: onedata
version: 0
name: Onedata Storage
description: |
If you have an account in Onedata, you can use such a file source to upload
your data directly from Onedata to Galaxy. You can also perform
an "Export datasets to remote files source" of a dataset from Galaxy to
Onedata.
The minimal supported Onezone version is 21.02.4.
More information on Onedata can be found on [Onedata's website](https://onedata.org/#/home).
variables:
onezone_domain:
label: Onezone domain
type: string
help: Domain of the Onezone service (e.g. datahub.egi.eu) to connect to.
disable_tls_certificate_validation:
label: Disable tls certificate validation?
type: boolean
help: |
Allows connection to Onedata servers that do not present trusted SSL certificates.
SHOULD NOT be used unless you really know what you are doing.
default: False
writable:
label: Writable?
type: boolean
help: |
Allow Galaxy to write data to this Onedata storage.
Requires an access token with write data access.
default: False
secrets:
access_token:
label: Access Token
help: |
Your [access token](https://onedata.org/#/home/documentation/topic/stable/tokens),
suitable for REST API access in a Oneprovider service. May be confined
to read-only data access, unless you wish to use the
"Export datasets to remote files source" tool (write permissions are needed then).
configuration:
type: onedata
access_token: "{{ secrets.access_token }}"
onezone_domain: "{{ variables.onezone_domain }}"
disable_tls_certificate_validation: "{{ variables.disable_tls_certificate_validation }}"
writable: "{{ variables.writable }}"
23 changes: 22 additions & 1 deletion lib/galaxy/files/templates/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
UserDetailsDict,
)

FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure"]
FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure", "onedata"]


class PosixFileSourceTemplateConfiguration(StrictModel):
Expand Down Expand Up @@ -104,17 +104,37 @@ class AzureFileSourceConfiguration(StrictModel):
writable: bool = False


class OnedataFileSourceTemplateConfiguration(StrictModel):
type: Literal["onedata"]
access_token: Union[str, TemplateExpansion]
onezone_domain: Union[str, TemplateExpansion]
disable_tls_certificate_validation: Union[bool, TemplateExpansion] = False
writable: Union[bool, TemplateExpansion] = False
template_start: Optional[str] = None
template_end: Optional[str] = None


class OnedataFileSourceConfiguration(StrictModel):
type: Literal["onedata"]
access_token: str
onezone_domain: str
disable_tls_certificate_validation: bool = False
writable: bool = False


FileSourceTemplateConfiguration = Union[
PosixFileSourceTemplateConfiguration,
S3FSFileSourceTemplateConfiguration,
FtpFileSourceTemplateConfiguration,
AzureFileSourceTemplateConfiguration,
OnedataFileSourceTemplateConfiguration,
]
FileSourceConfiguration = Union[
PosixFileSourceConfiguration,
S3FSFileSourceConfiguration,
FtpFileSourceConfiguration,
AzureFileSourceConfiguration,
OnedataFileSourceConfiguration,
]


Expand Down Expand Up @@ -175,6 +195,7 @@ def template_to_configuration(
"posix": PosixFileSourceConfiguration,
"s3fs": S3FSFileSourceConfiguration,
"azure": AzureFileSourceConfiguration,
"onedata": OnedataFileSourceConfiguration,
}


Expand Down
19 changes: 14 additions & 5 deletions lib/galaxy/objectstore/onedata.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@
except ImportError:
OnedataFileRESTClient = None

from galaxy.util import string_as_bool
from galaxy.util import (
mapped_chars,
string_as_bool,
)
from ._caching_base import CachingConcreteObjectStore
from .caching import (
enable_cache_monitor,
Expand Down Expand Up @@ -44,7 +47,7 @@ def _parse_config_xml(config_xml):

space_xml = _get_config_xml_elements(config_xml, "space")[0]
space_name = space_xml.get("name")
galaxy_root_dir = space_xml.get("path", "")
galaxy_root_dir = space_xml.get("galaxy_root_dir", "")

cache_dict = parse_caching_config_dict_from_xml(config_xml)

Expand Down Expand Up @@ -107,8 +110,8 @@ def __init__(self, config, config_dict):

cache_dict = config_dict.get("cache") or {}
self.enable_cache_monitor, self.cache_monitor_interval = enable_cache_monitor(config, config_dict)
self.cache_size = cache_dict["size"] or self.config.object_store_cache_size
self.staging_path = cache_dict["path"] or self.config.object_store_cache_path
self.cache_size = cache_dict.get("size") or self.config.object_store_cache_size
self.staging_path = cache_dict.get("path") or self.config.object_store_cache_path
self.cache_updated_data = cache_dict.get("cache_updated_data", True)

extra_dirs = {e["type"]: e["path"] for e in config_dict.get("extra_dirs", [])}
Expand All @@ -125,8 +128,14 @@ def _initialize(self):
f"(disable_tls_certificate_validation={self.disable_tls_certificate_validation})"
)

alt_space_fqn_separators = [mapped_chars["@"]] if "@" in mapped_chars else None
verify_ssl = not self.disable_tls_certificate_validation
self._client = OnedataFileRESTClient(self.onezone_domain, self.access_token, verify_ssl=verify_ssl)
self._client = OnedataFileRESTClient(
self.onezone_domain,
self.access_token,
alt_space_fqn_separators=alt_space_fqn_separators,
verify_ssl=verify_ssl,
)

self._ensure_staging_path_writable()
self._start_cache_monitor_if_needed()
Expand Down
Loading

0 comments on commit 55a9c22

Please sign in to comment.