From 0d94c8782c69b5ef66861d55f9a88b5ce0a30e7a Mon Sep 17 00:00:00 2001 From: enricovianello Date: Wed, 17 Oct 2018 16:59:29 +0200 Subject: [PATCH 1/4] Introduced new logic for publishing WebDAV Endpoints Still supporting the oldest with GRIDHTTPS vars --- src/info_provider/configuration.py | 20 +- src/info_provider/glue/glue2.py | 190 +++++++++++------- .../storm_storage_service_builder.py | 21 +- src/resources/logging.ini | 2 +- src/tests/resources/storm.def | 64 ++---- src/tests/resources/storm.gridhttps.def | 53 +++++ src/tests/test_configuration.py | 2 +- src/tests/test_storage_service.py | 76 ++++++- src/tests/utils.py | 18 +- 9 files changed, 294 insertions(+), 152 deletions(-) create mode 100644 src/tests/resources/storm.gridhttps.def diff --git a/src/info_provider/configuration.py b/src/info_provider/configuration.py index 3488d42..6c7f1c6 100644 --- a/src/info_provider/configuration.py +++ b/src/info_provider/configuration.py @@ -13,8 +13,7 @@ class Configuration: _mandatories = ["SITE_NAME", "STORM_BACKEND_HOST", "STORM_DEFAULT_ROOT", "STORM_FRONTEND_PATH", "STORM_FRONTEND_PORT", "STORM_FRONTEND_PUBLIC_HOST", "STORM_BACKEND_REST_SERVICES_PORT", - "VOS", "STORM_ENDPOINT_QUALITY_LEVEL", "STORM_ENDPOINT_CAPABILITY", - "STORM_GRIDHTTPS_PUBLIC_HOST"] + "VOS", "STORM_ENDPOINT_QUALITY_LEVEL"] def __init__(self, filepath): logging.debug("Init configuration from file %s ...", filepath) @@ -73,7 +72,7 @@ def get_enabled_access_protocols(self): enabled.append("http") if self.get("STORM_INFO_HTTPS_SUPPORT").lower() == "true": enabled.append("https") - if self.has_gridhttps(): + if self.has_webdav(): enabled.append("webdav") return enabled @@ -86,7 +85,9 @@ def get_frontend_list(self): return self.get("STORM_FRONTEND_HOST_LIST").split(',') def has_gridhttps(self): - return self.get('STORM_GRIDHTTPS_ENABLED').lower() == "true" + if 'STORM_GRIDHTTPS_ENABLED' in self._configuration: + return self.get('STORM_GRIDHTTPS_ENABLED').lower() == "true" + return False def get_gridhttps_list(self): return self.get("STORM_GRIDHTTPS_POOL_LIST").split(',') @@ -94,6 +95,12 @@ def get_gridhttps_list(self): def is_HTTP_endpoint_enabled(self): return self.get('STORM_GRIDHTTPS_HTTP_ENABLED').lower() == "true" + def has_webdav(self): + if 'STORM_WEBDAV_POOL_LIST' in self._configuration: + if self.get_webdav_endpoints(): + return True + return False + def get_public_srm_endpoint(self): host = self.get("STORM_FRONTEND_PUBLIC_HOST") port = str(self.get("STORM_FRONTEND_PORT")) @@ -110,6 +117,11 @@ def get_public_http_endpoint(self): port = str(self.get("STORM_GRIDHTTPS_HTTP_PORT")) return "http://" + host + ":" + port + "/" + def get_webdav_endpoints(self): + endpoints = self.get("STORM_WEBDAV_POOL_LIST").split(',') + logging.debug(endpoints) + return endpoints + def is_info_overwrite(self): return self.get("STORM_INFO_OVERWRITE").lower() == "true" diff --git a/src/info_provider/glue/glue2.py b/src/info_provider/glue/glue2.py index 8c0c206..d2108c6 100644 --- a/src/info_provider/glue/glue2.py +++ b/src/info_provider/glue/glue2.py @@ -1,6 +1,7 @@ import logging import os import re +from urlparse import urlparse from info_provider.glue.commons import INFO_PROVIDER_SCRIPT, \ INPUT_YAIM_CONFIGURATION @@ -38,14 +39,14 @@ def _get_domain(self): def _get_manager_id(self): return self._get_service_id() + "/manager" - def _get_srm_endpoint_id(self): - return self._get_service_id() + "/endpoint/SRM" + def _get_srm_endpoint_id(self, index=0): + return self._get_service_id() + "/endpoint/SRM" + str(index) - def _get_http_endpoint_id(self): - return self._get_service_id() + "/endpoint/HTTP" + def _get_http_endpoint_id(self, index=0): + return self._get_service_id() + "/endpoint/HTTP" + str(index) - def _get_https_endpoint_id(self): - return self._get_service_id() + "/endpoint/HTTPS" + def _get_https_endpoint_id(self, index=0): + return self._get_service_id() + "/endpoint/HTTPS" + str(index) def _get_endpoint_policy_id(self, endpoint_id): return endpoint_id + "_Policy" @@ -101,7 +102,7 @@ def configure(self, spaceinfo): return def _create_service_config_file(self): - params = { + params = { 'SITEID': self._get_site_id(), 'SERVICEID': self._get_service_id(), 'QUALITY_LEVEL': self._configuration.get_quality_level() @@ -184,14 +185,14 @@ def get_static_ldif_nodes(self, spaceinfo): node = GLUE2StorageServiceCapacity(sc_id, service_id) node.init().add({ 'GLUE2StorageServiceCapacityType': "online", - 'GLUE2StorageServiceCapacityTotalSize': + 'GLUE2StorageServiceCapacityTotalSize': as_gigabytes(spaceinfo.get_summary().get_total()), - 'GLUE2StorageServiceCapacityFreeSize': + 'GLUE2StorageServiceCapacityFreeSize': as_gigabytes(spaceinfo.get_summary().get_free()), 'GLUE2StorageServiceCapacityUsedSize': as_gigabytes(spaceinfo.get_summary().get_used()), 'GLUE2StorageServiceCapacityReservedSize': - as_gigabytes(spaceinfo.get_summary().get_reserved()) + as_gigabytes(spaceinfo.get_summary().get_reserved()) }) nodes.append(node) @@ -249,7 +250,7 @@ def get_static_ldif_nodes(self, spaceinfo): }) nodes.append(node) - # Glue2Share, GLUE2MappingPolicy and Glue2StorageShareCapacity for each + # Glue2Share, GLUE2MappingPolicy and Glue2StorageShareCapacity for each # VFS for name, data in spaceinfo.get_vfs().items(): @@ -282,20 +283,20 @@ def get_static_ldif_nodes(self, spaceinfo): 'GLUE2PolicyRule': data.get_approachablerules() }) if not self._is_anonymous(data.get_voname()): - node.add({ + node.add({ 'GLUE2PolicyUserDomainForeignKey': data.get_voname() }) nodes.append(node) - + # Glue2StorageShareCapacities - if data.get_space().has_online_capacity(): + if data.get_space().has_online_capacity(): # Glue2StorageShareCapacity online capacity_id = self._get_share_capacity_id(name, "online") node = GLUE2StorageShareCapacity(capacity_id, share_id, service_id) node.init().add({ 'GLUE2StorageShareCapacityType': "online", - 'GLUE2StorageShareCapacityTotalSize': + 'GLUE2StorageShareCapacityTotalSize': as_gigabytes(data.get_space().get_total()), 'GLUE2StorageShareCapacityFreeSize': as_gigabytes(data.get_space().get_free()), @@ -306,15 +307,15 @@ def get_static_ldif_nodes(self, spaceinfo): }) nodes.append(node) - if data.get_space().has_nearline_capacity(): + if data.get_space().has_nearline_capacity(): # Glue2StorageShareCapacity near-line capacity_id = self._get_share_capacity_id(name, "nearline") node = GLUE2StorageShareCapacity(capacity_id, share_id, service_id) node.init().add({ 'GLUE2StorageShareCapacityType': "nearline", - 'GLUE2StorageShareCapacityTotalSize': + 'GLUE2StorageShareCapacityTotalSize': as_gigabytes(data.get_space().get_nearline()), - 'GLUE2StorageShareCapacityFreeSize': + 'GLUE2StorageShareCapacityFreeSize': as_gigabytes(data.get_space().get_nearline()), 'GLUE2StorageShareCapacityUsedSize': 0, 'GLUE2StorageShareCapacityReservedSize': 0 @@ -325,49 +326,83 @@ def get_static_ldif_nodes(self, spaceinfo): for vo in self._configuration.get_used_VOs(): access_policy_rules.append("vo:" + vo) - if self._configuration.has_gridhttps(): + # NEW LOGIC + if self._configuration.has_webdav(): + + i = 0; + for endpoint in self._configuration.get_webdav_endpoints(): + protocol = urlparse(endpoint).scheme.upper() + if protocol == "HTTP": + endpoint_id = self._get_http_endpoint_id(i) + elif protocol == "HTTPS": + endpoint_id = self._get_https_endpoint_id(i) + else: + raise ValueError("unable to read a valid protocol from " + endpoint) + node = GLUE2WebDAVStorageEndpoint(endpoint_id, service_id) + node.init().add({ + 'GLUE2EndpointURL': endpoint, + 'GLUE2EndpointImplementationVersion': storm_version, + 'GLUE2EndpointQualityLevel': self._configuration.get_quality_level(), + 'GLUE2EndpointServingState': self._configuration.get_serving_state(), + 'GLUE2EndpointIssuerCA': issuer_ca + }) + nodes.append(node) + # Add Endpoint Policy + policy_id = self._get_endpoint_policy_id(endpoint_id) + node = GLUE2AccessPolicy(policy_id, endpoint_id, service_id) + node.init().add({ + 'GLUE2PolicyRule': access_policy_rules, + 'GLUE2PolicyUserDomainForeignKey': self._configuration.get_used_VOs() + }) + nodes.append(node) + i += 1 - # Add HTTP WebDAV Endpoint - endpoint_id = self._get_http_endpoint_id() - node = GLUE2WebDAVStorageEndpoint(endpoint_id, service_id) - node.init().add({ - 'GLUE2EndpointURL': self._configuration.get_public_http_endpoint(), - 'GLUE2EndpointImplementationVersion': storm_version, - 'GLUE2EndpointQualityLevel': self._configuration.get_quality_level(), - 'GLUE2EndpointServingState': self._configuration.get_serving_state(), - 'GLUE2EndpointIssuerCA': issuer_ca - }) - nodes.append(node) + else: - # Add Endpoint Policy - policy_id = self._get_endpoint_policy_id(endpoint_id) - node = GLUE2AccessPolicy(policy_id, endpoint_id, service_id) - node.init().add({ - 'GLUE2PolicyRule': access_policy_rules, - 'GLUE2PolicyUserDomainForeignKey': self._configuration.get_used_VOs() - }) - nodes.append(node) + # OLD_LOGIC + if self._configuration.has_gridhttps(): - # Add HTTPs Endpoint - endpoint_id = self._get_https_endpoint_id() - node = GLUE2WebDAVStorageEndpoint(endpoint_id, service_id) - node.init().add({ - 'GLUE2EndpointURL': self._configuration.get_public_https_endpoint(), - 'GLUE2EndpointImplementationVersion': storm_version, - 'GLUE2EndpointQualityLevel': self._configuration.get_quality_level(), - 'GLUE2EndpointServingState': self._configuration.get_serving_state(), - 'GLUE2EndpointIssuerCA': issuer_ca - }) - nodes.append(node) + # Add HTTP WebDAV Endpoint + endpoint_id = self._get_http_endpoint_id() + node = GLUE2WebDAVStorageEndpoint(endpoint_id, service_id) + node.init().add({ + 'GLUE2EndpointURL': self._configuration.get_public_http_endpoint(), + 'GLUE2EndpointImplementationVersion': storm_version, + 'GLUE2EndpointQualityLevel': self._configuration.get_quality_level(), + 'GLUE2EndpointServingState': self._configuration.get_serving_state(), + 'GLUE2EndpointIssuerCA': issuer_ca + }) + nodes.append(node) - # Add Endpoint Policy - policy_id = self._get_endpoint_policy_id(endpoint_id) - node = GLUE2AccessPolicy(policy_id, endpoint_id, service_id) - node.init().add({ - 'GLUE2PolicyRule': access_policy_rules, - 'GLUE2PolicyUserDomainForeignKey': self._configuration.get_used_VOs() - }) - nodes.append(node) + # Add Endpoint Policy + policy_id = self._get_endpoint_policy_id(endpoint_id) + node = GLUE2AccessPolicy(policy_id, endpoint_id, service_id) + node.init().add({ + 'GLUE2PolicyRule': access_policy_rules, + 'GLUE2PolicyUserDomainForeignKey': self._configuration.get_used_VOs() + }) + nodes.append(node) + + # Add HTTPs Endpoint + endpoint_id = self._get_https_endpoint_id() + node = GLUE2WebDAVStorageEndpoint(endpoint_id, service_id) + node.init().add({ + 'GLUE2EndpointURL': self._configuration.get_public_https_endpoint(), + 'GLUE2EndpointImplementationVersion': storm_version, + 'GLUE2EndpointQualityLevel': self._configuration.get_quality_level(), + 'GLUE2EndpointServingState': self._configuration.get_serving_state(), + 'GLUE2EndpointIssuerCA': issuer_ca + }) + nodes.append(node) + + # Add Endpoint Policy + policy_id = self._get_endpoint_policy_id(endpoint_id) + node = GLUE2AccessPolicy(policy_id, endpoint_id, service_id) + node.init().add({ + 'GLUE2PolicyRule': access_policy_rules, + 'GLUE2PolicyUserDomainForeignKey': self._configuration.get_used_VOs() + }) + nodes.append(node) return nodes @@ -384,7 +419,16 @@ def get_update_ldif_endpoints(self, serving_state_value): node.add({ 'GLUE2EndpointServingState': serving_state_value }) nodes.append(node) - if self._configuration.has_gridhttps(): + if self._configuration.has_webdav(): + + for endpoint in self._configuration.get_webdav_endpoints(): + endpoint_id = self._get_webdav_endpoint_id(endpoint) + # Glue2StorageEndpoint http webdav serving_state_value + node = GLUE2StorageEndpoint(endpoint_id, service_ID) + node.add({ 'GLUE2EndpointServingState': serving_state_value }) + nodes.append(node) + + elif self._configuration.has_gridhttps(): # Glue2StorageEndpoint http webdav serving_state_value node = GLUE2StorageEndpoint(self._get_http_endpoint_id(), @@ -412,8 +456,8 @@ def get_update_ldif_spaceinfo(self, spaceinfo, serving_state_value): if spaceinfo.get_summary().has_online_capacity(): sc_id = self._get_service_capacity_id("online") node = GLUE2StorageServiceCapacity(sc_id, service_ID) - node.add({ - 'GLUE2StorageServiceCapacityTotalSize': + node.add({ + 'GLUE2StorageServiceCapacityTotalSize': as_gigabytes(spaceinfo.get_summary().get_total()), 'GLUE2StorageServiceCapacityFreeSize': as_gigabytes(spaceinfo.get_summary().get_free()), @@ -428,17 +472,17 @@ def get_update_ldif_spaceinfo(self, spaceinfo, serving_state_value): if spaceinfo.get_summary().has_nearline_capacity(): sc_id = self._get_service_capacity_id("nearline") node = GLUE2StorageServiceCapacity(sc_id, service_ID) - node.add({ - 'GLUE2StorageServiceCapacityTotalSize': + node.add({ + 'GLUE2StorageServiceCapacityTotalSize': as_gigabytes(spaceinfo.get_summary().get_nearline()), - 'GLUE2StorageServiceCapacityFreeSize': + 'GLUE2StorageServiceCapacityFreeSize': as_gigabytes(spaceinfo.get_summary().get_nearline()), 'GLUE2StorageServiceCapacityUsedSize': 0, 'GLUE2StorageServiceCapacityReservedSize': 0 }) nodes.append(node) - # Glue2Share, GLUE2MappingPolicy and Glue2StorageShareCapacity for each + # Glue2Share, GLUE2MappingPolicy and Glue2StorageShareCapacity for each # VFS for name, data in spaceinfo.get_vfs().iteritems(): @@ -452,24 +496,24 @@ def get_update_ldif_spaceinfo(self, spaceinfo, serving_state_value): if data.get_space().has_online_capacity(): capacity_id = self._get_share_capacity_id(name, "online") node = GLUE2StorageShareCapacity(capacity_id, share_id, service_ID) - node.add({ - 'GLUE2StorageShareCapacityTotalSize': + node.add({ + 'GLUE2StorageShareCapacityTotalSize': as_gigabytes(data.get_space().get_total()), - 'GLUE2StorageShareCapacityFreeSize': + 'GLUE2StorageShareCapacityFreeSize': as_gigabytes(data.get_space().get_free()), - 'GLUE2StorageShareCapacityUsedSize': + 'GLUE2StorageShareCapacityUsedSize': as_gigabytes(data.get_space().get_used()), - 'GLUE2StorageShareCapacityReservedSize': + 'GLUE2StorageShareCapacityReservedSize': as_gigabytes(data.get_space().get_reserved()) }) nodes.append(node) if data.get_space().has_nearline_capacity(): capacity_id = self._get_share_capacity_id(name, "nearline") node = GLUE2StorageShareCapacity(capacity_id, share_id, service_ID) - node.add({ - 'GLUE2StorageShareCapacityTotalSize': + node.add({ + 'GLUE2StorageShareCapacityTotalSize': as_gigabytes(data.get_space().get_nearline()), - 'GLUE2StorageShareCapacityFreeSize': + 'GLUE2StorageShareCapacityFreeSize': as_gigabytes(data.get_space().get_nearline()), 'GLUE2StorageShareCapacityUsedSize': 0, 'GLUE2StorageShareCapacityReservedSize': 0 @@ -486,4 +530,4 @@ def _delete_backup_files(self): os.remove(os.path.join(parent_directory, f)) removed_list.append(f) logging.debug("Removed backup files: [%s]", removed_list) - return len(removed_list) + return len(removed_list) diff --git a/src/info_provider/storm_storage_service_builder.py b/src/info_provider/storm_storage_service_builder.py index 936bbe4..10ac57e 100644 --- a/src/info_provider/storm_storage_service_builder.py +++ b/src/info_provider/storm_storage_service_builder.py @@ -1,5 +1,7 @@ import logging +from urlparse import urlparse + from info_provider.model.storage import StorageShare, StorageEndpoint, \ StorageService @@ -28,16 +30,27 @@ def build(self): sa = StorageShare(name=token, access_latency=access_latency, retention_policy=retention_policy, total_size=total_size, used_size=used_size, path=paths, vo_list=vos) service.add_share(sa) - srm_endpoint = StorageEndpoint(name=sitename + "_srm", url=self._configuration.get_public_srm_endpoint(), type="srm", version="2.2", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level) + srm_endpoint_url = self._configuration.get_public_srm_endpoint() + srm_endpoint = StorageEndpoint(name="SRM_0", url=srm_endpoint_url, type="srm", version="2.2", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level) service.add_endpoint(srm_endpoint) #gsiftp_endpoint = StorageEndpoint(name=name + "_gridftp", url="", type="gsiftp", version="1.0.0", capabilities=['data.management.transfer'], quality_level=quality_level) #service.add_endpoint(gsiftp_endpoint) - if self._configuration.has_gridhttps(): - dav_http_endpoint = StorageEndpoint(name=sitename + "_http", url=self._configuration.get_public_http_endpoint(), type="DAV", version="1.1", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level) + if self._configuration.has_webdav(): + counter = {} + counter["HTTP"] = 0 + counter["HTTPS"] = 0 + for dav_endpoint_url in self._configuration.get_webdav_endpoints(): + protocol = urlparse(dav_endpoint_url).scheme.upper() + name = protocol + "_" + str(counter[protocol]) + se = StorageEndpoint(name=name, url=dav_endpoint_url, type="DAV", version="1.1", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level) + service.add_endpoint(se) + counter[protocol] += 1 + elif self._configuration.has_gridhttps(): + dav_http_endpoint = StorageEndpoint(name="HTTP_0", url=self._configuration.get_public_http_endpoint(), type="DAV", version="1.1", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level) service.add_endpoint(dav_http_endpoint) - dav_https_endpoint = StorageEndpoint(name=sitename + "_https", url=self._configuration.get_public_https_endpoint(), type="DAV", version="1.1", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level) + dav_https_endpoint = StorageEndpoint(name="HTTPS_0", url=self._configuration.get_public_https_endpoint(), type="DAV", version="1.1", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level) service.add_endpoint(dav_https_endpoint) return service diff --git a/src/resources/logging.ini b/src/resources/logging.ini index 3cab0fd..ef7d8ea 100644 --- a/src/resources/logging.ini +++ b/src/resources/logging.ini @@ -8,7 +8,7 @@ keys=consoleHandler keys=simpleFormatter [logger_root] -level=INFO +level=DEBUG handlers=consoleHandler [handler_consoleHandler] diff --git a/src/tests/resources/storm.def b/src/tests/resources/storm.def index 5cb510e..7f41013 100644 --- a/src/tests/resources/storm.def +++ b/src/tests/resources/storm.def @@ -1,53 +1,17 @@ +ISSUER_CA=INFN-CA-2016 MY_DOMAIN=cnaf.infn.it SITE_NAME=storm-testbed -STORM_ACLMODE=aot -STORM_ANONYMOUS_HTTP_READ=false -STORM_AUTH=permit-all -STORM_AUTHENTICATED_HTTP_READ=false STORM_BACKEND_HOST=centos6-devel.cnaf.infn.it STORM_BACKEND_REST_SERVICES_PORT=9998 -STORM_BDII_OVERWRITE=true -STORM_BE_XMLRPC_PATH=/RPC2 -STORM_BE_XMLRPC_PORT=8080 -STORM_BE_XMLRPC_TOKEN=secret -STORM_CERT_DIR=/etc/grid-security/storm -STORM_DB_HOST=centos6-devel.cnaf.infn.it -STORM_DB_PWD=secret -STORM_DB_USER=storm STORM_DEFAULT_ROOT=/storage -STORM_ENDPOINT_CAPABILITY=data.management.storage STORM_ENDPOINT_QUALITY_LEVEL=1 STORM_ENDPOINT_SERVING_STATE=4 -STORM_FE_ENABLE_MAPPING=false -STORM_FE_ENABLE_VOMSCHECK=false -STORM_FE_GSOAP_MAXPENDING=2000 -STORM_FE_LOG_FILE=/var/log/storm/storm-frontend.log -STORM_FE_LOG_LEVEL=INFO -STORM_FE_MONITORING_DETAILED=false -STORM_FE_MONITORING_ENABLED=true -STORM_FE_MONITORING_TIME_INTERVAL=60 -STORM_FE_THREADS_MAXPENDING=200 -STORM_FE_THREADS_NUMBER=50 -STORM_FE_USER_BLACKLISTING=false -STORM_FE_WSDL=/usr/share/wsdl/srm.v2.2.wsdl STORM_FRONTEND_HOST_LIST=centos6-devel.cnaf.infn.it,fe2.cnaf.infn.it -STORM_FRONTEND_OVERWRITE=true STORM_FRONTEND_PATH=/srm/managerv2 STORM_FRONTEND_PORT=8444 STORM_FRONTEND_PUBLIC_HOST=centos6-devel.cnaf.infn.it -STORM_FSTYPE=posixfs STORM_GRIDFTP_POOL_LIST=centos6-devel.cnaf.infn.it -STORM_GRIDFTP_POOL_STRATEGY=round-robin -STORM_GRIDHTTPS_CERT_DIR=/etc/grid-security/storm -STORM_GRIDHTTPS_ENABLED=true -STORM_GRIDHTTPS_HTTPS_PORT=8443 -STORM_GRIDHTTPS_HTTP_PORT=8085 -STORM_GRIDHTTPS_PUBLIC_HOST=centos6-devel.cnaf.infn.it -STORM_GRIDHTTPS_USER=storm -STORM_IGI_AUTHENTICATED_HTTP_READ=true -STORM_IGI_ONLINE_SIZE=4 -STORM_IGI_TOKEN=IGI_CUSTOM_TOKEN -STORM_IGI_VONAME=test.vo test.vo.2 +STORM_IMPLEMENTATION_VERSION=1.11.15 STORM_INFO_FILE_SUPPORT=true STORM_INFO_GRIDFTP_SUPPORT=true STORM_INFO_HTTPS_SUPPORT=true @@ -55,23 +19,23 @@ STORM_INFO_HTTP_SUPPORT=true STORM_INFO_OVERWRITE=true STORM_INFO_RFIO_SUPPORT=true STORM_INFO_ROOT_SUPPORT=true -STORM_INFO_USE_GPFS_PREALLOCATION=false -STORM_NAMESPACE_OVERWRITE=true +STORM_ROOT_HOST=centos6-devel.cnaf.infn.it +STORM_SERVING_STATE=production +STORM_SERVING_STATE_VALUE=1 +STORM_STORAGEAREA_LIST='test.vo test.vo.2 noauth igi nested tape test.vo.bis' +STORM_SURL_ENDPOINT_LIST=srm://centos6-devel.cnaf.infn.it:8444/srm/managerv2,srm://fe2.cnaf.infn.it:8444/srm/managerv2 +STORM_WEBDAV_POOL_LIST=http://centos6-devel.cnaf.infn.it:8085,https://centos6-devel.cnaf.infn.it:8443,https://another-host.cnaf.infn.it:8443 +VOS='test.vo test.vo.2' +STORM_IGI_AUTHENTICATED_HTTP_READ=true +STORM_IGI_ONLINE_SIZE=4 +STORM_IGI_TOKEN=IGI_CUSTOM_TOKEN +STORM_IGI_VONAME=test.vo test.vo.2 STORM_NESTED_ACCESSPOINT='/test.vo.2/nested /alias' STORM_NESTED_ONLINE_SIZE=4 STORM_NESTED_VONAME=test.vo.2 STORM_NOAUTH_ANONYMOUS_HTTP_READ=true STORM_NOAUTH_ONLINE_SIZE=4 STORM_NOAUTH_VONAME=test.vo -STORM_PEPC_RESOURCEID=storm -STORM_PROXY_HOME=/etc/storm/tmp -STORM_PTG_SKIP_ACL_SETUP=false -STORM_RFIO_HOST=centos6-devel.cnaf.infn.it -STORM_ROOT_HOST=centos6-devel.cnaf.infn.it -STORM_SERVICE_SURL_DEF_PORTS=8444 -STORM_SIZE_LIMIT=true -STORM_STORAGEAREA_LIST='test.vo test.vo.2 noauth igi nested tape test.vo.bis' -STORM_SURL_ENDPOINT_LIST=srm://centos6-devel.cnaf.infn.it:8444/srm/managerv2,srm://fe2.cnaf.infn.it:8444/srm/managerv2 STORM_TAPE_ACLMODE=aot STORM_TAPE_NEARLINE_SIZE=8 STORM_TAPE_ONLINE_SIZE=2 @@ -84,5 +48,3 @@ STORM_TESTVOBIS_ONLINE_SIZE=4 STORM_TESTVOBIS_VONAME=test.vo.2 STORM_TESTVO_DEFAULT_ACL_LIST=testvo:R STORM_TESTVO_ONLINE_SIZE=4 -STORM_USER=storm -VOS='test.vo test.vo.2' diff --git a/src/tests/resources/storm.gridhttps.def b/src/tests/resources/storm.gridhttps.def new file mode 100644 index 0000000..2c2fd7e --- /dev/null +++ b/src/tests/resources/storm.gridhttps.def @@ -0,0 +1,53 @@ +ISSUER_CA=INFN-CA-2016 +MY_DOMAIN=cnaf.infn.it +SITE_NAME=storm-testbed +STORM_BACKEND_HOST=centos6-devel.cnaf.infn.it +STORM_BACKEND_REST_SERVICES_PORT=9998 +STORM_DEFAULT_ROOT=/storage +STORM_ENDPOINT_QUALITY_LEVEL=1 +STORM_ENDPOINT_SERVING_STATE=4 +STORM_FRONTEND_HOST_LIST=centos6-devel.cnaf.infn.it,fe2.cnaf.infn.it +STORM_FRONTEND_PATH=/srm/managerv2 +STORM_FRONTEND_PORT=8444 +STORM_FRONTEND_PUBLIC_HOST=centos6-devel.cnaf.infn.it +STORM_GRIDFTP_POOL_LIST=centos6-devel.cnaf.infn.it +STORM_GRIDHTTPS_ENABLED=true +STORM_GRIDHTTPS_HTTPS_PORT=8443 +STORM_GRIDHTTPS_HTTP_PORT=8085 +STORM_GRIDHTTPS_PUBLIC_HOST=centos6-devel.cnaf.infn.it +STORM_IMPLEMENTATION_VERSION=1.11.15 +STORM_INFO_FILE_SUPPORT=true +STORM_INFO_GRIDFTP_SUPPORT=true +STORM_INFO_HTTPS_SUPPORT=true +STORM_INFO_HTTP_SUPPORT=true +STORM_INFO_OVERWRITE=true +STORM_INFO_RFIO_SUPPORT=true +STORM_INFO_ROOT_SUPPORT=true +STORM_ROOT_HOST=centos6-devel.cnaf.infn.it +STORM_SERVING_STATE=production +STORM_SERVING_STATE_VALUE=1 +STORM_STORAGEAREA_LIST='test.vo test.vo.2 noauth igi nested tape test.vo.bis' +STORM_SURL_ENDPOINT_LIST=srm://centos6-devel.cnaf.infn.it:8444/srm/managerv2,srm://fe2.cnaf.infn.it:8444/srm/managerv2 +VOS='test.vo test.vo.2' +STORM_IGI_AUTHENTICATED_HTTP_READ=true +STORM_IGI_ONLINE_SIZE=4 +STORM_IGI_TOKEN=IGI_CUSTOM_TOKEN +STORM_IGI_VONAME=test.vo test.vo.2 +STORM_NESTED_ACCESSPOINT='/test.vo.2/nested /alias' +STORM_NESTED_ONLINE_SIZE=4 +STORM_NESTED_VONAME=test.vo.2 +STORM_NOAUTH_ANONYMOUS_HTTP_READ=true +STORM_NOAUTH_ONLINE_SIZE=4 +STORM_NOAUTH_VONAME=test.vo +STORM_TAPE_ACLMODE=aot +STORM_TAPE_NEARLINE_SIZE=8 +STORM_TAPE_ONLINE_SIZE=2 +STORM_TAPE_RFIO_SUPPORT=false +STORM_TAPE_ROOT_SUPPORT=false +STORM_TAPE_STORAGECLASS=T1D0 +STORM_TAPE_VONAME=test.vo.2 +STORM_TESTVO2_ONLINE_SIZE=4 +STORM_TESTVOBIS_ONLINE_SIZE=4 +STORM_TESTVOBIS_VONAME=test.vo.2 +STORM_TESTVO_DEFAULT_ACL_LIST=testvo:R +STORM_TESTVO_ONLINE_SIZE=4 diff --git a/src/tests/test_configuration.py b/src/tests/test_configuration.py index df3efc3..4189a8b 100644 --- a/src/tests/test_configuration.py +++ b/src/tests/test_configuration.py @@ -31,7 +31,7 @@ def _check_is_test_configuration(self, configuration, filepath): self.assertEqual(configuration.get_serving_state(), "production") self.assertEqual(configuration.get_backend_hostname(), config["STORM_BACKEND_HOST"]) self.assertEqual(configuration.get_quality_level(), "testing") - self.assertEqual(configuration.get_implementation_version(), "1.11.13") + self.assertEqual(configuration.get_implementation_version(), "1.11.15") self.assertEqual(configuration.get_storage_area_list(), config["STORM_STORAGEAREA_LIST"].split(' ')) self.assertEqual(configuration.get_frontend_list(), config["STORM_FRONTEND_HOST_LIST"].split(',')) logging.debug("Check success") diff --git a/src/tests/test_storage_service.py b/src/tests/test_storage_service.py index 3bf6861..93a7335 100644 --- a/src/tests/test_storage_service.py +++ b/src/tests/test_storage_service.py @@ -6,7 +6,7 @@ from info_provider.storm_storage_service_builder import StorageServiceBuilder from tests.utils import get_default_test_configuration,\ get_default_space_info_summary_from_configuration,\ - get_default_storm_gateway + get_default_storm_gateway, get_test_configuration from info_provider.storm_gateway import StormGateway from tests.test_gateway import TestGateway from info_provider.storm_space_info_builder import SpaceInfoBuilder @@ -33,34 +33,43 @@ def test_storage_service_builder(self): self.assertNotEqual(service.get_capabilities().index("data.management.transfer"), -1) self.assertNotEqual(service.get_capabilities().index("data.management.storage"), -1) self.assertIsNotNone(service.get_latest_update()) - self.assertEqual(len(service.get_storage_endpoints()), 3) + self.assertEqual(len(service.get_storage_endpoints()), 4) # SRM self.assertEqual(service.get_storage_endpoints()[0].get_url(), configuration.get_public_srm_endpoint()) self.assertEqual(service.get_storage_endpoints()[0].get_type(), "srm") self.assertEqual(service.get_storage_endpoints()[0].get_version(), "2.2") - self.assertEqual(service.get_storage_endpoints()[0].get_name(), configuration.get_sitename() + "_srm") + self.assertEqual(service.get_storage_endpoints()[0].get_name(), "SRM_0") self.assertEqual(service.get_storage_endpoints()[0].get_quality_level(), configuration.get_quality_level()) self.assertNotEqual(service.get_storage_endpoints()[0].get_capabilities().index("data.management.transfer"), -1) self.assertNotEqual(service.get_storage_endpoints()[0].get_capabilities().index("data.management.storage"), -1) self.assertEqual(service.get_storage_endpoints()[0].get_assigned_shares(), ["all"]) - # HTTP - self.assertEqual(service.get_storage_endpoints()[1].get_url(), configuration.get_public_http_endpoint()) + # WebDAV - HTTP 8085 + self.assertEqual(service.get_storage_endpoints()[1].get_url(), configuration.get_webdav_endpoints()[0]) self.assertEqual(service.get_storage_endpoints()[1].get_type(), "DAV") self.assertEqual(service.get_storage_endpoints()[1].get_version(), "1.1") - self.assertEqual(service.get_storage_endpoints()[1].get_name(), configuration.get_sitename() + "_http") + self.assertEqual(service.get_storage_endpoints()[1].get_name(), "HTTP_0") self.assertEqual(service.get_storage_endpoints()[1].get_quality_level(), configuration.get_quality_level()) self.assertNotEqual(service.get_storage_endpoints()[1].get_capabilities().index("data.management.transfer"), -1) self.assertNotEqual(service.get_storage_endpoints()[1].get_capabilities().index("data.management.storage"), -1) self.assertEqual(service.get_storage_endpoints()[1].get_assigned_shares(), ["all"]) - # HTTPS - self.assertEqual(service.get_storage_endpoints()[2].get_url(), configuration.get_public_https_endpoint()) + # WebDAV - HTTPS 8443 + self.assertEqual(service.get_storage_endpoints()[2].get_url(), configuration.get_webdav_endpoints()[1]) self.assertEqual(service.get_storage_endpoints()[2].get_type(), "DAV") self.assertEqual(service.get_storage_endpoints()[2].get_version(), "1.1") - self.assertEqual(service.get_storage_endpoints()[2].get_name(), configuration.get_sitename() + "_https") + self.assertEqual(service.get_storage_endpoints()[2].get_name(), "HTTPS_0") self.assertEqual(service.get_storage_endpoints()[2].get_quality_level(), configuration.get_quality_level()) self.assertNotEqual(service.get_storage_endpoints()[2].get_capabilities().index("data.management.transfer"), -1) self.assertNotEqual(service.get_storage_endpoints()[2].get_capabilities().index("data.management.storage"), -1) self.assertEqual(service.get_storage_endpoints()[2].get_assigned_shares(), ["all"]) + # WebDAV - HTTPS another-host 8443 + self.assertEqual(service.get_storage_endpoints()[3].get_url(), configuration.get_webdav_endpoints()[2]) + self.assertEqual(service.get_storage_endpoints()[3].get_type(), "DAV") + self.assertEqual(service.get_storage_endpoints()[3].get_version(), "1.1") + self.assertEqual(service.get_storage_endpoints()[3].get_name(), "HTTPS_1") + self.assertEqual(service.get_storage_endpoints()[3].get_quality_level(), configuration.get_quality_level()) + self.assertNotEqual(service.get_storage_endpoints()[3].get_capabilities().index("data.management.transfer"), -1) + self.assertNotEqual(service.get_storage_endpoints()[3].get_capabilities().index("data.management.storage"), -1) + self.assertEqual(service.get_storage_endpoints()[3].get_assigned_shares(), ["all"]) # check storage shares self.assertEqual(len(service.get_storage_shares()), 7) # check it's a valid JSON string @@ -84,5 +93,54 @@ def test_storage_service_builder_when_remote_gateway_has_http_error(self, mock_u self.assertEqual(spaceinfo.get_summary().get_busy(), expected_summary.get_busy()) self.assertEqual(spaceinfo.get_summary().get_nearline(), expected_summary.get_nearline()) + def test_storage_service_builder_with_gridhttps_env_vars(self): + configuration = get_test_configuration("resources/storm.gridhttps.def") + storm_gateway = get_default_storm_gateway() + # build space info + space_info = SpaceInfoBuilder(configuration, storm_gateway).build() + # build storage service and check results + service = StorageServiceBuilder(configuration, space_info).build() + self.assertEqual(service.get_name(), configuration.get_sitename()) + self.assertEqual(service.get_implementation(), "storm") + self.assertEqual(service.get_implementation_version(), configuration.get_implementation_version()) + self.assertEqual(service.get_quality_level(), configuration.get_quality_level()) + self.assertNotEqual(service.get_capabilities().index("data.management.transfer"), -1) + self.assertNotEqual(service.get_capabilities().index("data.management.storage"), -1) + self.assertIsNotNone(service.get_latest_update()) + self.assertEqual(len(service.get_storage_endpoints()), 3) + # SRM + self.assertEqual(service.get_storage_endpoints()[0].get_url(), configuration.get_public_srm_endpoint()) + self.assertEqual(service.get_storage_endpoints()[0].get_type(), "srm") + self.assertEqual(service.get_storage_endpoints()[0].get_version(), "2.2") + self.assertEqual(service.get_storage_endpoints()[0].get_name(), "SRM_0") + self.assertEqual(service.get_storage_endpoints()[0].get_quality_level(), configuration.get_quality_level()) + self.assertNotEqual(service.get_storage_endpoints()[0].get_capabilities().index("data.management.transfer"), -1) + self.assertNotEqual(service.get_storage_endpoints()[0].get_capabilities().index("data.management.storage"), -1) + self.assertEqual(service.get_storage_endpoints()[0].get_assigned_shares(), ["all"]) + # HTTP + self.assertEqual(service.get_storage_endpoints()[1].get_url(), configuration.get_public_http_endpoint()) + self.assertEqual(service.get_storage_endpoints()[1].get_type(), "DAV") + self.assertEqual(service.get_storage_endpoints()[1].get_version(), "1.1") + self.assertEqual(service.get_storage_endpoints()[1].get_name(), "HTTP_0") + self.assertEqual(service.get_storage_endpoints()[1].get_quality_level(), configuration.get_quality_level()) + self.assertNotEqual(service.get_storage_endpoints()[1].get_capabilities().index("data.management.transfer"), -1) + self.assertNotEqual(service.get_storage_endpoints()[1].get_capabilities().index("data.management.storage"), -1) + self.assertEqual(service.get_storage_endpoints()[1].get_assigned_shares(), ["all"]) + # HTTPS + self.assertEqual(service.get_storage_endpoints()[2].get_url(), configuration.get_public_https_endpoint()) + self.assertEqual(service.get_storage_endpoints()[2].get_type(), "DAV") + self.assertEqual(service.get_storage_endpoints()[2].get_version(), "1.1") + self.assertEqual(service.get_storage_endpoints()[2].get_name(), "HTTPS_0") + self.assertEqual(service.get_storage_endpoints()[2].get_quality_level(), configuration.get_quality_level()) + self.assertNotEqual(service.get_storage_endpoints()[2].get_capabilities().index("data.management.transfer"), -1) + self.assertNotEqual(service.get_storage_endpoints()[2].get_capabilities().index("data.management.storage"), -1) + self.assertEqual(service.get_storage_endpoints()[2].get_assigned_shares(), ["all"]) + # check storage shares + self.assertEqual(len(service.get_storage_shares()), 7) + # check it's a valid JSON string + report = Report(storage_service=service) + json.loads(report.to_json()) + logging.debug(report.to_json()) + if __name__ == '__main__': unittest.main() diff --git a/src/tests/utils.py b/src/tests/utils.py index 2099766..db0ce7c 100644 --- a/src/tests/utils.py +++ b/src/tests/utils.py @@ -7,20 +7,20 @@ from info_provider.model.space import SpaceRecord from info_provider.storm_gateway import StormGateway +def get_test_configuration_filepath(filepath): + return os.path.join(os.path.dirname(__file__), filepath) def get_default_test_configuration_filepath(): - return os.path.join(os.path.dirname(__file__), "resources/storm.def") + return get_test_configuration_filepath("resources/storm.def") def get_incomplete_test_configuration_filepath(): - return os.path.join(os.path.dirname(__file__), "resources/storm-incomplete.def") + return get_test_configuration_filepath("resources/storm-incomplete.def") def get_default_test_configuration(): - configuration = Configuration(get_default_test_configuration_filepath()) - configuration.set("STORM_SERVING_STATE_VALUE", 1) - configuration.set("STORM_SERVING_STATE", "production") - configuration.set("STORM_IMPLEMENTATION_VERSION", "1.11.13") - configuration.set("ISSUER_CA", "INFN-CA-2016") - return configuration + return Configuration(get_default_test_configuration_filepath()) + +def get_test_configuration(filepath): + return Configuration(get_test_configuration_filepath(filepath)) def get_default_storm_gateway(): configuration = get_default_test_configuration() @@ -85,4 +85,4 @@ def get_default_space_info_summary_from_configuration(): "reserved": 0, "busy": 0, "near_line": 8000000000 - }) \ No newline at end of file + }) From 684a50b57294161960612a52ef693a08739bead3 Mon Sep 17 00:00:00 2001 From: enricovianello Date: Thu, 8 Nov 2018 17:50:25 +0100 Subject: [PATCH 2/4] Fixed error in case webdav pool list is empty --- src/info_provider/configuration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/info_provider/configuration.py b/src/info_provider/configuration.py index 6c7f1c6..2a42a3e 100644 --- a/src/info_provider/configuration.py +++ b/src/info_provider/configuration.py @@ -97,7 +97,7 @@ def is_HTTP_endpoint_enabled(self): def has_webdav(self): if 'STORM_WEBDAV_POOL_LIST' in self._configuration: - if self.get_webdav_endpoints(): + if len(self.get_webdav_endpoints()) > 0: return True return False @@ -118,8 +118,8 @@ def get_public_http_endpoint(self): return "http://" + host + ":" + port + "/" def get_webdav_endpoints(self): - endpoints = self.get("STORM_WEBDAV_POOL_LIST").split(',') - logging.debug(endpoints) + endpoints = filter(None, self.get("STORM_WEBDAV_POOL_LIST").split(',')) + logging.debug("webdav endpoints: " + str(endpoints)) return endpoints def is_info_overwrite(self): From 89b20d8c1fe3cad2cce04f19170a11fcce808426 Mon Sep 17 00:00:00 2001 From: enricovianello Date: Wed, 14 Nov 2018 15:07:54 +0100 Subject: [PATCH 3/4] Cosmetic fix - added debug message --- src/info_provider/glue/glue2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/info_provider/glue/glue2.py b/src/info_provider/glue/glue2.py index d2108c6..54af4c8 100644 --- a/src/info_provider/glue/glue2.py +++ b/src/info_provider/glue/glue2.py @@ -329,6 +329,7 @@ def get_static_ldif_nodes(self, spaceinfo): # NEW LOGIC if self._configuration.has_webdav(): + logging.debug("new logic webdav endpoints") i = 0; for endpoint in self._configuration.get_webdav_endpoints(): protocol = urlparse(endpoint).scheme.upper() From 4987c2f5c3e9fe7a9e093e7ffb3361765bae6300 Mon Sep 17 00:00:00 2001 From: enricovianello Date: Thu, 29 Nov 2018 11:15:57 +0100 Subject: [PATCH 4/4] Removed unused STORM_INFO_OVERWRITE stuff --- src/info_provider/configuration.py | 3 --- src/tests/resources/storm.def | 1 - src/tests/resources/storm.gridhttps.def | 1 - 3 files changed, 5 deletions(-) diff --git a/src/info_provider/configuration.py b/src/info_provider/configuration.py index 2a42a3e..2084d0f 100644 --- a/src/info_provider/configuration.py +++ b/src/info_provider/configuration.py @@ -122,9 +122,6 @@ def get_webdav_endpoints(self): logging.debug("webdav endpoints: " + str(endpoints)) return endpoints - def is_info_overwrite(self): - return self.get("STORM_INFO_OVERWRITE").lower() == "true" - def vfs_has_custom_token(self, vfs_name): return "STORM_" + vfs_name[:-3] + "_TOKEN" in self._configuration diff --git a/src/tests/resources/storm.def b/src/tests/resources/storm.def index 7f41013..c256b1e 100644 --- a/src/tests/resources/storm.def +++ b/src/tests/resources/storm.def @@ -16,7 +16,6 @@ STORM_INFO_FILE_SUPPORT=true STORM_INFO_GRIDFTP_SUPPORT=true STORM_INFO_HTTPS_SUPPORT=true STORM_INFO_HTTP_SUPPORT=true -STORM_INFO_OVERWRITE=true STORM_INFO_RFIO_SUPPORT=true STORM_INFO_ROOT_SUPPORT=true STORM_ROOT_HOST=centos6-devel.cnaf.infn.it diff --git a/src/tests/resources/storm.gridhttps.def b/src/tests/resources/storm.gridhttps.def index 2c2fd7e..990f6d0 100644 --- a/src/tests/resources/storm.gridhttps.def +++ b/src/tests/resources/storm.gridhttps.def @@ -20,7 +20,6 @@ STORM_INFO_FILE_SUPPORT=true STORM_INFO_GRIDFTP_SUPPORT=true STORM_INFO_HTTPS_SUPPORT=true STORM_INFO_HTTP_SUPPORT=true -STORM_INFO_OVERWRITE=true STORM_INFO_RFIO_SUPPORT=true STORM_INFO_ROOT_SUPPORT=true STORM_ROOT_HOST=centos6-devel.cnaf.infn.it