Merge pull request #10 from italiangrid/fix/STOR-1020

release 1.8.1

src/info_provider/configuration.py

@@ -13,8 +13,7 @@ class Configuration:
     _mandatories = ["SITE_NAME", "STORM_BACKEND_HOST", "STORM_DEFAULT_ROOT",
         "STORM_FRONTEND_PATH", "STORM_FRONTEND_PORT",
         "STORM_FRONTEND_PUBLIC_HOST", "STORM_BACKEND_REST_SERVICES_PORT",
-        "VOS", "STORM_ENDPOINT_QUALITY_LEVEL", "STORM_ENDPOINT_CAPABILITY",
-        "STORM_GRIDHTTPS_PUBLIC_HOST"]
+        "VOS", "STORM_ENDPOINT_QUALITY_LEVEL"]
 
     def __init__(self, filepath):
         logging.debug("Init configuration from file %s ...", filepath)
@@ -73,7 +72,7 @@ def get_enabled_access_protocols(self):
             enabled.append("http")
         if self.get("STORM_INFO_HTTPS_SUPPORT").lower() == "true":
             enabled.append("https")
-        if self.has_gridhttps():
+        if self.has_webdav():
             enabled.append("webdav")
         return enabled
 
@@ -86,14 +85,22 @@ def get_frontend_list(self):
         return self.get("STORM_FRONTEND_HOST_LIST").split(',')
 
     def has_gridhttps(self):
-        return self.get('STORM_GRIDHTTPS_ENABLED').lower() == "true"
+        if 'STORM_GRIDHTTPS_ENABLED' in self._configuration:
+            return self.get('STORM_GRIDHTTPS_ENABLED').lower() == "true"
+        return False
 
     def get_gridhttps_list(self):
         return self.get("STORM_GRIDHTTPS_POOL_LIST").split(',')
 
     def is_HTTP_endpoint_enabled(self):
         return self.get('STORM_GRIDHTTPS_HTTP_ENABLED').lower() == "true"
 
+    def has_webdav(self):
+        if 'STORM_WEBDAV_POOL_LIST' in self._configuration:
+            if len(self.get_webdav_endpoints()) > 0:
+                return True
+        return False
+
     def get_public_srm_endpoint(self):
         host = self.get("STORM_FRONTEND_PUBLIC_HOST")
         port = str(self.get("STORM_FRONTEND_PORT"))
@@ -110,8 +117,10 @@ def get_public_http_endpoint(self):
         port = str(self.get("STORM_GRIDHTTPS_HTTP_PORT"))
         return "http://" + host + ":" + port + "/"
 
-    def is_info_overwrite(self):
-        return self.get("STORM_INFO_OVERWRITE").lower() == "true"
+    def get_webdav_endpoints(self):
+        endpoints = filter(None, self.get("STORM_WEBDAV_POOL_LIST").split(','))
+        logging.debug("webdav endpoints: " + str(endpoints))
+        return endpoints
 
     def vfs_has_custom_token(self, vfs_name):
         return "STORM_" + vfs_name[:-3] + "_TOKEN" in self._configuration

src/info_provider/storm_storage_service_builder.py

@@ -1,5 +1,7 @@
 import logging
 
+from urlparse import urlparse
+
 from info_provider.model.storage import StorageShare, StorageEndpoint, \
     StorageService
 
@@ -28,16 +30,27 @@ def build(self):
             sa = StorageShare(name=token, access_latency=access_latency, retention_policy=retention_policy, total_size=total_size, used_size=used_size, path=paths, vo_list=vos)
             service.add_share(sa)
 
-        srm_endpoint = StorageEndpoint(name=sitename + "_srm", url=self._configuration.get_public_srm_endpoint(), type="srm", version="2.2", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level)
+        srm_endpoint_url = self._configuration.get_public_srm_endpoint()
+        srm_endpoint = StorageEndpoint(name="SRM_0", url=srm_endpoint_url, type="srm", version="2.2", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level)
         service.add_endpoint(srm_endpoint)
 
         #gsiftp_endpoint = StorageEndpoint(name=name + "_gridftp", url="", type="gsiftp", version="1.0.0", capabilities=['data.management.transfer'], quality_level=quality_level)
         #service.add_endpoint(gsiftp_endpoint)
 
-        if self._configuration.has_gridhttps():
-            dav_http_endpoint = StorageEndpoint(name=sitename + "_http", url=self._configuration.get_public_http_endpoint(), type="DAV", version="1.1", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level)
+        if self._configuration.has_webdav():
+            counter = {}
+            counter["HTTP"] = 0
+            counter["HTTPS"] = 0
+            for dav_endpoint_url in self._configuration.get_webdav_endpoints():
+                protocol = urlparse(dav_endpoint_url).scheme.upper()
+                name = protocol + "_" + str(counter[protocol])
+                se = StorageEndpoint(name=name, url=dav_endpoint_url, type="DAV", version="1.1", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level)
+                service.add_endpoint(se)
+                counter[protocol] += 1
+        elif self._configuration.has_gridhttps():
+            dav_http_endpoint = StorageEndpoint(name="HTTP_0", url=self._configuration.get_public_http_endpoint(), type="DAV", version="1.1", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level)
             service.add_endpoint(dav_http_endpoint)
-            dav_https_endpoint = StorageEndpoint(name=sitename + "_https", url=self._configuration.get_public_https_endpoint(), type="DAV", version="1.1", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level)
+            dav_https_endpoint = StorageEndpoint(name="HTTPS_0", url=self._configuration.get_public_https_endpoint(), type="DAV", version="1.1", capabilities=['data.management.transfer', 'data.management.storage'], quality_level=quality_level)
             service.add_endpoint(dav_https_endpoint)
 
         return service

src/resources/logging.ini

@@ -8,7 +8,7 @@ keys=consoleHandler
 keys=simpleFormatter
 
 [logger_root]
-level=INFO
+level=DEBUG
 handlers=consoleHandler
 
 [handler_consoleHandler]

src/tests/resources/storm.def

@@ -1,77 +1,40 @@
+ISSUER_CA=INFN-CA-2016
 MY_DOMAIN=cnaf.infn.it
 SITE_NAME=storm-testbed
-STORM_ACLMODE=aot
-STORM_ANONYMOUS_HTTP_READ=false
-STORM_AUTH=permit-all
-STORM_AUTHENTICATED_HTTP_READ=false
 STORM_BACKEND_HOST=centos6-devel.cnaf.infn.it
 STORM_BACKEND_REST_SERVICES_PORT=9998
-STORM_BDII_OVERWRITE=true
-STORM_BE_XMLRPC_PATH=/RPC2
-STORM_BE_XMLRPC_PORT=8080
-STORM_BE_XMLRPC_TOKEN=secret
-STORM_CERT_DIR=/etc/grid-security/storm
-STORM_DB_HOST=centos6-devel.cnaf.infn.it
-STORM_DB_PWD=secret
-STORM_DB_USER=storm
 STORM_DEFAULT_ROOT=/storage
-STORM_ENDPOINT_CAPABILITY=data.management.storage
 STORM_ENDPOINT_QUALITY_LEVEL=1
 STORM_ENDPOINT_SERVING_STATE=4
-STORM_FE_ENABLE_MAPPING=false
-STORM_FE_ENABLE_VOMSCHECK=false
-STORM_FE_GSOAP_MAXPENDING=2000
-STORM_FE_LOG_FILE=/var/log/storm/storm-frontend.log
-STORM_FE_LOG_LEVEL=INFO
-STORM_FE_MONITORING_DETAILED=false
-STORM_FE_MONITORING_ENABLED=true
-STORM_FE_MONITORING_TIME_INTERVAL=60
-STORM_FE_THREADS_MAXPENDING=200
-STORM_FE_THREADS_NUMBER=50
-STORM_FE_USER_BLACKLISTING=false
-STORM_FE_WSDL=/usr/share/wsdl/srm.v2.2.wsdl
 STORM_FRONTEND_HOST_LIST=centos6-devel.cnaf.infn.it,fe2.cnaf.infn.it
-STORM_FRONTEND_OVERWRITE=true
 STORM_FRONTEND_PATH=/srm/managerv2
 STORM_FRONTEND_PORT=8444
 STORM_FRONTEND_PUBLIC_HOST=centos6-devel.cnaf.infn.it
-STORM_FSTYPE=posixfs
 STORM_GRIDFTP_POOL_LIST=centos6-devel.cnaf.infn.it
-STORM_GRIDFTP_POOL_STRATEGY=round-robin
-STORM_GRIDHTTPS_CERT_DIR=/etc/grid-security/storm
-STORM_GRIDHTTPS_ENABLED=true
-STORM_GRIDHTTPS_HTTPS_PORT=8443
-STORM_GRIDHTTPS_HTTP_PORT=8085
-STORM_GRIDHTTPS_PUBLIC_HOST=centos6-devel.cnaf.infn.it
-STORM_GRIDHTTPS_USER=storm
-STORM_IGI_AUTHENTICATED_HTTP_READ=true
-STORM_IGI_ONLINE_SIZE=4
-STORM_IGI_TOKEN=IGI_CUSTOM_TOKEN
-STORM_IGI_VONAME=test.vo test.vo.2
+STORM_IMPLEMENTATION_VERSION=1.11.15
 STORM_INFO_FILE_SUPPORT=true
 STORM_INFO_GRIDFTP_SUPPORT=true
 STORM_INFO_HTTPS_SUPPORT=true
 STORM_INFO_HTTP_SUPPORT=true
-STORM_INFO_OVERWRITE=true
 STORM_INFO_RFIO_SUPPORT=true
 STORM_INFO_ROOT_SUPPORT=true
-STORM_INFO_USE_GPFS_PREALLOCATION=false
-STORM_NAMESPACE_OVERWRITE=true
+STORM_ROOT_HOST=centos6-devel.cnaf.infn.it
+STORM_SERVING_STATE=production
+STORM_SERVING_STATE_VALUE=1
+STORM_STORAGEAREA_LIST='test.vo test.vo.2 noauth igi nested tape test.vo.bis'
+STORM_SURL_ENDPOINT_LIST=srm://centos6-devel.cnaf.infn.it:8444/srm/managerv2,srm://fe2.cnaf.infn.it:8444/srm/managerv2
+STORM_WEBDAV_POOL_LIST=http://centos6-devel.cnaf.infn.it:8085,https://centos6-devel.cnaf.infn.it:8443,https://another-host.cnaf.infn.it:8443
+VOS='test.vo test.vo.2'
+STORM_IGI_AUTHENTICATED_HTTP_READ=true
+STORM_IGI_ONLINE_SIZE=4
+STORM_IGI_TOKEN=IGI_CUSTOM_TOKEN
+STORM_IGI_VONAME=test.vo test.vo.2
 STORM_NESTED_ACCESSPOINT='/test.vo.2/nested /alias'
 STORM_NESTED_ONLINE_SIZE=4
 STORM_NESTED_VONAME=test.vo.2
 STORM_NOAUTH_ANONYMOUS_HTTP_READ=true
 STORM_NOAUTH_ONLINE_SIZE=4
 STORM_NOAUTH_VONAME=test.vo
-STORM_PEPC_RESOURCEID=storm
-STORM_PROXY_HOME=/etc/storm/tmp
-STORM_PTG_SKIP_ACL_SETUP=false
-STORM_RFIO_HOST=centos6-devel.cnaf.infn.it
-STORM_ROOT_HOST=centos6-devel.cnaf.infn.it
-STORM_SERVICE_SURL_DEF_PORTS=8444
-STORM_SIZE_LIMIT=true
-STORM_STORAGEAREA_LIST='test.vo test.vo.2 noauth igi nested tape test.vo.bis'
-STORM_SURL_ENDPOINT_LIST=srm://centos6-devel.cnaf.infn.it:8444/srm/managerv2,srm://fe2.cnaf.infn.it:8444/srm/managerv2
 STORM_TAPE_ACLMODE=aot
 STORM_TAPE_NEARLINE_SIZE=8
 STORM_TAPE_ONLINE_SIZE=2
@@ -84,5 +47,3 @@ STORM_TESTVOBIS_ONLINE_SIZE=4
 STORM_TESTVOBIS_VONAME=test.vo.2
 STORM_TESTVO_DEFAULT_ACL_LIST=testvo:R
 STORM_TESTVO_ONLINE_SIZE=4
-STORM_USER=storm
-VOS='test.vo test.vo.2'

src/tests/resources/storm.gridhttps.def

@@ -0,0 +1,52 @@
+ISSUER_CA=INFN-CA-2016
+MY_DOMAIN=cnaf.infn.it
+SITE_NAME=storm-testbed
+STORM_BACKEND_HOST=centos6-devel.cnaf.infn.it
+STORM_BACKEND_REST_SERVICES_PORT=9998
+STORM_DEFAULT_ROOT=/storage
+STORM_ENDPOINT_QUALITY_LEVEL=1
+STORM_ENDPOINT_SERVING_STATE=4
+STORM_FRONTEND_HOST_LIST=centos6-devel.cnaf.infn.it,fe2.cnaf.infn.it
+STORM_FRONTEND_PATH=/srm/managerv2
+STORM_FRONTEND_PORT=8444
+STORM_FRONTEND_PUBLIC_HOST=centos6-devel.cnaf.infn.it
+STORM_GRIDFTP_POOL_LIST=centos6-devel.cnaf.infn.it
+STORM_GRIDHTTPS_ENABLED=true
+STORM_GRIDHTTPS_HTTPS_PORT=8443
+STORM_GRIDHTTPS_HTTP_PORT=8085
+STORM_GRIDHTTPS_PUBLIC_HOST=centos6-devel.cnaf.infn.it
+STORM_IMPLEMENTATION_VERSION=1.11.15
+STORM_INFO_FILE_SUPPORT=true
+STORM_INFO_GRIDFTP_SUPPORT=true
+STORM_INFO_HTTPS_SUPPORT=true
+STORM_INFO_HTTP_SUPPORT=true
+STORM_INFO_RFIO_SUPPORT=true
+STORM_INFO_ROOT_SUPPORT=true
+STORM_ROOT_HOST=centos6-devel.cnaf.infn.it
+STORM_SERVING_STATE=production
+STORM_SERVING_STATE_VALUE=1
+STORM_STORAGEAREA_LIST='test.vo test.vo.2 noauth igi nested tape test.vo.bis'
+STORM_SURL_ENDPOINT_LIST=srm://centos6-devel.cnaf.infn.it:8444/srm/managerv2,srm://fe2.cnaf.infn.it:8444/srm/managerv2
+VOS='test.vo test.vo.2'
+STORM_IGI_AUTHENTICATED_HTTP_READ=true
+STORM_IGI_ONLINE_SIZE=4
+STORM_IGI_TOKEN=IGI_CUSTOM_TOKEN
+STORM_IGI_VONAME=test.vo test.vo.2
+STORM_NESTED_ACCESSPOINT='/test.vo.2/nested /alias'
+STORM_NESTED_ONLINE_SIZE=4
+STORM_NESTED_VONAME=test.vo.2
+STORM_NOAUTH_ANONYMOUS_HTTP_READ=true
+STORM_NOAUTH_ONLINE_SIZE=4
+STORM_NOAUTH_VONAME=test.vo
+STORM_TAPE_ACLMODE=aot
+STORM_TAPE_NEARLINE_SIZE=8
+STORM_TAPE_ONLINE_SIZE=2
+STORM_TAPE_RFIO_SUPPORT=false
+STORM_TAPE_ROOT_SUPPORT=false
+STORM_TAPE_STORAGECLASS=T1D0
+STORM_TAPE_VONAME=test.vo.2
+STORM_TESTVO2_ONLINE_SIZE=4
+STORM_TESTVOBIS_ONLINE_SIZE=4
+STORM_TESTVOBIS_VONAME=test.vo.2
+STORM_TESTVO_DEFAULT_ACL_LIST=testvo:R
+STORM_TESTVO_ONLINE_SIZE=4

src/tests/test_configuration.py

@@ -31,7 +31,7 @@ def _check_is_test_configuration(self, configuration, filepath):
         self.assertEqual(configuration.get_serving_state(), "production")
         self.assertEqual(configuration.get_backend_hostname(), config["STORM_BACKEND_HOST"])
         self.assertEqual(configuration.get_quality_level(), "testing")
-        self.assertEqual(configuration.get_implementation_version(), "1.11.13")
+        self.assertEqual(configuration.get_implementation_version(), "1.11.15")
         self.assertEqual(configuration.get_storage_area_list(), config["STORM_STORAGEAREA_LIST"].split(' '))
         self.assertEqual(configuration.get_frontend_list(), config["STORM_FRONTEND_HOST_LIST"].split(','))
         logging.debug("Check success")