Merge pull request #90 from eth-cscs/dev

PR for version 1.7.2

deploy/docker/storage/Dockerfile

@@ -11,6 +11,8 @@ RUN pip3 install -r deps/requirements.txt
 
 ADD src/storage/storage.py storage.py
 ADD src/storage/keystone.py keystone.py
+ADD src/storage/keystoneoidc.py keystoneoidc.py
+ADD src/storage/keystonesaml.py keystonesaml.py
 ADD src/storage/objectstorage.py objectstorage.py
 ADD src/storage/s3v2OS.py s3v2OS.py
 ADD src/storage/s3v4OS.py s3v4OS.py

doc/openapi/firecrest-api.yaml

@@ -9,7 +9,7 @@ servers:
   - url: 'http://FIRECREST_URL'
   - url: 'https://FIRECREST_URL'
 info:
-  version: 1.7.1-beta4
+  version: 1.7.2-beta1
   title: FirecREST Developers API
   description: >
     This API specification is intended for FirecREST developers only. There're some endpoints that are not available in the public version for client developers.

doc/openapi/firecrest-developers-api.yaml

@@ -9,7 +9,7 @@ servers:
   - url: 'http://FIRECREST_URL'
   - url: 'https://FIRECREST_URL'
 info:
-  version: 1.7.1-beta4
+  version: 1.7.2-beta1
   title: FirecREST API
   description: >
     FirecREST platform, a RESTful Services Gateway to HPC resources, is a

src/storage/keystone.py

@@ -4,88 +4,25 @@
 #  Please, refer to the LICENSE file in the root directory.
 #  SPDX-License-Identifier: BSD-3-Clause
 #
-from keystoneauth1.identity import v3
-from keystoneauth1 import session as keystonesession
-from keystoneauth1 import exceptions as keystoneexception
-from keystoneauth1.extras._saml2 import V3Saml2Password
+from abc import ABCMeta,abstractmethod
 
-import logging
-import requests
-import os
 
+class Keystone:
+    __metaclass__ = ABCMeta
 
-OS_AUTH_URL             = os.environ.get("F7T_OS_AUTH_URL")
-OS_IDENTITY_PROVIDER    = os.environ.get("F7T_OS_IDENTITY_PROVIDER")
-OS_IDENTITY_PROVIDER_URL= os.environ.get("F7T_OS_IDENTITY_PROVIDER_URL")
-OS_PROTOCOL             = os.environ.get("F7T_OS_PROTOCOL")
-OS_INTERFACE            = os.environ.get("F7T_OS_INTERFACE")
-OS_PROJECT_ID           = os.environ.get("F7T_OS_PROJECT_ID")
+    # default constructor
+    @abstractmethod
+    def __init__(self,url):
+        pass
 
+    # returns a valid token if username & password are valid keystone credentials
+    @abstractmethod
+    def authenticate(username,password):
+        pass
 
-logging.basicConfig(level=logging.INFO)
-log = logging.getLogger(__name__)
+    # Checks if token is valid directly with keystone API
+    @abstractmethod
+    def is_token_valid(token):
+        pass
 
-# returns a valid token if username & password are valid keystone credentials
-def authenticate(username,password):
 
-    try:
-
-        auth = V3Saml2Password(auth_url=OS_AUTH_URL, identity_provider=OS_IDENTITY_PROVIDER, protocol=OS_PROTOCOL,
-                               identity_provider_url=OS_IDENTITY_PROVIDER_URL, username=username, password=password)
-
-
-
-        sess = keystonesession.Session(auth=auth)
-        try:
-
-            log.info(sess.get_token())
-        except AttributeError as e:
-            log.info(e)
-            log.info(e.args)
-
-
-        auth = v3.token.Token(auth_url=OS_AUTH_URL, token=sess.get_token(), project_id=OS_PROJECT_ID)
-
-        sess = keystonesession.Session(auth=auth)
-
-        OS_TOKEN = sess.get_token()
-
-        return {"error":0,"OS_TOKEN":OS_TOKEN}
-
-    except keystoneexception.http.BadRequest as e:
-        log.error(e)
-        log.error(e.message)
-        log.error(e.details)
-        return {"error":1,"msg":e.message}
-
-
-    except Exception as e:
-
-        log.error(type(e))
-        return {"error":1,"msg":e}
-
-
-# Checks if token is valid directly with keystone API
-def is_token_valid(token):
-
-    url = "{os_auth_url}/auth/tokens".format(os_auth_url=OS_AUTH_URL)
-
-    headers = {"X-Auth-Token":token,
-               "X-Subject-Token":token}
-
-    try:
-
-        r = requests.get(url=url,headers=headers)
-
-        if r.status_code == 200:
-            logging.info("Valid token")
-            return True
-
-        logging.warning("Invalid token")
-
-        return False
-
-    except requests.exceptions.RequestException as re:
-        logging.error(re)
-        logging.error("Invalid token request")
-        return False

src/storage/keystoneoidc.py

@@ -0,0 +1,97 @@
+#
+#  Copyright (c) 2019-2021, ETH Zurich. All rights reserved.
+#
+#  Please, refer to the LICENSE file in the root directory.
+#  SPDX-License-Identifier: BSD-3-Clause
+#
+from keystoneauth1.identity import v3
+from keystoneauth1 import session as keystonesession
+from keystoneauth1 import exceptions as keystoneexception
+from keystoneauth1.identity import V3OidcPassword
+
+import logging
+import requests
+import os
+from keystone import Keystone
+
+
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger(__name__)
+
+class KeystoneOIDC(Keystone):
+
+    def __init__(self):
+        self.OS_AUTH_URL             = os.environ.get("F7T_OS_AUTH_URL")
+        self.OS_IDENTITY_PROVIDER    = os.environ.get("F7T_OS_IDENTITY_PROVIDER")
+        self.OS_PROTOCOL             = os.environ.get("F7T_OS_PROTOCOL")
+        self.OS_INTERFACE            = os.environ.get("F7T_OS_INTERFACE")
+        self.OS_PROJECT_ID           = os.environ.get("F7T_OS_PROJECT_ID")
+        self.OS_CLIENT_ID            = os.environ.get("F7T_OS_CLIENT_ID")
+        self.OS_CLIENT_SECRET        = os.environ.get("F7T_OS_CLIENT_SECRET")
+        self.OS_DISCOVERY_ENDPOINT   = os.environ.get("F7T_OS_DISCOVERY_ENDPOINT")
+
+    # returns a valid token if username & password are valid keystone credentials
+    def authenticate(self,username,password):
+
+        try:
+
+            auth = V3OidcPassword(auth_url=self.OS_AUTH_URL, identity_provider=self.OS_IDENTITY_PROVIDER, protocol=self.OS_PROTOCOL,
+                                client_id=self.OS_CLIENT_ID, client_secret=self.OS_CLIENT_SECRET, discovery_endpoint=self.OS_DISCOVERY_ENDPOINT, 
+                                username=username, password=password)
+
+
+
+            sess = keystonesession.Session(auth=auth)
+            try:
+
+                log.info(sess.get_token())
+            except AttributeError as e:
+                log.info(e)
+                log.info(e.args)
+
+
+            auth = v3.token.Token(auth_url=self.OS_AUTH_URL, token=sess.get_token(), project_id=self.OS_PROJECT_ID)
+
+            sess = keystonesession.Session(auth=auth)
+
+            OS_TOKEN = sess.get_token()
+
+            return {"error":0,"OS_TOKEN":OS_TOKEN}
+
+        except keystoneexception.http.BadRequest as e:
+            log.error(e)
+            log.error(e.message)
+            log.error(e.details)
+            return {"error":1,"msg":e.message}
+
+
+        except Exception as e:
+
+            log.error(type(e))
+            return {"error":1,"msg":e}
+
+
+    # Checks if token is valid directly with keystone API
+    def is_token_valid(self,token):
+
+        url = f"{self.OS_AUTH_URL}/auth/tokens"
+
+        headers = {"X-Auth-Token":token,
+                "X-Subject-Token":token}
+
+        try:
+
+            r = requests.get(url=url,headers=headers)
+
+            if r.status_code == 200:
+                logging.info("Valid token")
+                return True
+
+            logging.warning("Invalid token")
+
+            return False
+
+        except requests.exceptions.RequestException as re:
+            logging.error(re)
+            logging.error("Invalid token request")
+            return False

src/storage/keystonesaml.py

@@ -0,0 +1,94 @@
+#
+#  Copyright (c) 2019-2021, ETH Zurich. All rights reserved.
+#
+#  Please, refer to the LICENSE file in the root directory.
+#  SPDX-License-Identifier: BSD-3-Clause
+#
+from keystoneauth1.identity import v3
+from keystoneauth1 import session as keystonesession
+from keystoneauth1 import exceptions as keystoneexception
+from keystoneauth1.extras._saml2 import V3Saml2Password
+
+import logging
+import requests
+import os
+from keystone import Keystone
+
+
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger(__name__)
+
+class KeystoneSAML(Keystone):
+
+    def __init__(self):
+        self.OS_AUTH_URL             = os.environ.get("F7T_OS_AUTH_URL")
+        self.OS_IDENTITY_PROVIDER    = os.environ.get("F7T_OS_IDENTITY_PROVIDER")
+        self.OS_IDENTITY_PROVIDER_URL= os.environ.get("F7T_OS_IDENTITY_PROVIDER_URL")
+        self.OS_PROTOCOL             = os.environ.get("F7T_OS_PROTOCOL")
+        self.OS_INTERFACE            = os.environ.get("F7T_OS_INTERFACE")
+        self.OS_PROJECT_ID           = os.environ.get("F7T_OS_PROJECT_ID")
+
+    # returns a valid token if username & password are valid keystone credentials
+    def authenticate(self,username,password):
+
+        try:
+
+            auth = V3Saml2Password(auth_url=self.OS_AUTH_URL, identity_provider=self.OS_IDENTITY_PROVIDER, protocol=self.OS_PROTOCOL,
+                                identity_provider_url=self.OS_IDENTITY_PROVIDER_URL, username=username, password=password)
+
+
+
+            sess = keystonesession.Session(auth=auth)
+            try:
+
+                log.info(sess.get_token())
+            except AttributeError as e:
+                log.info(e)
+                log.info(e.args)
+
+
+            auth = v3.token.Token(auth_url=self.OS_AUTH_URL, token=sess.get_token(), project_id=self.OS_PROJECT_ID)
+
+            sess = keystonesession.Session(auth=auth)
+
+            OS_TOKEN = sess.get_token()
+
+            return {"error":0,"OS_TOKEN":OS_TOKEN}
+
+        except keystoneexception.http.BadRequest as e:
+            log.error(e)
+            log.error(e.message)
+            log.error(e.details)
+            return {"error":1,"msg":e.message}
+
+
+        except Exception as e:
+
+            log.error(type(e))
+            return {"error":1,"msg":e}
+
+
+    # Checks if token is valid directly with keystone API
+    def is_token_valid(self,token):
+
+        url = "{os_auth_url}/auth/tokens".format(os_auth_url=self.OS_AUTH_URL)
+
+        headers = {"X-Auth-Token":token,
+                "X-Subject-Token":token}
+
+        try:
+
+            r = requests.get(url=url,headers=headers)
+
+            if r.status_code == 200:
+                logging.info("Valid token")
+                return True
+
+            logging.warning("Invalid token")
+
+            return False
+
+        except requests.exceptions.RequestException as re:
+            logging.error(re)
+            logging.error("Invalid token request")
+            return False

src/storage/storage.py

@@ -5,8 +5,6 @@
 #  SPDX-License-Identifier: BSD-3-Clause
 #
 from flask import Flask, request, jsonify
-
-import keystone
 import json, tempfile, os
 import urllib
 import datetime

src/storage/swiftOS.py

@@ -5,9 +5,9 @@
 #  SPDX-License-Identifier: BSD-3-Clause
 #
 from objectstorage import ObjectStorage
+import os
 import logging
 import requests
-import keystone
 from time import time
 from datetime import datetime
 import hmac
@@ -23,15 +23,31 @@ def __init__(self,url,user,passwd,secret):
         self.passwd = passwd
         self.secret = secret
 
+        # keystone authentication type selection
+        OS_KEYSTONE_AUTH = os.environ.get("F7T_OS_KEYSTONE_AUTH",None)
+        if OS_KEYSTONE_AUTH == "oidc":
+            from keystoneoidc import KeystoneOIDC as KeystoneAuth
+            self.keystone = KeystoneAuth()
+        elif OS_KEYSTONE_AUTH == "saml":
+            from keystonesaml import KeystoneSAML as KeystoneAuth
+            self.keystone = KeystoneAuth()
+        else:
+            self.keystone = None
+
+
     def get_object_storage(self):
         return "OpenStack Swift"
 
 
     # authenticate SWIFT against keystone
     def authenticate(self):
+
+        if not self.keystone:
+            return False
+
         logging.info("GET TOKEN: {user} ".format(user=self.user))
 
-        retVal = keystone.authenticate(self.user, self.passwd)
+        retVal = self.keystone.authenticate(self.user, self.passwd)
 
         if retVal["error"] == 1:
             logging.error("Keystone Auth Error:\n{msg}".format(msg=retVal["msg"]))
@@ -41,7 +57,10 @@ def authenticate(self):
         return True
 
     def is_token_valid(self):
-        return keystone.is_token_valid(self.auth)
+        if not self.keystone:
+            return False
+
+        return self.keystone.is_token_valid(self.auth)
 
     # return list of containers created
     def get_users(self):