diff --git a/deploy/docker/storage/Dockerfile b/deploy/docker/storage/Dockerfile index cacba021..56af5295 100644 --- a/deploy/docker/storage/Dockerfile +++ b/deploy/docker/storage/Dockerfile @@ -11,6 +11,8 @@ RUN pip3 install -r deps/requirements.txt ADD src/storage/storage.py storage.py ADD src/storage/keystone.py keystone.py +ADD src/storage/keystoneoidc.py keystoneoidc.py +ADD src/storage/keystonesaml.py keystonesaml.py ADD src/storage/objectstorage.py objectstorage.py ADD src/storage/s3v2OS.py s3v2OS.py ADD src/storage/s3v4OS.py s3v4OS.py diff --git a/doc/openapi/firecrest-api.yaml b/doc/openapi/firecrest-api.yaml index 7b8d41ef..3a8d3210 100644 --- a/doc/openapi/firecrest-api.yaml +++ b/doc/openapi/firecrest-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.1-beta4 + version: 1.7.2-beta1 title: FirecREST Developers API description: > This API specification is intended for FirecREST developers only. There're some endpoints that are not available in the public version for client developers. diff --git a/doc/openapi/firecrest-developers-api.yaml b/doc/openapi/firecrest-developers-api.yaml index 10a50ec8..40aad4d1 100644 --- a/doc/openapi/firecrest-developers-api.yaml +++ b/doc/openapi/firecrest-developers-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.1-beta4 + version: 1.7.2-beta1 title: FirecREST API description: > FirecREST platform, a RESTful Services Gateway to HPC resources, is a diff --git a/src/storage/keystone.py b/src/storage/keystone.py index b746609f..6801562c 100644 --- a/src/storage/keystone.py +++ b/src/storage/keystone.py @@ -4,88 +4,25 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause # -from keystoneauth1.identity import v3 -from keystoneauth1 import session as keystonesession -from keystoneauth1 import exceptions as keystoneexception -from keystoneauth1.extras._saml2 import V3Saml2Password +from abc import ABCMeta,abstractmethod -import logging -import requests -import os +class Keystone: + __metaclass__ = ABCMeta -OS_AUTH_URL = os.environ.get("F7T_OS_AUTH_URL") -OS_IDENTITY_PROVIDER = os.environ.get("F7T_OS_IDENTITY_PROVIDER") -OS_IDENTITY_PROVIDER_URL= os.environ.get("F7T_OS_IDENTITY_PROVIDER_URL") -OS_PROTOCOL = os.environ.get("F7T_OS_PROTOCOL") -OS_INTERFACE = os.environ.get("F7T_OS_INTERFACE") -OS_PROJECT_ID = os.environ.get("F7T_OS_PROJECT_ID") + # default constructor + @abstractmethod + def __init__(self,url): + pass + # returns a valid token if username & password are valid keystone credentials + @abstractmethod + def authenticate(username,password): + pass -logging.basicConfig(level=logging.INFO) -log = logging.getLogger(__name__) + # Checks if token is valid directly with keystone API + @abstractmethod + def is_token_valid(token): + pass -# returns a valid token if username & password are valid keystone credentials -def authenticate(username,password): - try: - - auth = V3Saml2Password(auth_url=OS_AUTH_URL, identity_provider=OS_IDENTITY_PROVIDER, protocol=OS_PROTOCOL, - identity_provider_url=OS_IDENTITY_PROVIDER_URL, username=username, password=password) - - - - sess = keystonesession.Session(auth=auth) - try: - - log.info(sess.get_token()) - except AttributeError as e: - log.info(e) - log.info(e.args) - - - auth = v3.token.Token(auth_url=OS_AUTH_URL, token=sess.get_token(), project_id=OS_PROJECT_ID) - - sess = keystonesession.Session(auth=auth) - - OS_TOKEN = sess.get_token() - - return {"error":0,"OS_TOKEN":OS_TOKEN} - - except keystoneexception.http.BadRequest as e: - log.error(e) - log.error(e.message) - log.error(e.details) - return {"error":1,"msg":e.message} - - - except Exception as e: - - log.error(type(e)) - return {"error":1,"msg":e} - - -# Checks if token is valid directly with keystone API -def is_token_valid(token): - - url = "{os_auth_url}/auth/tokens".format(os_auth_url=OS_AUTH_URL) - - headers = {"X-Auth-Token":token, - "X-Subject-Token":token} - - try: - - r = requests.get(url=url,headers=headers) - - if r.status_code == 200: - logging.info("Valid token") - return True - - logging.warning("Invalid token") - - return False - - except requests.exceptions.RequestException as re: - logging.error(re) - logging.error("Invalid token request") - return False diff --git a/src/storage/keystoneoidc.py b/src/storage/keystoneoidc.py new file mode 100644 index 00000000..91ac9940 --- /dev/null +++ b/src/storage/keystoneoidc.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2019-2021, ETH Zurich. All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause +# +from keystoneauth1.identity import v3 +from keystoneauth1 import session as keystonesession +from keystoneauth1 import exceptions as keystoneexception +from keystoneauth1.identity import V3OidcPassword + +import logging +import requests +import os +from keystone import Keystone + + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger(__name__) + +class KeystoneOIDC(Keystone): + + def __init__(self): + self.OS_AUTH_URL = os.environ.get("F7T_OS_AUTH_URL") + self.OS_IDENTITY_PROVIDER = os.environ.get("F7T_OS_IDENTITY_PROVIDER") + self.OS_PROTOCOL = os.environ.get("F7T_OS_PROTOCOL") + self.OS_INTERFACE = os.environ.get("F7T_OS_INTERFACE") + self.OS_PROJECT_ID = os.environ.get("F7T_OS_PROJECT_ID") + self.OS_CLIENT_ID = os.environ.get("F7T_OS_CLIENT_ID") + self.OS_CLIENT_SECRET = os.environ.get("F7T_OS_CLIENT_SECRET") + self.OS_DISCOVERY_ENDPOINT = os.environ.get("F7T_OS_DISCOVERY_ENDPOINT") + + # returns a valid token if username & password are valid keystone credentials + def authenticate(self,username,password): + + try: + + auth = V3OidcPassword(auth_url=self.OS_AUTH_URL, identity_provider=self.OS_IDENTITY_PROVIDER, protocol=self.OS_PROTOCOL, + client_id=self.OS_CLIENT_ID, client_secret=self.OS_CLIENT_SECRET, discovery_endpoint=self.OS_DISCOVERY_ENDPOINT, + username=username, password=password) + + + + sess = keystonesession.Session(auth=auth) + try: + + log.info(sess.get_token()) + except AttributeError as e: + log.info(e) + log.info(e.args) + + + auth = v3.token.Token(auth_url=self.OS_AUTH_URL, token=sess.get_token(), project_id=self.OS_PROJECT_ID) + + sess = keystonesession.Session(auth=auth) + + OS_TOKEN = sess.get_token() + + return {"error":0,"OS_TOKEN":OS_TOKEN} + + except keystoneexception.http.BadRequest as e: + log.error(e) + log.error(e.message) + log.error(e.details) + return {"error":1,"msg":e.message} + + + except Exception as e: + + log.error(type(e)) + return {"error":1,"msg":e} + + + # Checks if token is valid directly with keystone API + def is_token_valid(self,token): + + url = f"{self.OS_AUTH_URL}/auth/tokens" + + headers = {"X-Auth-Token":token, + "X-Subject-Token":token} + + try: + + r = requests.get(url=url,headers=headers) + + if r.status_code == 200: + logging.info("Valid token") + return True + + logging.warning("Invalid token") + + return False + + except requests.exceptions.RequestException as re: + logging.error(re) + logging.error("Invalid token request") + return False diff --git a/src/storage/keystonesaml.py b/src/storage/keystonesaml.py new file mode 100644 index 00000000..f0194b68 --- /dev/null +++ b/src/storage/keystonesaml.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2019-2021, ETH Zurich. All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause +# +from keystoneauth1.identity import v3 +from keystoneauth1 import session as keystonesession +from keystoneauth1 import exceptions as keystoneexception +from keystoneauth1.extras._saml2 import V3Saml2Password + +import logging +import requests +import os +from keystone import Keystone + + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger(__name__) + +class KeystoneSAML(Keystone): + + def __init__(self): + self.OS_AUTH_URL = os.environ.get("F7T_OS_AUTH_URL") + self.OS_IDENTITY_PROVIDER = os.environ.get("F7T_OS_IDENTITY_PROVIDER") + self.OS_IDENTITY_PROVIDER_URL= os.environ.get("F7T_OS_IDENTITY_PROVIDER_URL") + self.OS_PROTOCOL = os.environ.get("F7T_OS_PROTOCOL") + self.OS_INTERFACE = os.environ.get("F7T_OS_INTERFACE") + self.OS_PROJECT_ID = os.environ.get("F7T_OS_PROJECT_ID") + + # returns a valid token if username & password are valid keystone credentials + def authenticate(self,username,password): + + try: + + auth = V3Saml2Password(auth_url=self.OS_AUTH_URL, identity_provider=self.OS_IDENTITY_PROVIDER, protocol=self.OS_PROTOCOL, + identity_provider_url=self.OS_IDENTITY_PROVIDER_URL, username=username, password=password) + + + + sess = keystonesession.Session(auth=auth) + try: + + log.info(sess.get_token()) + except AttributeError as e: + log.info(e) + log.info(e.args) + + + auth = v3.token.Token(auth_url=self.OS_AUTH_URL, token=sess.get_token(), project_id=self.OS_PROJECT_ID) + + sess = keystonesession.Session(auth=auth) + + OS_TOKEN = sess.get_token() + + return {"error":0,"OS_TOKEN":OS_TOKEN} + + except keystoneexception.http.BadRequest as e: + log.error(e) + log.error(e.message) + log.error(e.details) + return {"error":1,"msg":e.message} + + + except Exception as e: + + log.error(type(e)) + return {"error":1,"msg":e} + + + # Checks if token is valid directly with keystone API + def is_token_valid(self,token): + + url = "{os_auth_url}/auth/tokens".format(os_auth_url=self.OS_AUTH_URL) + + headers = {"X-Auth-Token":token, + "X-Subject-Token":token} + + try: + + r = requests.get(url=url,headers=headers) + + if r.status_code == 200: + logging.info("Valid token") + return True + + logging.warning("Invalid token") + + return False + + except requests.exceptions.RequestException as re: + logging.error(re) + logging.error("Invalid token request") + return False diff --git a/src/storage/storage.py b/src/storage/storage.py index ae572c88..c9657d14 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -5,8 +5,6 @@ # SPDX-License-Identifier: BSD-3-Clause # from flask import Flask, request, jsonify - -import keystone import json, tempfile, os import urllib import datetime diff --git a/src/storage/swiftOS.py b/src/storage/swiftOS.py index 4bb3a697..b332db54 100644 --- a/src/storage/swiftOS.py +++ b/src/storage/swiftOS.py @@ -5,9 +5,9 @@ # SPDX-License-Identifier: BSD-3-Clause # from objectstorage import ObjectStorage +import os import logging import requests -import keystone from time import time from datetime import datetime import hmac @@ -23,15 +23,31 @@ def __init__(self,url,user,passwd,secret): self.passwd = passwd self.secret = secret + # keystone authentication type selection + OS_KEYSTONE_AUTH = os.environ.get("F7T_OS_KEYSTONE_AUTH",None) + if OS_KEYSTONE_AUTH == "oidc": + from keystoneoidc import KeystoneOIDC as KeystoneAuth + self.keystone = KeystoneAuth() + elif OS_KEYSTONE_AUTH == "saml": + from keystonesaml import KeystoneSAML as KeystoneAuth + self.keystone = KeystoneAuth() + else: + self.keystone = None + + def get_object_storage(self): return "OpenStack Swift" # authenticate SWIFT against keystone def authenticate(self): + + if not self.keystone: + return False + logging.info("GET TOKEN: {user} ".format(user=self.user)) - retVal = keystone.authenticate(self.user, self.passwd) + retVal = self.keystone.authenticate(self.user, self.passwd) if retVal["error"] == 1: logging.error("Keystone Auth Error:\n{msg}".format(msg=retVal["msg"])) @@ -41,7 +57,10 @@ def authenticate(self): return True def is_token_valid(self): - return keystone.is_token_valid(self.auth) + if not self.keystone: + return False + + return self.keystone.is_token_valid(self.auth) # return list of containers created def get_users(self):