diff --git a/.env_template b/.env_template index 8658dd3..ff63470 100644 --- a/.env_template +++ b/.env_template @@ -1,14 +1,16 @@ # Marketplace URL -MP_HOST="https://dataspace.reaxpro.eu" +MP_HOST="https://materials-marketplace.eu/" # Access token for markeplace connection. Need to change time to time when expired MP_ACCESS_TOKEN=.... # Env variables needed for data-sink connection otherwise optional # Client-ID of the datasink application -CLIENT_ID="2c791805-ea52-4446-af97-80c0355a73b4" +CLIENT_ID=.... # optional: if incase if we want to get access_token directly from key_cloak -KEYCLOAK_SERVER_URL="https://dataspace.reaxpro.eu/auth/" +# keycloak configuration details can be obtained from marketplace admin +# you can ignore setting MP_ACCESS_TOKEN if you configure keycloak authentication +KEYCLOAK_SERVER_URL="https://materials-marketplace.eu/" KEYCLOAK_CLIENT_ID=.... KEYCLOAK_REALM_NAME=.... KEYCLOAK_CLIENT_SECRET_KEY=.... diff --git a/examples/datasink_client/collection_dcat.py b/examples/datasink_client/collection_dcat.py index 1786e71..a46d788 100644 --- a/examples/datasink_client/collection_dcat.py +++ b/examples/datasink_client/collection_dcat.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.get_collection_dcat(collection_name="c1") diff --git a/examples/datasink_client/dataset_dcat.py b/examples/datasink_client/dataset_dcat.py index 130fade..0d3bc11 100644 --- a/examples/datasink_client/dataset_dcat.py +++ b/examples/datasink_client/dataset_dcat.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.get_dataset_dcat(collection_name="c1", dataset_name="d1") diff --git a/examples/datasink_client/delete_collection.py b/examples/datasink_client/delete_collection.py index 4916ebc..e3543a0 100644 --- a/examples/datasink_client/delete_collection.py +++ b/examples/datasink_client/delete_collection.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.delete_collection(collection_name="c1") diff --git a/examples/datasink_client/delete_dataset.py b/examples/datasink_client/delete_dataset.py index e40eec5..a344623 100644 --- a/examples/datasink_client/delete_dataset.py +++ b/examples/datasink_client/delete_dataset.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.delete_dataset(collection_name="c1", dataset_name="d1") diff --git a/examples/datasink_client/download_file.py b/examples/datasink_client/download_file.py index 47980ab..ce37062 100644 --- a/examples/datasink_client/download_file.py +++ b/examples/datasink_client/download_file.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.download_dataset( diff --git a/examples/datasink_client/download_folder.py b/examples/datasink_client/download_folder.py index bb97e4f..78b43ae 100644 --- a/examples/datasink_client/download_folder.py +++ b/examples/datasink_client/download_folder.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.download_datasets_from_collection( diff --git a/examples/datasink_client/list_collections.py b/examples/datasink_client/list_collections.py index edee211..bf31a11 100644 --- a/examples/datasink_client/list_collections.py +++ b/examples/datasink_client/list_collections.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.list_collections() diff --git a/examples/datasink_client/list_datasets.py b/examples/datasink_client/list_datasets.py index 4cc21a0..ee7bcf9 100644 --- a/examples/datasink_client/list_datasets.py +++ b/examples/datasink_client/list_datasets.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.list_datasets(collection_name="c1") diff --git a/examples/datasink_client/query.py b/examples/datasink_client/query.py index 70bc9fc..24dd7de 100644 --- a/examples/datasink_client/query.py +++ b/examples/datasink_client/query.py @@ -1,6 +1,13 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession + +marketplace_url = "https://materials-marketplace.eu/" +access_token = "PASTE_TOKEN_HERE" + +client_id = "edb56699-9377-4f41-b1c7-ef2f46dac707" query = """SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object . }} LIMIT 5""" -with MPSession() as test: +with MPSession( + marketplace_host_url=marketplace_url, access_token=access_token, client_id=client_id +) as test: objects = test.query(query=query) print(objects) diff --git a/examples/datasink_client/query_dataset.py b/examples/datasink_client/query_dataset.py index 271a72e..a3c069b 100644 --- a/examples/datasink_client/query_dataset.py +++ b/examples/datasink_client/query_dataset.py @@ -1,6 +1,14 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession -query = "SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object . }} LIMIT 5" -with MPSession() as test: - objects = test.query_dataset(collection_name="c1", dataset_name="d1", query=query) +marketplace_url = "https://materials-marketplace.eu/" +access_token = "PASTE_TOKEN_HERE" +client_id = "edb56699-9377-4f41-b1c7-ef2f46dac707" + +query = """SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object . }} LIMIT 5""" +with MPSession( + marketplace_host_url=marketplace_url, access_token=access_token, client_id=client_id +) as test: + objects = test.query_dataset( + collection_name="c1", dataset_name="data_test", query=query + ) print(objects) diff --git a/examples/datasink_client/upload_files_from_path.py b/examples/datasink_client/upload_files_from_path.py index 3bf19b4..1121a8a 100644 --- a/examples/datasink_client/upload_files_from_path.py +++ b/examples/datasink_client/upload_files_from_path.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.create_dataset_from_path( diff --git a/examples/datasink_client/upload_folder.py b/examples/datasink_client/upload_folder.py index b2e62e0..8641c1d 100644 --- a/examples/datasink_client/upload_folder.py +++ b/examples/datasink_client/upload_folder.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.create_datasets_from_sourcedir( diff --git a/marketplace/app/v0/base.py b/marketplace/app/v0/base.py index 253ac07..a2ed6b8 100644 --- a/marketplace/app/v0/base.py +++ b/marketplace/app/v0/base.py @@ -1,7 +1,7 @@ from typing import Optional from urllib.parse import urljoin -from fastapi.responses import HTMLResponse +from fastapi.responses import HTMLResponse, JSONResponse, Response from marketplace_standard_app_api.models.system import GlobalSearchResponse from marketplace.client import MarketPlaceClient @@ -41,3 +41,19 @@ def global_search( params={"q": q, "limit": limit, "offset": offset}, ).json() ) + + @check_capability_availability + def get_logs( + self, id: Optional[str], limit: int = 100, offset: int = 0 + ) -> Response: + return self._client.get( + self._proxy_path("getLogs"), + params={"id": id, "limit": limit, "offset": offset}, + ).content + + @check_capability_availability + def get_info(self, config: dict = None) -> JSONResponse: + params = {} + if config is not None: + params.update(config) + return self._client.get(self._proxy_path("getInfo"), params=params).json() diff --git a/marketplace/app/v0/object_storage.py b/marketplace/app/v0/object_storage.py index a8bdc06..5545e01 100644 --- a/marketplace/app/v0/object_storage.py +++ b/marketplace/app/v0/object_storage.py @@ -8,6 +8,8 @@ from .base import _MarketPlaceAppBase from .utils import _decode_metadata, _encode_metadata +DEFAULT_COLLECTION_NAME = "DEFAULT_COLLECTION" + class MarketPlaceObjectStorageApp(_MarketPlaceAppBase): @check_capability_availability @@ -106,7 +108,7 @@ def create_collection( @check_capability_availability def create_dataset( self, - collection_name: object_storage.CollectionName, + collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME, dataset_name: object_storage.DatasetName = None, metadata: dict = None, file: UploadFile = None, @@ -153,8 +155,8 @@ def create_dataset_metadata( @check_capability_availability def get_dataset( self, - collection_name: object_storage.CollectionName, dataset_name: object_storage.DatasetName, + collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME, ) -> Union[Dict, str]: return self._client.get( self._proxy_path("getDataset"), @@ -196,8 +198,8 @@ def create_or_replace_dataset_metadata( @check_capability_availability def delete_dataset( self, - collection_name: object_storage.CollectionName, dataset_name: object_storage.DatasetName, + collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME, ): return self._client.delete( self._proxy_path("deleteDataset"), @@ -252,8 +254,8 @@ def get_collection_metadata_dcat( @check_capability_availability def get_dataset_metadata_dcat( self, - collection_name: object_storage.CollectionName, dataset_name: object_storage.DatasetName, + collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME, ) -> Union[Dict, str]: response: dict = self._client.get( self._proxy_path("getDatasetMetadataDcat"), @@ -264,9 +266,9 @@ def get_dataset_metadata_dcat( @check_capability_availability def query_dataset( self, - collection_name: object_storage.CollectionName, dataset_name: object_storage.DatasetName, query: str, + collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME, ) -> Union[Dict, str]: response: dict = self._client.post( self._proxy_path("queryDataset"), diff --git a/marketplace/app/v0/transformation.py b/marketplace/app/v0/transformation.py index d02c1f2..459b7f4 100644 --- a/marketplace/app/v0/transformation.py +++ b/marketplace/app/v0/transformation.py @@ -18,11 +18,33 @@ def get_transformation_list( @check_capability_availability def new_transformation( - self, new_transformation: transformation.NewTransformationModel + self, + new_transformation: transformation.NewTransformationModel, + config: dict = None, ) -> transformation.TransformationCreateResponse: + """ + Creates a new transformation. + + Args: + - new_transformation (NewTransformationModel): A dictionary representing + parameters to create a new transformation. + - config (dict): A dictionary representing query parameters. + Any key-value passed inside this dictionary are sent as query parameters + to the application. + + Retruns: + TransformationCreateResponse: A dictionary containing id of the + created transformation. + """ + params = {} + # send additional key value as query parameters if some app needs it + if config is not None: + params.update(config) return transformation.TransformationCreateResponse.parse_obj( self._client.post( - self._proxy_path("newTransformation"), json=new_transformation + self._proxy_path("newTransformation"), + json=new_transformation, + params=params, ).json() ) diff --git a/marketplace/client.py b/marketplace/client.py index 5a4a25a..868b044 100644 --- a/marketplace/client.py +++ b/marketplace/client.py @@ -6,9 +6,11 @@ """ import os +from functools import wraps from urllib.parse import urljoin import requests +from keycloak import KeycloakOpenID from requests import Response from .version import __version__ @@ -16,6 +18,58 @@ MP_DEFAULT_HOST = "https://materials-marketplace.eu/" +def configure_token(func): + @wraps(func) + def func_(self, *arg, **kwargs): + r = func(self, *arg, **kwargs) + if r.status_code == 401: + response = configure() + if not response["status"] == "success": + raise Exception( + "User authentication failure. Reason:" + response["message"] + ) + token = response["token"] + os.environ["MP_ACCESS_TOKEN"] = token + self.access_token = token + + r = func(self, *arg, **kwargs) + if r.status_code > 400: + raise Exception("Server returned with an Exception. Details: " + r.text) + return r + + return func_ + + +def configure(): + """ + Authenticates a user with marketplace username and password using keycloak + authentication module. If the authentication is succesfull then this method returns + user token within a dictionary. Otherwise, an exception is caught and the + resaon for failure is returned as dict. In order to use keycloak authentication + with username and password we have to configure all the necessary keycloak + environment variables. Configurations can be obtained from market place admin. + """ + # Configure client + server_url = os.environ.get("KEYCLOAK_SERVER_URL") + client_id = os.environ.get("KEYCLOAK_CLIENT_ID") + realm_name = os.environ.get("KEYCLOAK_REALM_NAME") + client_key = os.environ.get("KEYCLOAK_CLIENT_SECRET_KEY") + user = os.environ.get("MARKETPLACE_USERNAME") + passwd = os.environ.get("MARKETPLACE_PASSWORD") + keycloak_openid = KeycloakOpenID( + server_url=server_url, + client_id=client_id, + realm_name=realm_name, + client_secret_key=client_key, + ) + try: + token = keycloak_openid.token(user, passwd) + token = token["access_token"] + return {"status": "success", "token": token, "message": ""} + except Exception as e: + return {"status": "failure", "token": None, "message": str(e)} + + class MarketPlaceClient: """Interact with the MarketPlace platform.""" @@ -24,7 +78,7 @@ def __init__(self, marketplace_host_url=None, access_token=None): "MP_HOST", MP_DEFAULT_HOST, ) - access_token = access_token or os.environ["MP_ACCESS_TOKEN"] + access_token = access_token or os.environ.get("MP_ACCESS_TOKEN") self.marketplace_host_url = marketplace_host_url self.access_token = access_token @@ -50,6 +104,7 @@ def userinfo(self): userinfo.raise_for_status() return userinfo.json() + @configure_token def _request(self, op, path, **kwargs) -> Response: kwargs.setdefault("headers", {}).update(self.default_headers) full_url = urljoin(self.marketplace_host_url, path) diff --git a/marketplace/datasink_client/cli.py b/marketplace/datasink_client/cli.py index 50466b1..a34bd79 100644 --- a/marketplace/datasink_client/cli.py +++ b/marketplace/datasink_client/cli.py @@ -6,7 +6,7 @@ import click -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession class CommaSeparatedListofPythonLiteralValues(click.Option): diff --git a/marketplace/datasink_client/session.py b/marketplace/datasink_client/session.py index 8b1f331..75bc857 100644 --- a/marketplace/datasink_client/session.py +++ b/marketplace/datasink_client/session.py @@ -1,61 +1,13 @@ import os import os.path -from functools import wraps - -# from dotenv import find_dotenv, load_dotenv -from keycloak import KeycloakOpenID from marketplace.app import MarketPlaceClient, get_app -from marketplace.data_sink_client.utils import ( +from marketplace.datasink_client.utils import ( get_collections_from_catalog, parse_objects_from_collection, ) -def reconfigure_if_expired(func): - @wraps(func) - def func_(self, *arg, **kwargs): - try: - r = func(self, *arg, **kwargs) - return r - except Exception as e: - print( - "API encountered exception. Please check if all your environment variables configured properly once. Error details: ", - str(e), - ) - # temporary work around to catch Un Authorized error - """error = str(e) - if len(error.split("401 Unauthorized")) > 0: - print("Token expired. Reconfiguring again.") - token = configure() - os.environ["MP_ACCESS_TOKEN"] = token - r = func(self, *arg, **kwargs) - return r - else: - raise RuntimeError(e)""" - - return func_ - - -def configure(): - # Configure client - server_url = os.environ.get("KEYCLOAK_SERVER_URL") - client_id = os.environ.get("KEYCLOAK_CLIENT_ID") - realm_name = os.environ.get("KEYCLOAK_REALM_NAME") - client_key = os.environ.get("KEYCLOAK_CLIENT_SECRET_KEY") - user = os.environ.get("MARKETPLACE_USERNAME") - passwd = os.environ.get("MARKETPLACE_PASSWORD") - keycloak_openid = KeycloakOpenID( - server_url=server_url, - client_id=client_id, - realm_name=realm_name, - client_secret_key=client_key, - ) - token = keycloak_openid.token(user, passwd) - token = token["access_token"] - return token - - class MPSession: """ReaxPro-MarketPlace Session API Wrapper. @@ -66,7 +18,6 @@ class MPSession: """ - @reconfigure_if_expired def __init__( self, marketplace_host_url=None, access_token=None, client_id=None, **kwargs ): @@ -77,7 +28,6 @@ def __init__( ) self.marketPlace = get_app(app_id=CLIENT_ID, client=mp_client) - @reconfigure_if_expired def create_dataset( self, collection_name, dataset_name, sub_collection_id, abs_path ): @@ -120,7 +70,6 @@ def create_dataset( ) return None - @reconfigure_if_expired def create_collection(self, collection_name, sub_collection_id): """create collection/catalog to the MarketPlace DataSink. @@ -138,10 +87,9 @@ def create_collection(self, collection_name, sub_collection_id): collection_name=collection_name, config=config ) if "collection_id" not in response: - print(response) return None else: - return response["collection_id"] + return response except Exception as e: print( @@ -150,7 +98,6 @@ def create_collection(self, collection_name, sub_collection_id): ) return None - @reconfigure_if_expired def get_dataset(self, collection_name=None, dataset_name=None): """Get binary data from a get request @@ -165,7 +112,6 @@ def get_dataset(self, collection_name=None, dataset_name=None): return response - @reconfigure_if_expired def get_collection_dcat(self, collection_name=None): """Get a collection/catalog object from a get request @@ -178,7 +124,6 @@ def get_collection_dcat(self, collection_name=None): ) return response - @reconfigure_if_expired def get_dataset_dcat(self, collection_name=None, dataset_name=None): """Get a dataset dcat object from a get request @@ -192,7 +137,6 @@ def get_dataset_dcat(self, collection_name=None, dataset_name=None): ) return response - @reconfigure_if_expired def list_collections(self): """Returns list of Collections. @@ -201,7 +145,6 @@ def list_collections(self): response = self.marketPlace.list_collections() return response - @reconfigure_if_expired def list_datasets(self, collection_name): """Returns list of datasets for a specific collection. @@ -212,7 +155,6 @@ def list_datasets(self, collection_name): response = self.marketPlace.list_datasets(collection_name) return response - @reconfigure_if_expired def delete_collection(self, collection_name): """Delete a collection from datasink. @@ -223,7 +165,6 @@ def delete_collection(self, collection_name): response = self.marketPlace.delete_collection(collection_name) return response - @reconfigure_if_expired def delete_dataset(self, collection_name, dataset_name): """Delete a dataset from datasink. @@ -235,7 +176,6 @@ def delete_dataset(self, collection_name, dataset_name): response = self.marketPlace.delete_dataset(collection_name, dataset_name) return response - @reconfigure_if_expired def query_dataset(self, collection_name, dataset_name, query): """Execute a aparql query on a dataset stored in datasink. @@ -245,10 +185,11 @@ def query_dataset(self, collection_name, dataset_name, query): :returns: List of data """ - response = self.marketPlace.query_dataset(collection_name, dataset_name, query) + response = self.marketPlace.query_dataset( + collection_name=collection_name, dataset_name=dataset_name, query=query + ) return response - @reconfigure_if_expired def query(self, query, meta_data=False): """Execute a aparql query on a dataset stored in datasink. @@ -260,7 +201,6 @@ def query(self, query, meta_data=False): response = self.marketPlace.query(query, meta_data=meta_data) return response - @reconfigure_if_expired def create_dataset_from_path(self, path, collection_name=None, dataset_name=None): if not os.path.exists(path): raise Exception("File " + path + " does not exist.") @@ -280,7 +220,6 @@ def create_dataset_from_path(self, path, collection_name=None, dataset_name=None ) return response - @reconfigure_if_expired def create_datasets_from_paths(self, paths, collection_name, dataset_names): """Inject a list of datasets. A single InformationPackage will be created. @@ -294,13 +233,11 @@ def create_datasets_from_paths(self, paths, collection_name, dataset_names): response_list = [] if collection_name is not None: - collection_id = self.create_collection( + response = self.create_collection( collection_name=collection_name, sub_collection_id=None ) - if collection_id is not None: - response_list.append((collection_name, collection_id)) - else: - return + if response is not None: + response_list.append((collection_name, response["collection_id"])) else: raise Exception("collection title cannot be empty.") @@ -315,7 +252,6 @@ def create_datasets_from_paths(self, paths, collection_name, dataset_names): return response_list - @reconfigure_if_expired def create_datasets_from_sourcedir( self, sourcedir: str, collection_name: str = None ): @@ -349,7 +285,6 @@ def create_datasets_from_sourcedir( return response_list - @reconfigure_if_expired def create_objects_from_sourcedir( self, collection_name, sourcedir, collection_id, response_list=[] ): @@ -378,7 +313,6 @@ def create_objects_from_sourcedir( # print("added file: ", (os.path.join(sourcedir, file), dataset_id)) return response_list - @reconfigure_if_expired def download_dataset( self, collection_name, @@ -401,7 +335,6 @@ def download_dataset( result.append({"download_path": file_path}) return result - @reconfigure_if_expired def download_datasets_from_search_query( self, collection_search_query, @@ -441,7 +374,6 @@ def download_datasets_from_search_query( return result - @reconfigure_if_expired def download_datasets_from_collection( self, collection_name, diff --git a/setup.cfg b/setup.cfg index d2da37b..e855911 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ classifiers = packages = find: install_requires = fastapi>0.75,<=0.95.1 - marketplace-standard-app-api~=0.4 + marketplace-standard-app-api~=0.6 packaging>=21.3,<=23.0 pika~=1.2 python-keycloak==2.12.0 @@ -31,18 +31,18 @@ python_requires = >=3.8 [options.entry_points] console_scripts = - list_collections = marketplace.data_sink_client.cli:list_collections - list_datasets = marketplace.data_sink_client.cli:list_datasets - get_collection_dcat = marketplace.data_sink_client.cli:get_collection_dcat - get_dataset_dcat = marketplace.data_sink_client.cli:get_dataset_dcat - delete_dataset = marketplace.data_sink_client.cli:delete_dataset - delete_collection = marketplace.data_sink_client.cli:delete_collection - upload_file_from_path = marketplace.data_sink_client.cli:upload_file_from_path - upload_folder = marketplace.data_sink_client.cli:upload_files_from_folder - download_folder = marketplace.data_sink_client.cli:download_folder - download_file = marketplace.data_sink_client.cli:download_file - query = marketplace.data_sink_client.cli:query - query_dataset = marketplace.data_sink_client.cli:query_dataset + list_collections = marketplace.datasink_client.cli:list_collections + list_datasets = marketplace.datasink_client.cli:list_datasets + get_collection_dcat = marketplace.datasink_client.cli:get_collection_dcat + get_dataset_dcat = marketplace.datasink_client.cli:get_dataset_dcat + delete_dataset = marketplace.datasink_client.cli:delete_dataset + delete_collection = marketplace.datasink_client.cli:delete_collection + upload_file_from_path = marketplace.datasink_client.cli:upload_file_from_path + upload_folder = marketplace.datasink_client.cli:upload_files_from_folder + download_folder = marketplace.datasink_client.cli:download_folder + download_file = marketplace.datasink_client.cli:download_file + query = marketplace.datasink_client.cli:query + query_dataset = marketplace.datasink_client.cli:query_dataset [options.extras_require] dev =