modification for reaxflow-workflow-service

.env_template

@@ -1,14 +1,16 @@
 # Marketplace URL
-MP_HOST="https://dataspace.reaxpro.eu"
+MP_HOST="https://materials-marketplace.eu/"
 # Access token for markeplace connection. Need to change time to time when expired
 MP_ACCESS_TOKEN=....
 
 # Env variables needed for data-sink connection otherwise optional
 # Client-ID of the datasink application
-CLIENT_ID="2c791805-ea52-4446-af97-80c0355a73b4"
+CLIENT_ID=....
 
 # optional: if incase if we want to get access_token directly from key_cloak
-KEYCLOAK_SERVER_URL="https://dataspace.reaxpro.eu/auth/"
+# keycloak configuration details can be obtained from marketplace admin
+# you can ignore setting MP_ACCESS_TOKEN if you configure keycloak authentication
+KEYCLOAK_SERVER_URL="https://materials-marketplace.eu/"
 KEYCLOAK_CLIENT_ID=....
 KEYCLOAK_REALM_NAME=....
 KEYCLOAK_CLIENT_SECRET_KEY=....

examples/datasink_client/collection_dcat.py

@@ -1,4 +1,4 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
 with MPSession() as test:
     objects = test.get_collection_dcat(collection_name="c1")

examples/datasink_client/dataset_dcat.py

@@ -1,4 +1,4 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
 with MPSession() as test:
     objects = test.get_dataset_dcat(collection_name="c1", dataset_name="d1")

examples/datasink_client/delete_collection.py

@@ -1,4 +1,4 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
 with MPSession() as test:
     objects = test.delete_collection(collection_name="c1")

examples/datasink_client/delete_dataset.py

@@ -1,4 +1,4 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
 with MPSession() as test:
     objects = test.delete_dataset(collection_name="c1", dataset_name="d1")

examples/datasink_client/download_file.py

@@ -1,4 +1,4 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
 with MPSession() as test:
     objects = test.download_dataset(

examples/datasink_client/download_folder.py

@@ -1,4 +1,4 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
 with MPSession() as test:
     objects = test.download_datasets_from_collection(

examples/datasink_client/list_collections.py

@@ -1,4 +1,4 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
 with MPSession() as test:
     objects = test.list_collections()

examples/datasink_client/list_datasets.py

@@ -1,4 +1,4 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
 with MPSession() as test:
     objects = test.list_datasets(collection_name="c1")

examples/datasink_client/query.py

@@ -1,6 +1,13 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
+
+marketplace_url = "https://materials-marketplace.eu/"
+access_token = "PASTE_TOKEN_HERE"
+
+client_id = "edb56699-9377-4f41-b1c7-ef2f46dac707"
 
 query = """SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object . }} LIMIT 5"""
-with MPSession() as test:
+with MPSession(
+    marketplace_host_url=marketplace_url, access_token=access_token, client_id=client_id
+) as test:
     objects = test.query(query=query)
     print(objects)

examples/datasink_client/query_dataset.py

@@ -1,6 +1,14 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
-query = "SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object . }} LIMIT 5"
-with MPSession() as test:
-    objects = test.query_dataset(collection_name="c1", dataset_name="d1", query=query)
+marketplace_url = "https://materials-marketplace.eu/"
+access_token = "PASTE_TOKEN_HERE"
+client_id = "edb56699-9377-4f41-b1c7-ef2f46dac707"
+
+query = """SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object . }} LIMIT 5"""
+with MPSession(
+    marketplace_host_url=marketplace_url, access_token=access_token, client_id=client_id
+) as test:
+    objects = test.query_dataset(
+        collection_name="c1", dataset_name="data_test", query=query
+    )
     print(objects)

examples/datasink_client/upload_files_from_path.py

@@ -1,4 +1,4 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
 with MPSession() as test:
     objects = test.create_dataset_from_path(

examples/datasink_client/upload_folder.py

@@ -1,4 +1,4 @@
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
 with MPSession() as test:
     objects = test.create_datasets_from_sourcedir(

marketplace/app/v0/base.py

@@ -1,7 +1,7 @@
 from typing import Optional
 from urllib.parse import urljoin
 
-from fastapi.responses import HTMLResponse
+from fastapi.responses import HTMLResponse, JSONResponse, Response
 from marketplace_standard_app_api.models.system import GlobalSearchResponse
 
 from marketplace.client import MarketPlaceClient
@@ -41,3 +41,19 @@ def global_search(
                 params={"q": q, "limit": limit, "offset": offset},
             ).json()
         )
+
+    @check_capability_availability
+    def get_logs(
+        self, id: Optional[str], limit: int = 100, offset: int = 0
+    ) -> Response:
+        return self._client.get(
+            self._proxy_path("getLogs"),
+            params={"id": id, "limit": limit, "offset": offset},
+        ).content
+
+    @check_capability_availability
+    def get_info(self, config: dict = None) -> JSONResponse:
+        params = {}
+        if config is not None:
+            params.update(config)
+        return self._client.get(self._proxy_path("getInfo"), params=params).json()

marketplace/app/v0/object_storage.py

@@ -8,6 +8,8 @@
 from .base import _MarketPlaceAppBase
 from .utils import _decode_metadata, _encode_metadata
 
+DEFAULT_COLLECTION_NAME = "DEFAULT_COLLECTION"
+
 
 class MarketPlaceObjectStorageApp(_MarketPlaceAppBase):
     @check_capability_availability
@@ -106,7 +108,7 @@ def create_collection(
     @check_capability_availability
     def create_dataset(
         self,
-        collection_name: object_storage.CollectionName,
+        collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME,
         dataset_name: object_storage.DatasetName = None,
         metadata: dict = None,
         file: UploadFile = None,
@@ -153,8 +155,8 @@ def create_dataset_metadata(
     @check_capability_availability
     def get_dataset(
         self,
-        collection_name: object_storage.CollectionName,
         dataset_name: object_storage.DatasetName,
+        collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME,
     ) -> Union[Dict, str]:
         return self._client.get(
             self._proxy_path("getDataset"),
@@ -196,8 +198,8 @@ def create_or_replace_dataset_metadata(
     @check_capability_availability
     def delete_dataset(
         self,
-        collection_name: object_storage.CollectionName,
         dataset_name: object_storage.DatasetName,
+        collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME,
     ):
         return self._client.delete(
             self._proxy_path("deleteDataset"),
@@ -252,8 +254,8 @@ def get_collection_metadata_dcat(
     @check_capability_availability
     def get_dataset_metadata_dcat(
         self,
-        collection_name: object_storage.CollectionName,
         dataset_name: object_storage.DatasetName,
+        collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME,
     ) -> Union[Dict, str]:
         response: dict = self._client.get(
             self._proxy_path("getDatasetMetadataDcat"),
@@ -264,9 +266,9 @@ def get_dataset_metadata_dcat(
     @check_capability_availability
     def query_dataset(
         self,
-        collection_name: object_storage.CollectionName,
         dataset_name: object_storage.DatasetName,
         query: str,
+        collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME,
     ) -> Union[Dict, str]:
         response: dict = self._client.post(
             self._proxy_path("queryDataset"),

marketplace/app/v0/transformation.py

@@ -18,11 +18,33 @@ def get_transformation_list(
 
     @check_capability_availability
     def new_transformation(
-        self, new_transformation: transformation.NewTransformationModel
+        self,
+        new_transformation: transformation.NewTransformationModel,
+        config: dict = None,
     ) -> transformation.TransformationCreateResponse:
+        """
+        Creates a new transformation.
+
+        Args:
+        - new_transformation (NewTransformationModel): A dictionary representing
+            parameters to create a new transformation.
+        - config (dict): A dictionary representing query parameters.
+            Any key-value passed inside this dictionary are sent as query parameters
+            to the application.
+
+        Retruns:
+            TransformationCreateResponse: A dictionary containing id of the
+            created transformation.
+        """
+        params = {}
+        # send additional key value as query parameters if some app needs it
+        if config is not None:
+            params.update(config)
         return transformation.TransformationCreateResponse.parse_obj(
             self._client.post(
-                self._proxy_path("newTransformation"), json=new_transformation
+                self._proxy_path("newTransformation"),
+                json=new_transformation,
+                params=params,
             ).json()
         )
 

marketplace/client.py

@@ -6,16 +6,70 @@
 """
 
 import os
+from functools import wraps
 from urllib.parse import urljoin
 
 import requests
+from keycloak import KeycloakOpenID
 from requests import Response
 
 from .version import __version__
 
 MP_DEFAULT_HOST = "https://materials-marketplace.eu/"
 
 
+def configure_token(func):
+    @wraps(func)
+    def func_(self, *arg, **kwargs):
+        r = func(self, *arg, **kwargs)
+        if r.status_code == 401:
+            response = configure()
+            if not response["status"] == "success":
+                raise Exception(
+                    "User authentication failure. Reason:" + response["message"]
+                )
+            token = response["token"]
+            os.environ["MP_ACCESS_TOKEN"] = token
+            self.access_token = token
+
+        r = func(self, *arg, **kwargs)
+        if r.status_code > 400:
+            raise Exception("Server returned with an Exception. Details: " + r.text)
+        return r
+
+    return func_
+
+
+def configure():
+    """
+    Authenticates a user with marketplace username and password using keycloak
+    authentication module. If the authentication is succesfull then this method returns
+    user token within a dictionary. Otherwise, an exception is caught and the
+    resaon for failure is returned as dict. In order to use keycloak authentication
+    with username and password we have to configure all the necessary keycloak
+    environment variables. Configurations can be obtained from market place admin.
+    """
+    # Configure client
+    server_url = os.environ.get("KEYCLOAK_SERVER_URL")
+    client_id = os.environ.get("KEYCLOAK_CLIENT_ID")
+    realm_name = os.environ.get("KEYCLOAK_REALM_NAME")
+    client_key = os.environ.get("KEYCLOAK_CLIENT_SECRET_KEY")
+    user = os.environ.get("MARKETPLACE_USERNAME")
+    passwd = os.environ.get("MARKETPLACE_PASSWORD")
+    keycloak_openid = KeycloakOpenID(
+        server_url=server_url,
+        client_id=client_id,
+        realm_name=realm_name,
+        client_secret_key=client_key,
+    )
+    try:
+        token = keycloak_openid.token(user, passwd)
+        token = token["access_token"]
+        return {"status": "success", "token": token, "message": ""}
+    except Exception as e:
+        return {"status": "failure", "token": None, "message": str(e)}
+
+
 class MarketPlaceClient:
     """Interact with the MarketPlace platform."""
 
@@ -24,7 +78,7 @@ def __init__(self, marketplace_host_url=None, access_token=None):
             "MP_HOST",
             MP_DEFAULT_HOST,
         )
-        access_token = access_token or os.environ["MP_ACCESS_TOKEN"]
+        access_token = access_token or os.environ.get("MP_ACCESS_TOKEN")
 
         self.marketplace_host_url = marketplace_host_url
         self.access_token = access_token
@@ -50,6 +104,7 @@ def userinfo(self):
         userinfo.raise_for_status()
         return userinfo.json()
 
+    @configure_token
     def _request(self, op, path, **kwargs) -> Response:
         kwargs.setdefault("headers", {}).update(self.default_headers)
         full_url = urljoin(self.marketplace_host_url, path)

marketplace/datasink_client/cli.py

@@ -6,7 +6,7 @@
 
 import click
 
-from marketplace.data_sink_client.session import MPSession
+from marketplace.datasink_client.session import MPSession
 
 
 class CommaSeparatedListofPythonLiteralValues(click.Option):