Merge pull request #157 from scaleoutsystems/release/v0.2.0

STACKn v0.2.0

.gitignore

@@ -1,5 +1,4 @@
 # Byte-compiled / optimized / DLL files
-migrations/
 .idea/
 .DS_Store
 __pycache__/
@@ -29,6 +28,8 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+Pipfile
+Pipfile.lock
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -135,3 +136,4 @@ media/generators/
 media/visualisers/
 
 tele_settings.py
+tele_settings_worker.py

README.md

@@ -1,3 +1,5 @@
+![alt text](https://thumb.tildacdn.com/tild3162-6435-4365-a230-656137616436/-/resize/560x/-/format/webp/stacknlogo3.png)
+
 <a href="#what-is-stackn">What is STACKn</a><br>
 <a href="#why-use-stackn">Why use STACKn</a><br>
 <a href="#core-features">Core features</a><br>
@@ -160,6 +162,9 @@ https://github.com/scaleoutsystems/aml-example-project
 
 Contributions are welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md).
 
+# Where is STACKn used?
+STACKn is used in various places, examples include [SciLifeLab Data Center](https://www.scilifelab.se/data) and within the EU-funded project [EOSC-Nordics](https://www.eosc-nordic.eu/).
+
 # Maintainers
 **Scaleout Systems AB** is the main contributing organization behind this project.
 - morganekmefjord

cli/MANIFEST.in

@@ -0,0 +1 @@
+include scaleout/cli/default-project.tar.gz

cli/scaleout/auth.py

@@ -11,22 +11,21 @@
 
 logger = logging.getLogger('cli')
 
-def keycloak_user_auth(username, password, keycloak_url, client_id='studio-api', realm='STACKn'):
-    discovery_url = '{}/auth/realms/{}'.format(keycloak_url, realm)
-    res = requests.get(discovery_url)
+def keycloak_user_auth(username, password, keycloak_url, client_id='studio-api', realm='STACKn', secure=True):
+    discovery_url = os.path.join(keycloak_url, 'auth/realms/{}'.format(realm))
+    res = requests.get(discovery_url, verify=secure)
     if res:
         realm_info = res.json()
         public_key = realm_info['public_key']
     else:
         print('Failed to discover realm settings: '+realm)
         return None
-
-    token_url = '{}/auth/realms/{}/protocol/openid-connect/token'.format(keycloak_url, realm)
+    token_url = os.path.join(keycloak_url, 'auth/realms/{}/protocol/openid-connect/token'.format(realm))
     req = {'client_id': client_id,
            'grant_type': 'password',
            'username': username,
            'password': password}
-    res = requests.post(token_url, data=req)
+    res = requests.post(token_url, data=req, verify=secure)
     if res:
         res = res.json()
     else:
@@ -70,7 +69,7 @@ def write_tokens(deployment, token, refresh_token, public_key, keycloak_host, st
     if not status:
         logger.info('Could not write tokens -- failed to write to file.')
 
-def get_stackn_config():
+def get_stackn_config(secure=True):
     # if not os.path.exists(os.path.expanduser('~/.scaleout/stackn.json')):
     #     print('You need to setup STACKn first.')
     #     login()
@@ -79,7 +78,7 @@ def get_stackn_config():
     if not load_status:
         print('Failed to load stackn config file.')
         print('You need to setup STACKn first.')
-        login()
+        login(secure=secure)
         stackn_config, load_status = load_from_file('stackn', os.path.expanduser('~/.scaleout'))
         # return None
 
@@ -100,7 +99,7 @@ def get_remote_config():
         print('No active project: Create a new project or set an existing project.')
         return [], False
 
-def get_token(client_id='studio-api', realm='STACKn'):
+def get_token(client_id='studio-api', realm='STACKn', secure=True):
     # stackn_config, load_status = load_from_file('stackn', os.path.expanduser('~/.scaleout/'))
     # if not load_status:
     #     print('Failed to load stackn config file.')
@@ -124,11 +123,11 @@ def get_token(client_id='studio-api', realm='STACKn'):
         # print('Token valid for user '+access_token_json['preferred_username'])
     except:
         # Try to refresh token
-        token_url = '{}/auth/realms/{}/protocol/openid-connect/token'.format(token_config['keycloak_url'], realm)
+        token_url = os.path.join(token_config['keycloak_url'], 'auth/realms/{}/protocol/openid-connect/token'.format(realm))
         req = {'grant_type': 'refresh_token',
                'client_id': client_id,
                'refresh_token': token_config['refresh_token']}
-        res = requests.post(token_url, data=req)
+        res = requests.post(token_url, data=req, verify=secure)
         resp = res.json()
         if 'access_token' in resp:
             access_token = resp['access_token']
@@ -149,7 +148,7 @@ def get_token(client_id='studio-api', realm='STACKn'):
 
 
 
-def login(client_id='studio-api', realm='STACKn', deployment=[], keycloak_host=[], studio_host=[]):
+def login(client_id='studio-api', realm='STACKn', deployment=[], keycloak_host=[], studio_host=[], secure=True):
     """ Login to Studio services. """
     if not deployment:
         deployment = input('Name: ')
@@ -159,7 +158,7 @@ def login(client_id='studio-api', realm='STACKn', deployment=[], keycloak_host=[
         studio_host = input('Studio host: ')
     username = input('Username: ')
     password = getpass()
-    access_token, refresh_token, public_key = keycloak_user_auth(username, password, keycloak_host)
+    access_token, refresh_token, public_key = keycloak_user_auth(username, password, keycloak_host, secure=secure)
     # dirname = base64.urlsafe_b64encode(host.encode("utf-8")).decode("utf-8")
     dirpath = os.path.expanduser('~/.scaleout/'+deployment)
     if not os.path.exists(dirpath):

cli/scaleout/cli/__init__.py

@@ -4,6 +4,7 @@
 from .create_cmd import create_cmd
 from .get_cmd import get_cmd
 from .delete_cmd import delete_cmd
-from .login_cmd import setup_cmd, status_cmd
+from .stackn_cmd import setup_cmd, status_cmd, predict_cmd
 from .set_cmd import set_cmd
-from .update_cmd import update_cmd
+from .update_cmd import update_cmd
+from .init_cmd import init_cmd

cli/scaleout/cli/create_cmd.py

@@ -19,7 +19,7 @@ def create_cmd(ctx, daemon):
       print('{} NYI should run as daemon...'.format(__file__))
 
 @create_cmd.command('model')
-@click.option('-f', '--model-file', required=True)
+@click.option('-f', '--model-file', required=False, default="")
 @click.option('-n', '--model-name', required=True)
 @click.option('-r', '--release-type', required=False)
 @click.option('-d', '--description', required=False,default="")
@@ -43,10 +43,11 @@ def create_deployment_definition(ctx, name, filepath, path_predict=''):
 @click.option('-m', '--model', required=True)
 @click.option('-v', '--model-version', default='latest')
 @click.option('-d', '--deploymentdefinition', required=True)
+@click.option('-s', '--settings', required=False)
 @click.pass_context
-def create_deployment_cmd(ctx, model, deploymentdefinition, model_version=[]):
+def create_deployment_cmd(ctx, model, deploymentdefinition, model_version=[], settings=[]):
     client = ctx.obj['CLIENT']
-    client.deploy_model(model, model_version, deploymentdefinition)
+    client.deploy_model(model, model_version, deploymentdefinition, settings)
 
 
 # Create project
@@ -59,4 +60,29 @@ def create_project_cmd(ctx, name, description='', repository=''):
     client = ctx.obj['CLIENT']
     client.create_project(name, description, repository)
 
-# Create dataset
+@create_cmd.command('lab')
+@click.option('-f', '--flavor', required=True)
+@click.option('-e', '--environment', required=True)
+@click.pass_context
+def create_session(ctx, flavor, environment):
+    client = ctx.obj['CLIENT']
+    client.create_session(flavor_slug=flavor, environment_slug=environment)
+
+
+# Create dataset
+
+
+@create_cmd.command('dataset')
+@click.option('-n', '--name', required=True)
+@click.option('-f', '--filenames', required=False)
+@click.option('-d', '--directory', required=False)
+@click.option('-r', '--release_type', required=False, default='minor')
+@click.pass_context
+def create_dataset(ctx, name, directory=[], filenames=[], release_type='minor', description='', bucket='dataset'):
+    client = ctx.obj['CLIENT']
+    client.create_dataset(name,
+                          release_type,
+                          filenames,
+                          directory,
+                          description=description,
+                          bucket=bucket)

cli/scaleout/cli/delete_cmd.py

@@ -35,6 +35,15 @@ def delete_deployment_cmd(ctx, name, version=None):
     client = ctx.obj['CLIENT']
     client.delete_deployment(name, version)
 
+@delete_cmd.command('dataset')
+@click.option('-n', '--name', required=True)
+@click.option('-v', '--version', required=True)
+@click.pass_context
+def delete_dataset_cmd(ctx, name, version=None):
+    """ Delete a model """
+    client = ctx.obj['CLIENT']
+    client.delete_dataset(name, version)
+
 # @delete_cmd.command('deployments')
 # @click.pass_context
 # def delete_deployment_cmd(ctx):

cli/scaleout/cli/get_cmd.py

@@ -48,6 +48,29 @@ def get_projects_cmd(ctx):
     keys = ["name", "created_at", "updated_at"]
     create_table(ctx, "projects", names, keys)
 
+@get_cmd.command('labs')
+@click.pass_context
+def lab_list_all_cmd(ctx):
+    """ List all Lab Sessions. """
+    names = ["Name", "Flavor", "Environment", "Status", "Created"]
+    keys = ["name", "flavor_slug", "environment_slug", "status", "created_at"]
+    create_table(ctx, "labs", names, keys)
+
+@get_cmd.command('members')
+@click.pass_context
+def members_list_cmd(ctx):
+    """ List all project members. """
+    names = ["Username"]
+    keys = ["username"]
+    create_table(ctx, "members", names, keys)
+
+@get_cmd.command('dataset')
+@click.pass_context
+def dataset_list_cmd(ctx):
+    """ List all project members. """
+    names = ["Name", "Version", "Release", "Project", "Created", "Created by"]
+    keys = ["name", "version", "release_type", "project_slug", "created_on", "created_by"]
+    create_table(ctx, "dataset", names, keys)
 
 # alliance
 

cli/scaleout/cli/init_cmd.py

@@ -0,0 +1,15 @@
+import click
+from .main import main
+import requests
+import os
+import tarfile
+
+@main.command('init')
+
+@click.pass_context
+def init_cmd(ctx):
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    file_path = os.path.join(dir_path, 'default-project.tar.gz')
+    import tarfile
+    tf = tarfile.open(file_path)
+    tf.extractall()

cli/scaleout/cli/main.py

@@ -1,6 +1,8 @@
 import click
 
 import logging
+from .. import version
+
 
 logging.basicConfig(format='%(asctime)s [%(filename)s:%(lineno)d] %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')  # , level=logging.DEBUG)
 
@@ -20,13 +22,17 @@
     ),
     type=click.Path(exists=False, dir_okay=True),
 )
+@click.version_option(
+    version=version.__version__,
+    prog_name='Scaleout STACKn CLI',
+    message='%(prog)s, %(version)s'
+)
 
 @click.pass_context
 def main(ctx, project_dir):
     ctx.obj = dict()
     ctx.obj['PROJECT_DIR'] = project_dir
-
-    if ctx.invoked_subcommand not in ('init','login','status','setup'):
+    if ctx.invoked_subcommand not in ('init','login','status','setup', 'set'):
         # TODO add support for cwd change, config-file specification
         from scaleout.project import Project
         from scaleout.runtime.runtime import Runtime

cli/scaleout/cli/project_cmd.py

@@ -1,10 +1,9 @@
 import click
-import os
+
 from .main import main
 from prettytable import PrettyTable
-
 from scaleout.project import init_project as init_p
-from scaleout.studioclient import StudioClient
+
 
 def init_project(project_dir):
     init_p(project_dir)
@@ -89,4 +88,30 @@ def project_get_deployment_definition(ctx, name):
     x = PrettyTable()
     x.field_names = ["Name","Definition","Bucket","File"]
     x.add_row([dd[0]["name"],dd[0]["definition"],dd[0]["bucket"],dd[0]["filename"]])
-    print(x)
+    print(x)
+
+
+@project_cmd.group('add')
+@click.pass_context
+def project_add_cmd(ctx):
+    pass
+
+@project_add_cmd.command('members')
+@click.option('-u', '--users', required=True)
+@click.option('-r', '--role', required=False, default='member')
+@click.pass_context
+def project_add_members(ctx, users, role):
+    client = ctx.obj['CLIENT']
+    client.add_members(users, role)
+
+@project_cmd.group('remove')
+@click.pass_context
+def project_remove_cmd(ctx):
+    pass
+
+@project_remove_cmd.command('members')
+@click.option('-u', '--users', required=True)
+@click.pass_context
+def project_remove_members(ctx, users):
+    client = ctx.obj['CLIENT']
+    client.remove_members(users)

cli/scaleout/cli/set_cmd.py

@@ -37,12 +37,20 @@ def remote_cmd(ctx, remote):
             sauth.write_stackn_config(stackn_config)
             print('New context: '+remote)
 
+@set_cmd.command('mode')
+@click.option('--secure/--insecure', default=True)
+@click.pass_context
+def secure_cmd(ctx, secure):
+    stackn_config, load_status = sauth.get_stackn_config()
+    stackn_config['secure'] = secure
+    sauth.write_stackn_config(stackn_config)
+
 @set_cmd.command('project')
 @click.option('-p', '--project', required=False)
 @click.pass_context
 def project_cmd(ctx, project):
     if not project:
         project = input('Project: ')
-
-    client = ctx.obj['CLIENT']
+    from scaleout.studioclient import StudioClient
+    client = StudioClient()
     client.set_project(project)