Merge pull request #86 from eth-cscs/dev

Merging from dev to master

ci/pre-prod/run_tests.yml

@@ -20,6 +20,9 @@
   #environment:
   #  PATH: "{{ ansible_env.PATH }}:/home/centos/.local/bin"
   tasks:
+    - name: Upgrade pip3
+      shell: pip3 install --user --upgrade pip==21.0.1
+
     - name: Install pytest modules
       shell: pip3 install --user -r requirements.txt
       args:

deploy/demo/ssl/f7t_internal.crt

@@ -1,8 +1,8 @@
 -----BEGIN CERTIFICATE-----
-MIIFojCCA4qgAwIBAgIUPPBK8HvPTkdgfjFWQq8DVOPyTb0wDQYJKoZIhvcNAQEL
+MIIFqDCCA5CgAwIBAgIUDl8L8XPoiiqkxAbFgkG93x5Q2ZQwDQYJKoZIhvcNAQEL
 BQAwZjELMAkGA1UEBhMCQ0gxDzANBgNVBAgMBlRpY2lubzEPMA0GA1UEBwwGTHVn
 YW5vMQ0wCwYDVQQKDARDU0NTMRIwEAYDVQQLDAlGaXJlY1JFU1QxEjAQBgNVBAMM
-CTEyNy4wLjAuMTAeFw0yMDExMTMxNjEwMjRaFw0zMDExMTExNjEwMjRaMGYxCzAJ
+CTEyNy4wLjAuMTAeFw0yMTAzMDExMTAwNTZaFw0zMTAyMjcxMTAwNTZaMGYxCzAJ
 BgNVBAYTAkNIMQ8wDQYDVQQIDAZUaWNpbm8xDzANBgNVBAcMBkx1Z2FubzENMAsG
 A1UECgwEQ1NDUzESMBAGA1UECwwJRmlyZWNSRVNUMRIwEAYDVQQDDAkxMjcuMC4w
 LjEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC/dGgWlhOE2NL22+zU
@@ -16,18 +16,18 @@ GjWiij2O8kcZ58x7q+Y43t5u3xTyl03gVPHliyjVpC78f5AlEH40cGUSyTztb8c5
 OQ2Y9t3R2XiN6w+fa7SuxyDuHCLIcP4qKBaDgB1kbFCf+cFMNC1ze810pkFTNX2J
 0G6zHuWcAXPVVCaKkf1QrKOgBzoCwJmvYBQ2QSn5/M9tVCLhyA8jiP2+a1eKWEVJ
 sW/L8Sx5ur1Cy6wGkA3L1m2DHi/gXNYNU5TSIDGX+y5JYCSpXVVOvcPxSwx3m4BC
-CmaczBdQn6bF0q0nAsgQJq1qowIDAQABo0gwRjBEBgNVHREEPTA7gglsb2NhbGhv
-c3SHBMCo3AuHBMCo3AmHBMCo3ASHBMCo3AWHBMCo3AaHBMCo3AeHBMCo3CiHBH8A
-AAEwDQYJKoZIhvcNAQELBQADggIBADbmLjRBLtAKYgOabjo120nI0rtXOja6Na5t
-2hgnlce/h4/Ir79761Ox3UkFF9D8vQSDibvWUyOqWoAqS7UgZ6wk0JZhk+3Xig4a
-z0ArkzTneDl/M7C1e02GAx4JWxcZb+ET0sSKDWGOJATZCWsXaE6GhCZO0FDHH512
-zI0/DUUAfPtTZqBmwdzcdE1QRO6cO/gNUJHIRdi+yTuCMB0Mlj1t2nYrF7JPCetL
-SkmSvpAtAgqE3D0oOs0GqlyzY2BNwwMVRT02wnEvPCY4lHtXgOtRcD1W2GUZ+1e4
-7xD+WvIc+BS+gRGYjPdB3j+yBo2xb2Nbbb6LUdeNBUqovRrB77IxXBLICwx2tgip
-fFPQPAFtSrHbrbLm4GwMGqjPs8bEDJtdWggDThoE6gD9QrYa74YY0thX8CXKy2wn
-NYbhF2ICSp3wn4mfxHgmntT394sLi5Aah4Pk33gN1qv/fOj2SMT0B2NEtNIL4MQQ
-WkoSy6bDugUNRinTqMLvUqgmp7sNOhAZdN/7tjIsezWj4Ykvq8p/BKSs2nhnuFtw
-27wcdHJEvFfAcnK21WNQQ5hKQMfxYrWnZ2VNQ0VK4kr6X5SxEwvLf4eu6A8D5EI7
-Yru1VRH1edGjWQ8P9HTllhhrC7/QaZJYDO4Bf90YRLNqF7TCCp8kDHd6OiDJHWng
-MoxMs8E+
+CmaczBdQn6bF0q0nAsgQJq1qowIDAQABo04wTDBKBgNVHREEQzBBgglsb2NhbGhv
+c3SHBMCo3AuHBMCo3AmHBMCo3ASHBMCo3AWHBMCo3AaHBMCo3AeHBMCo3AiHBMCo
+3CiHBH8AAAEwDQYJKoZIhvcNAQELBQADggIBAD977LacT/KqhCajm4GayB+Bucyw
+chit/H5D1YmMfedod1xDhI+T6pS0jILYlaUqtTCJQMae6FSGWg1BgY67yJZOsRQM
+2d5l8aII28zd7Ku8GAMW94HkhmHTu2EEW2w+yjdkB82L71GOakx7MrcOLNzx2WiI
+DRxvTSiQRORtr+lErORaFp86aBBAHQrCvy0ImUfvJwWdA6BpEz6OAz0iw7kOoE8q
+36nyRsurk43drPdELm4UxTzwUsgl4Ml2q3NSBBEaR1eOhvav6Oan3hdaRshUkA+T
+lkiC+MTgbiFp60QKgsXzSE2LFd9D4y3effWj2B5SYxKBjVKcn7OfNTg7EMvkUxeY
+C0NQqnYjtFAXatMIMzIR6SWwPDnyqi5SAseG1xc2696/EbnUQ8+e5+WnzIIG2w2N
+9j9s7kQKFwjiF23coOmdgE5aLDYUNe5e4ChE+TBR31JEWVFqP8DqzDPzOmRkSa+U
+LaJTyncDR5a1r3GLeFjQ7oi/1tT6jryYzUX5TclXlZZ5LXEYhbClPY0yKH1NOTgG
+IQiU3yw9TW+LEUooWEVmfuvdW4dGEzoYyTzss7xR+sQBgCA8bcm3+MTd0AsWpHs+
+ucKw0sGevdl1OWDEJ76uZYkEZGReDnxqoWoTQIWtTINrcoJ2d0m6RdLpp4hdm6kN
+pn2lF3QeQZkntTKf
 -----END CERTIFICATE-----

deploy/docker/base/requirements.txt

@@ -1,4 +1,4 @@
-cryptography==2.8
+cryptography==3.4.6
 Flask==1.1.2
 PyJWT==1.7.1
 requests==2.22.0

deploy/test-build/cluster/Dockerfile

@@ -59,6 +59,19 @@ RUN set -x \
     && chown -R slurm:slurm /var/log/slurm   \
     && chown -R slurm:slurm /var/spool/slurm*
 
+
+ADD cluster/slurm/*.conf /etc/slurm/
+ADD cluster/slurm/*.sh /
+RUN chmod 644 /etc/slurm/* && chmod 755 /*.sh
+
+RUN mkdir /etc/slurm/plugstack.conf.d
+RUN chmod 755 /etc/slurm/plugstack.conf.d
+ADD cluster/slurm/plugstack.conf.d/cscs-slurm-nohome.conf /etc/slurm/plugstack.conf.d/
+
+RUN mkdir /spank
+ADD cluster/spank/nohome-1.*.rpm /spank/
+RUN rpm -i /spank/nohome-*.rpm
+
 RUN set -x \
     && /sbin/create-munge-key
 
@@ -68,10 +81,8 @@ ADD cluster/supervisord.conf /etc/supervisord.conf
 
 RUN chown -R munge:munge /var/log/munge && chmod 755 /var/log/munge && chmod 755 /run/munge
 
-ADD cluster/slurm/*.conf /etc/slurm/
-ADD cluster/slurm/*.sh /
 
-RUN chmod 644 /etc/slurm/* && chmod 755 /*.sh
+RUN chmod 755 /*.sh
 
 RUN useradd -m -s /bin/bash test1 && useradd -m -s /bin/bash test2
 RUN echo 'test1:test11' | chpasswd  && echo 'test2:test22' | chpasswd

deploy/test-build/cluster/slurm/plugstack.conf

@@ -0,0 +1 @@
+include plugstack.conf.d/*

deploy/test-build/cluster/slurm/plugstack.conf.d/cscs-slurm-nohome.conf

@@ -0,0 +1 @@
+optional    /opt/cscs/nohome/nohome.so

deploy/test-build/cluster/spank/test.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# TODO: this test fails if root is group owner of some other ownership variations, a more sophisticated check that looks at folder contents should be implemented
+
+# with enabled $HOME the home directory is owned by the user
+ENABLED_HOME=$(sshpass -p test11 ssh [email protected] -p 2223 srun stat -c %G /home/test1)
+echo $ENABLED_HOME
+if [[ $ENABLED_HOME = "test1" ]]
+then
+    echo PASS
+else
+    echo FAILED
+fi
+
+# with disabled $HOME, the home directory is now group owned by root 
+DISABLED_HOME=$(sshpass -p test11 ssh [email protected] -p 2223 srun --nohome stat -c %G /home/test1)
+
+if [[ $DISABLED_HOME = "root" ]]
+then
+    echo PASS
+else
+    echo FAILED
+fi
+
+

deploy/test-build/docker-compose.yml

@@ -125,10 +125,13 @@ services:
       context: ./
       dockerfile: ./cluster/Dockerfile
     hostname: cluster
+    cap_add:
+      - SYS_ADMIN
     networks:
       - backend
       - frontend
 
+
   minio:
     # runs on private network so "cluster" can reach it
     container_name: minio_test_build

doc/openapi/firecrest-api.yaml

@@ -9,7 +9,7 @@ servers:
   - url: 'http://FIRECREST_URL'
   - url: 'https://FIRECREST_URL'
 info:
-  version: 1.7.0
+  version: 1.7.1-beta4
   title: FirecREST Developers API
   description: >
     This API specification is intended for FirecREST developers only. There're some endpoints that are not available in the public version for client developers.

doc/openapi/firecrest-developers-api.yaml

@@ -9,7 +9,7 @@ servers:
   - url: 'http://FIRECREST_URL'
   - url: 'https://FIRECREST_URL'
 info:
-  version: 1.7.0
+  version: 1.7.1-beta4
   title: FirecREST API
   description: >
     FirecREST platform, a RESTful Services Gateway to HPC resources, is a

src/certificator/certificator.py

@@ -14,6 +14,19 @@
 import base64
 import requests
 
+# Checks if an environment variable injected to F7T is a valid True value
+# var <- object
+# returns -> boolean
+def get_boolean_var(var):
+
+    # ensure variable to be a string
+    var = str(var)
+    # True, true or TRUE
+    # Yes, yes or YES
+    # 1
+
+    return var.upper() == "TRUE" or var.upper() == "YES" or var == "1"
+
 AUTH_HEADER_NAME = 'Authorization'
 
 AUTH_AUDIENCE = os.environ.get("F7T_AUTH_TOKEN_AUD", '').strip('\'"')
@@ -25,12 +38,12 @@
 CERTIFICATOR_PORT = os.environ.get("F7T_CERTIFICATOR_PORT", 5000)
 
 # OPA endpoint
-OPA_USE = os.environ.get("F7T_OPA_USE",False)
+OPA_USE = get_boolean_var(os.environ.get("F7T_OPA_USE",False))
 OPA_URL = os.environ.get("F7T_OPA_URL","http://localhost:8181").strip('\'"')
 POLICY_PATH = os.environ.get("F7T_POLICY_PATH","v1/data/f7t/authz").strip('\'"')
 
 ### SSL parameters
-USE_SSL = os.environ.get("F7T_USE_SSL", False)
+USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False))
 SSL_CRT = os.environ.get("F7T_SSL_CRT", "")
 SSL_KEY = os.environ.get("F7T_SSL_KEY", "")
 
@@ -41,7 +54,7 @@
     realm_pubkey = '-----BEGIN PUBLIC KEY-----\n' + realm_pubkey + '\n-----END PUBLIC KEY-----'
     realm_pubkey_type = os.environ.get("F7T_REALM_RSA_TYPE").strip('\'"')
 
-debug = os.environ.get("F7T_DEBUG_MODE", False)
+debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False))
 
 app = Flask(__name__)
 

src/common/cscs_api_common.py

@@ -20,7 +20,21 @@
 import io
 import time
 
-debug = os.environ.get("F7T_DEBUG_MODE", None)
+# Checks if an environment variable injected to F7T is a valid True value
+# var <- object
+# returns -> boolean
+def get_boolean_var(var):
+
+    # ensure variable to be a string
+    var = str(var)
+    # True, true or TRUE
+    # Yes, yes or YES
+    # 1
+
+    return var.upper() == "TRUE" or var.upper() == "YES" or var == "1"
+
+
+debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False))
 
 AUTH_HEADER_NAME = 'Authorization'
 
@@ -41,15 +55,15 @@
 CERTIFICATOR_URL = os.environ.get("F7T_CERTIFICATOR_URL")
 TASKS_URL = os.environ.get("F7T_TASKS_URL")
 
-F7T_SSH_CERTIFICATE_WRAPPER = os.environ.get("F7T_SSH_CERTIFICATE_WRAPPER", None)
+F7T_SSH_CERTIFICATE_WRAPPER = get_boolean_var(os.environ.get("F7T_SSH_CERTIFICATE_WRAPPER", False))
 
 # OPA endpoint
-OPA_USE = os.environ.get("F7T_OPA_USE",False)
+OPA_USE = get_boolean_var(os.environ.get("F7T_OPA_USE",False))
 OPA_URL = os.environ.get("F7T_OPA_URL","http://localhost:8181").strip('\'"')
 POLICY_PATH = os.environ.get("F7T_POLICY_PATH","v1/data/f7t/authz").strip('\'"')
 
 ### SSL parameters
-USE_SSL = os.environ.get("F7T_USE_SSL", False)
+USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False))
 SSL_CRT = os.environ.get("F7T_SSL_CRT", "")
 SSL_KEY = os.environ.get("F7T_SSL_KEY", "")
 
@@ -759,4 +773,7 @@ def check_command_error(error_str, error_code, service_msg):
         header = {"X-Permission-Denied": "User does not have permissions to access path"}
         return {"description": service_msg, "status_code": 400, "header": header}
     header = {"X-Error": error_str}
-    return {"description": service_msg, "error": error_str, "status_code": 400, "header": header}
+    return {"description": service_msg, "error": error_str, "status_code": 400, "header": header}
+
+
+

src/compute/compute.py

@@ -9,9 +9,12 @@
 from logging.handlers import TimedRotatingFileHandler
 import threading
 import async_task
+import traceback
+import sys
 
 from cscs_api_common import check_auth_header, get_username, \
-    exec_remote_command, create_task, update_task, clean_err_output, in_str, is_valid_file
+    exec_remote_command, create_task, update_task, clean_err_output, \
+    in_str, is_valid_file, get_boolean_var
 
 from job_time import check_sacctTime
 
@@ -37,7 +40,7 @@
 COMPUTE_PORT    = os.environ.get("F7T_COMPUTE_PORT", 5000)
 
 ### SSL parameters
-USE_SSL = os.environ.get("F7T_USE_SSL", False)
+USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False))
 SSL_CRT = os.environ.get("F7T_SSL_CRT", "")
 SSL_KEY = os.environ.get("F7T_SSL_KEY", "")
 
@@ -68,7 +71,7 @@
 # max content length for upload in bytes
 app.config['MAX_CONTENT_LENGTH'] = int(MAX_FILE_SIZE) * 1024 * 1024
 
-debug = os.environ.get("F7T_DEBUG_MODE", None)
+debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False))
 
 
 def is_jobid(jobid):
@@ -194,13 +197,14 @@ def submit_job_task(auth_header, system_name, system_addr, job_file, job_dir, ta
         update_task(task_id, auth_header, async_task.SUCCESS, job_extra_info, True)
 
     except IOError as e:
-        app.logger.error(e.filename)
+        app.logger.error(e.filename, exc_info=True, stack_info=True)
         app.logger.error(e.strerror)
         update_task(task_id, auth_header,async_task.ERROR, e.message)
     except Exception as e:
-        app.logger.error(type(e))
+        app.logger.error(type(e), exc_info=True, stack_info=True)
         app.logger.error(e)
-        update_task(task_id, auth_header, async_task.ERROR, e.message)
+        traceback.print_exc(file=sys.stdout)
+        update_task(task_id, auth_header, async_task.ERROR)
 
 
 
@@ -242,13 +246,13 @@ def get_slurm_files(auth_header, system_name, system_addr, task_id,job_info,outp
     # if it's ok, we can add information
     control_resp = resp["msg"]
 
+    # tokens are expected to be space-separated and with a k=v form. See man scontrol show
     control_list = control_resp.split()
+    control_dict = { value.split("=")[0] : value.split("=")[1] for value in control_list if "=" in value }
 
-    control_dict = { value.split("=")[0] : value.split("=")[1] for value in control_list }
-
-    control_info["job_file_out"] = control_dict["StdOut"]
-    control_info["job_file_err"] = control_dict["StdErr"]
-    control_info["job_file"] = control_dict["Command"]
+    control_info["job_file_out"] = control_dict.get("StdOut", "stdout-file-not-found")
+    control_info["job_file_err"] = control_dict.get("StdErr", "stderr-file-not-found")
+    control_info["job_file"] = control_dict.get("Command", "command-not-found")
     control_info["job_data_out"] = ""
     control_info["job_data_err"] = ""
     # if all fine:
@@ -874,7 +878,7 @@ def cancel_job(jobid):
     action = f"scancel -v {jobid}"
 
     try:
-        # obtain new task from TASKS microservice
+        # obtain new task from TASKS microservice.
         task_id = create_task(auth_header,service="compute")
 
         # if error in creating task:

src/reservations/reservations.py

@@ -12,7 +12,7 @@
 import os
 import logging
 from logging.handlers import TimedRotatingFileHandler
-from cscs_api_common import check_auth_header, exec_remote_command, in_str
+from cscs_api_common import check_auth_header, exec_remote_command, in_str, get_boolean_var
 
 import re
 import datetime
@@ -31,13 +31,13 @@
 TIMEOUT = os.environ.get("F7T_UTILITIES_TIMEOUT", 5)
 
 ### SSL parameters
-USE_SSL = os.environ.get("F7T_USE_SSL", False)
+USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False))
 SSL_CRT = os.environ.get("F7T_SSL_CRT", "")
 SSL_KEY = os.environ.get("F7T_SSL_KEY", "")
 
 RESERVATION_CMD = os.environ.get("F7T_RESERVATION_CMD", "rsvmgmt")
 
-debug = os.environ.get("F7T_DEBUG_MODE", None)
+debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False))
 
 
 app = Flask(__name__)

src/status/status.py

@@ -11,7 +11,7 @@
 import multiprocessing as mp
 
 # common modules
-from cscs_api_common import check_auth_header
+from cscs_api_common import check_auth_header, get_boolean_var
 
 import paramiko
 import socket
@@ -34,7 +34,7 @@
 SERVICES_DICT = {}
 
 ### SSL parameters
-USE_SSL = os.environ.get("F7T_USE_SSL", False)
+USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False))
 SSL_CRT = os.environ.get("F7T_SSL_CRT", "")
 SSL_KEY = os.environ.get("F7T_SSL_KEY", "")
 
@@ -47,7 +47,7 @@
 OBJECT_STORAGE=os.environ.get("F7T_OBJECT_STORAGE")
 
 # debug on console
-debug = os.environ.get("F7T_DEBUG_MODE", None)
+debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False))
 
 
 app = Flask(__name__)

src/storage/s3v4OS.py

@@ -666,7 +666,7 @@ def delete_object_after(self,containername,prefix,objectname,ttl):
             resp = requests.put(url, data=body, headers=headers)
 
             if resp.ok:
-                logging.info("Object marked as delete-at succesfully")
+                logging.info(f"Object was marked as to be deleted at {_delete_at_iso}")
 
                 return 0