Merge branch 'main' into wip/support-us-gov-east-1

Author: hehe7318

.github/workflows/demo_deploy.yml

@@ -13,7 +13,7 @@ concurrency: ci-${{ github.ref }}
 
 jobs:
   frontend-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repo
@@ -43,7 +43,7 @@ jobs:
         working-directory: ./frontend
 
   backend-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repo
@@ -62,7 +62,7 @@ jobs:
         run: pytest
 
   update-infrastructure:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     needs: [frontend-tests, backend-tests]
 
@@ -80,7 +80,7 @@ jobs:
         run: ./infrastructure/update-environment-infra.sh demo
 
   build-and-deploy:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     needs: [update-infrastructure]
 

.github/workflows/production_release.yml

@@ -10,7 +10,7 @@ permissions:
 
 jobs:
   frontend-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repo
@@ -40,7 +40,7 @@ jobs:
         working-directory: ./frontend
 
   backend-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repo
@@ -59,7 +59,7 @@ jobs:
         run: pytest
 
   release:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     needs: [frontend-tests, backend-tests]
 

.github/workflows/pull_request.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   frontend-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repo
@@ -37,7 +37,7 @@ jobs:
         working-directory: ./frontend
 
   backend-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repo

Dockerfile

@@ -1,4 +1,4 @@
-FROM tiangolo/uwsgi-nginx-flask:python3.8-alpine
+FROM tiangolo/uwsgi-nginx-flask:python3.12-alpine
 
 ENV PYTHONUNBUFFERED=1
 ENV STATIC_URL /static

Dockerfile.awslambda

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/lambda/python:3.8-x86_64
+FROM public.ecr.aws/lambda/python:3.12-x86_64
 
 COPY --from=frontend-awslambda /app/build ${LAMBDA_TASK_ROOT}/frontend/public
 

api/PclusterApiHandler.py

@@ -25,20 +25,21 @@
 from api.pcm_globals import set_auth_cookies_in_context, logger, auth_cookies
 from api.security.csrf.constants import CSRF_COOKIE_NAME
 from api.security.csrf.csrf import csrf_needed
-from api.utils import disable_auth
+from api.utils import disable_auth, read_and_delete_ssm_output_from_cloudwatch
 from api.validation import validated
 from api.validation.schemas import PCProxyArgs, PCProxyBody
 
 USER_POOL_ID = os.getenv("USER_POOL_ID")
 AUTH_PATH = os.getenv("AUTH_PATH")
 API_BASE_URL = os.getenv("API_BASE_URL")
-API_VERSION = os.getenv("API_VERSION", "3.1.0")
+API_VERSION = sorted(set(os.getenv("API_VERSION", "3.1.0").strip().split(",")), key=lambda x: [-int(n) for n in x.split('.')])
+# Default version must be highest version so that it can be used for read operations due to backwards compatibility
+DEFAULT_API_VERSION = API_VERSION[0]
 API_USER_ROLE = os.getenv("API_USER_ROLE")
 OIDC_PROVIDER = os.getenv("OIDC_PROVIDER")
 CLIENT_ID = os.getenv("CLIENT_ID")
 CLIENT_SECRET = os.getenv("CLIENT_SECRET")
 SECRET_ID = os.getenv("SECRET_ID")
-SITE_URL = os.getenv("SITE_URL", API_BASE_URL)
 SCOPES_LIST = os.getenv("SCOPES_LIST")
 REGION = os.getenv("AWS_DEFAULT_REGION")
 TOKEN_URL = os.getenv("TOKEN_URL", f"{AUTH_PATH}/oauth2/token")
@@ -47,6 +48,8 @@
 JWKS_URL = os.getenv("JWKS_URL")
 AUDIENCE = os.getenv("AUDIENCE")
 USER_ROLES_CLAIM = os.getenv("USER_ROLES_CLAIM", "cognito:groups")
+SSM_LOG_GROUP_NAME = os.getenv("SSM_LOG_GROUP_NAME")
+ARG_VERSION="version"
 
 try:
     if (not USER_POOL_ID or USER_POOL_ID == "") and SECRET_ID:
@@ -62,6 +65,19 @@
     JWKS_URL = os.getenv("JWKS_URL",
                          f"https://cognito-idp.{REGION}.amazonaws.com/{USER_POOL_ID}/" ".well-known/jwks.json")
 
+def create_url_map(url_list):
+    url_map = {}
+    if url_list:
+        for url in url_list.split(","):
+            if url:
+                pair=url.split("=")
+                url_map[pair[0]] = pair[1]
+    return url_map
+
+API_BASE_URL_MAPPING = create_url_map(API_BASE_URL)
+SITE_URL = os.getenv("SITE_URL", API_BASE_URL_MAPPING.get(DEFAULT_API_VERSION))
+
+
 
 def jwt_decode(token, audience=None, access_token=None):
     return jwt.decode(
@@ -164,7 +180,7 @@ def authenticate(groups):
 
     if (not groups):
         return abort(403)
-        
+
     jwt_roles = set(decoded.get(USER_ROLES_CLAIM, []))
     groups_granted = groups.intersection(jwt_roles)
     if len(groups_granted) == 0:
@@ -190,7 +206,7 @@ def get_scopes_list():
 
 def get_redirect_uri():
   return f"{SITE_URL}/login"
-  
+
 # Local Endpoints
 
 
@@ -232,9 +248,9 @@ def ec2_action():
 def get_cluster_config_text(cluster_name, region=None):
     url = f"/v3/clusters/{cluster_name}"
     if region:
-        info_resp = sigv4_request("GET", API_BASE_URL, url, params={"region": region})
+        info_resp = sigv4_request("GET", get_base_url(request), url, params={"region": region})
     else:
-        info_resp = sigv4_request("GET", API_BASE_URL, url)
+        info_resp = sigv4_request("GET", get_base_url(request), url)
     if info_resp.status_code != 200:
         abort(info_resp.status_code)
 
@@ -264,10 +280,16 @@ def ssm_command(region, instance_id, user, run_command):
         DocumentName="AWS-RunShellScript",
         Comment=f"Run ssm command.",
         Parameters={"commands": [command]},
+        CloudWatchOutputConfig={
+            'CloudWatchLogGroupName': SSM_LOG_GROUP_NAME,
+            'CloudWatchOutputEnabled': True
+        },
     )
 
     command_id = ssm_resp["Command"]["CommandId"]
 
+    logger.info(f"Submitted SSM command {command_id}")
+
     # Wait for command to complete
     time.sleep(0.75)
     while time.time() - start < 60:
@@ -282,7 +304,13 @@ def ssm_command(region, instance_id, user, run_command):
     if status["Status"] != "Success":
         raise Exception(status["StandardErrorContent"])
 
-    output = status["StandardOutputContent"]
+    output = read_and_delete_ssm_output_from_cloudwatch(
+        region=region,
+        log_group_name=SSM_LOG_GROUP_NAME,
+        command_id=command_id,
+        instance_id=instance_id,
+    )
+
     return output
 
 
@@ -352,7 +380,7 @@ def sacct():
             user,
             f"sacct {sacct_args} --json "
             + "| jq -c .jobs[0:120]\\|\\map\\({name,user,partition,state,job_id,exit_code\\}\\)",
-        )
+            )
         if type(accounting) is tuple:
             return accounting
     else:
@@ -471,7 +499,7 @@ def get_dcv_session():
 
 
 def get_custom_image_config():
-    image_info = sigv4_request("GET", API_BASE_URL, f"/v3/images/custom/{request.args.get('image_id')}").json()
+    image_info = sigv4_request("GET", get_base_url(request), f"/v3/images/custom/{request.args.get('image_id')}").json()
     configuration = requests.get(image_info["imageConfiguration"]["url"])
     return configuration.text
 
@@ -553,13 +581,7 @@ def get_instance_types():
         ec2 = boto3.client("ec2", config=config)
     else:
         ec2 = boto3.client("ec2")
-    filters = [
-        {"Name": "current-generation", "Values": ["true"]},
-        {"Name": "instance-type",
-         "Values": [
-             "c5*", "c6*", "c7*", "g4*", "g5*", "g6*", "hpc*", "p3*", "p4*", "p5*", "t2*", "t3*", "m6*", "m7*", "r*"
-         ]},
-    ]
+    filters = [{"Name": "current-generation", "Values": ["true"]}]
     instance_paginator = ec2.get_paginator("describe_instance_types")
     instances_paginator = instance_paginator.paginate(Filters=filters)
     instance_types = []
@@ -583,9 +605,9 @@ def _get_identity_from_token(decoded, claims):
         identity["username"] = decoded["username"]
 
     for claim in claims:
-      if claim in decoded:
-        identity["attributes"][claim] = decoded[claim]
-    
+        if claim in decoded:
+            identity["attributes"][claim] = decoded[claim]
+
     return identity
 
 def get_identity():
@@ -722,14 +744,20 @@ def _get_params(_request):
     params.pop("path")
     return params
 
+def get_base_url(request):
+    version = request.args.get(ARG_VERSION)
+    if version and str(version) in API_VERSION:
+        return API_BASE_URL_MAPPING[str(version)]
+    return API_BASE_URL_MAPPING[DEFAULT_API_VERSION]
+
 
 pc = Blueprint('pc', __name__)
 
 @pc.get('/', strict_slashes=False)
 @authenticated({'admin'})
 @validated(params=PCProxyArgs)
 def pc_proxy_get():
-    response = sigv4_request(request.method, API_BASE_URL, request.args.get("path"), _get_params(request))
+    response = sigv4_request(request.method, get_base_url(request), request.args.get("path"), _get_params(request))
     return response.json(), response.status_code
 
 @pc.route('/', methods=['POST','PUT','PATCH','DELETE'], strict_slashes=False)
@@ -743,5 +771,5 @@ def pc_proxy():
     except:
         pass
 
-    response = sigv4_request(request.method, API_BASE_URL, request.args.get("path"), _get_params(request), body=body)
+    response = sigv4_request(request.method, get_base_url(request), request.args.get("path"), _get_params(request), body=body)
     return response.json(), response.status_code

api/tests/exceptions/test_exception_handlers.py

@@ -47,11 +47,11 @@ def test_unauthenticated_error_handler(client, app, monkeypatch):
 def test_validation_error_handler(client, app, monkeypatch):
     with app.test_request_context('/'):
         app.preprocess_request()
-        response, status_code = validation_error_handler(ValidationError('Input validation failed for /manager/ec2_action', data={'field': ['validation-error']}))
+        response, status_code = validation_error_handler(ValidationError('Input validation failed for requested resource /manager/ec2_action', data={'field': ['validation-error']}))
 
     assert status_code == 400
     assert response.json == {
-        'code': 400, 'message': 'Input validation failed for /manager/ec2_action',
+        'code': 400, 'message': 'Input validation failed for requested resource /manager/ec2_action',
         'validation_errors': {'field': ['validation-error']}
     }
 

api/tests/test_pcluster_api_handler.py

@@ -1,5 +1,10 @@
 from unittest import mock
-from api.PclusterApiHandler import login
+from api.PclusterApiHandler import login, get_base_url, create_url_map
+
+class MockRequest:
+    cookies = {'int_value': 100}
+    args = {'version': '3.12.0'}
+    json = {'username': '[email protected]'}
 
 
 @mock.patch("api.PclusterApiHandler.requests.post")
@@ -27,3 +32,14 @@ def test_login_with_no_access_token_returns_401(mocker, app):
         login()
 
         mock_abort.assert_called_once_with(401)
+
+def test_get_base_url(monkeypatch):
+    monkeypatch.setattr('api.PclusterApiHandler.API_VERSION', ['3.12.0', '3.11.0'])
+    monkeypatch.setattr('api.PclusterApiHandler.API_BASE_URL', '3.12.0=https://example.com,3.11.0=https://example1.com,')
+    monkeypatch.setattr('api.PclusterApiHandler.API_BASE_URL_MAPPING', {'3.12.0': 'https://example.com', '3.11.0': 'https://example1.com'})
+
+    assert 'https://example.com' == get_base_url(MockRequest())
+
+def test_create_url_map():
+    assert {'3.12.0': 'https://example.com', '3.11.0': 'https://example1.com'} == create_url_map('3.12.0=https://example.com,3.11.0=https://example1.com,')
+