Merge pull request #1132 from uc-cdis/chore/sec

(PPS-635): chore(security): update urllib3 and related deps

docs/google_architecture.md

@@ -17,7 +17,7 @@ We'll talk about each one of those in-depth here (and even delve into the intern
 
 ### Fence -> cirrus -> Google: A library wrapping Google's API
 
-We have a library that wraps Google's public API called [cirrus](https://github.com/uc-cdis/cirrus). Our design is such that fence does not hit Google's API directly, but goes through cirrus. For all of cirrus's features to work, a very specific setup is required, which is detailed in cirrus's README.
+We have a library that wraps Google's public API called [cirrus](https://github.com/uc-cdis/cirrus). Our design is such that fence does not hit Google's API directly, but goes through gen3cirrus. For all of cirrus's features to work, a very specific setup is required, which is detailed in cirrus's README.
 
 Essentially, cirrus requires a Google Cloud Identity account (for group management) and
 Google Cloud Platform project(s). In order to automate group management in Google Cloud Identity with cirrus, you must go through a manual process of allowing API access and delegating a specific service account from a Google Cloud Platform project to have group management authority. Details can be found in cirrus's README.

fence/blueprints/data/indexd.py

@@ -6,8 +6,8 @@
 
 from sqlalchemy.sql.functions import user
 from cached_property import cached_property
-import cirrus
-from cirrus import GoogleCloudManager
+import gen3cirrus
+from gen3cirrus import GoogleCloudManager
 from cdislogging import get_logger
 from cdispyutils.config import get_value
 from cdispyutils.hmac4 import generate_aws_presigned_url
@@ -162,7 +162,7 @@ def get_signed_url_for_file(
     _log_signed_url_data_info(
         indexed_file=indexed_file,
         user_sub=flask.g.audit_data.get("sub", ""),
-        requested_protocol=requested_protocol
+        requested_protocol=requested_protocol,
     )
 
     return {"url": signed_url}
@@ -1197,7 +1197,7 @@ def _generate_anonymous_google_storage_signed_url(
     ):
         # we will use the main fence SA service account to sign anonymous requests
         private_key = get_google_app_creds()
-        final_url = cirrus.google_cloud.utils.get_signed_url(
+        final_url = gen3cirrus.google_cloud.utils.get_signed_url(
             resource_path,
             http_verb,
             expires_in,
@@ -1338,7 +1338,7 @@ def _generate_google_storage_signed_url(
         if config["BILLING_PROJECT_FOR_SIGNED_URLS"] and not r_pays_project:
             r_pays_project = config["BILLING_PROJECT_FOR_SIGNED_URLS"]
 
-        final_url = cirrus.google_cloud.utils.get_signed_url(
+        final_url = gen3cirrus.google_cloud.utils.get_signed_url(
             resource_path,
             http_verb,
             expires_in,

fence/blueprints/google.py

@@ -7,9 +7,9 @@
 import flask
 from flask_restful import Resource
 
-from cirrus import GoogleCloudManager
-from cirrus.errors import CirrusNotFound
-from cirrus.google_cloud.errors import GoogleAPIError
+from gen3cirrus import GoogleCloudManager
+from gen3cirrus.errors import CirrusNotFound
+from gen3cirrus.google_cloud.errors import GoogleAPIError
 
 from fence.auth import current_token, require_auth_header
 from fence.restful import RestfulApi

fence/blueprints/link.py

@@ -6,7 +6,7 @@
 
 from cdislogging import get_logger
 
-from cirrus import GoogleCloudManager
+from gen3cirrus import GoogleCloudManager
 from fence.blueprints.login.redirect import validate_redirect
 from fence.restful import RestfulApi
 from fence.errors import NotFound

fence/blueprints/storage_creds/google.py

@@ -6,8 +6,8 @@
 from flask_restful import Resource
 from flask import current_app
 
-from cirrus import GoogleCloudManager
-from cirrus.config import config as cirrus_config
+from gen3cirrus import GoogleCloudManager
+from gen3cirrus.config import config as cirrus_config
 
 from fence.config import config
 from fence.auth import require_auth_header

fence/config-default.yaml

@@ -691,6 +691,10 @@ GS_BUCKETS: {}
 #   bucket3:
 #     region: 'us-east-1'
 
+# When using the Cleversafe storageclient, whether or not to send verify=true 
+# for requests
+VERIFY_CLEVERSAFE_CERT: true
+
 # Names of the S3 buckets to which data files can be uploaded. They should be
 # configured in `S3_BUCKETS`.
 ALLOWED_DATA_UPLOAD_BUCKETS: []

fence/config.py

@@ -2,7 +2,7 @@
 from yaml import safe_load as yaml_load
 import urllib.parse
 
-import cirrus
+import gen3cirrus
 from gen3config import Config
 
 from cdislogging import get_logger
@@ -92,7 +92,7 @@ def post_process(self):
         if self._configs.get("MOCK_STORAGE", False):
             self._configs["STORAGE_CREDENTIALS"] = {}
 
-        cirrus.config.config.update(**self._configs.get("CIRRUS_CFG", {}))
+        gen3cirrus.config.config.update(**self._configs.get("CIRRUS_CFG", {}))
 
         # if we have a default google project for billing requester pays, we should
         # NOT allow end-users to have permission to create Temporary Google Service

fence/resources/admin/admin_users.py

@@ -1,6 +1,6 @@
 from cdislogging import get_logger
-from cirrus import GoogleCloudManager
-from cirrus.google_cloud.utils import get_proxy_group_name_for_user
+from gen3cirrus import GoogleCloudManager
+from gen3cirrus.google_cloud.utils import get_proxy_group_name_for_user
 from fence.config import config
 from fence.errors import NotFound, UserError, UnavailableError
 from fence.models import (
@@ -363,7 +363,7 @@ def delete_user(current_session, username):
             # and check if it exists in cirrus, in case Fence db just
             # didn't know about it.
             logger.debug(
-                "Could not find Google proxy group for this user in Fence db. Checking cirrus..."
+                "Could not find Google proxy group for this user in Fence db. Checking gen3cirrus..."
             )
             pgname = get_proxy_group_name_for_user(
                 user.id, user.username, prefix=config["GOOGLE_GROUP_PREFIX"]
@@ -377,7 +377,7 @@ def delete_user(current_session, username):
 
         if not gpg_email:
             logger.info(
-                "Could not find Google proxy group for user in Fence db or in cirrus. "
+                "Could not find Google proxy group for user in Fence db or in gen3cirrus. "
                 "Assuming Google not in use as IdP. Proceeding with Fence deletes."
             )
         else:

fence/resources/google/access_utils.py

@@ -8,10 +8,10 @@
 from urllib.parse import unquote
 import traceback
 
-from cirrus.google_cloud.iam import GooglePolicyMember
-from cirrus.google_cloud.errors import GoogleAPIError
-from cirrus.google_cloud.iam import GooglePolicy
-from cirrus import GoogleCloudManager
+from gen3cirrus.google_cloud.iam import GooglePolicyMember
+from gen3cirrus.google_cloud.errors import GoogleAPIError
+from gen3cirrus.google_cloud.iam import GooglePolicy
+from gen3cirrus import GoogleCloudManager
 
 import fence
 from cdislogging import get_logger
@@ -218,7 +218,7 @@ def get_google_project_valid_users_and_service_accounts(
             Will make call to Google API if membership is None
 
     Return:
-        List[cirrus.google_cloud.iam.GooglePolicyMember]: Members on the
+        List[gen3cirrus.google_cloud.iam.GooglePolicyMember]: Members on the
             google project
 
     Raises:

fence/resources/google/utils.py

@@ -9,9 +9,9 @@
 from sqlalchemy import desc, func
 
 from cdislogging import get_logger
-from cirrus import GoogleCloudManager
-from cirrus.google_cloud.iam import GooglePolicyMember
-from cirrus.google_cloud.utils import (
+from gen3cirrus import GoogleCloudManager
+from gen3cirrus.google_cloud.iam import GooglePolicyMember
+from gen3cirrus.google_cloud.utils import (
     get_valid_service_account_id_for_client,
     get_valid_service_account_id_for_user,
 )

fence/resources/google/validity.py

@@ -27,7 +27,7 @@
     is_user_member_of_google_project,
     is_user_member_of_all_google_projects,
 )
-from cirrus.google_cloud import GoogleCloudManager
+from gen3cirrus.google_cloud import GoogleCloudManager
 
 from cdislogging import get_logger
 

fence/resources/storage/__init__.py

@@ -1,7 +1,7 @@
 import copy
 from functools import wraps
 
-from storageclient import get_client
+from fence.resources.storage.storageclient import get_client
 
 from fence.models import (
     CloudProvider,

fence/resources/storage/storageclient/__init__.py

@@ -0,0 +1,12 @@
+from fence.resources.storage.storageclient.cleversafe import CleversafeClient
+from fence.resources.storage.storageclient.google import GoogleCloudStorageClient
+
+
+def get_client(config=None, backend=None):
+    try:
+        clients = {"cleversafe": CleversafeClient, "google": GoogleCloudStorageClient}
+        return clients[backend](config)
+    except KeyError as ex:
+        raise NotImplementedError(
+            "The input storage is currently not supported!: {0}".format(ex)
+        )