Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle 'None' string in fence config #1161

Open
wants to merge 26 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
5dc1da9
adding default empty dict
AlbertSnows Jul 18, 2024
67dc533
adding sanity happy path test
AlbertSnows Jul 18, 2024
19384ad
redesigning code to handle none case without getting too much more co…
AlbertSnows Jul 18, 2024
9032344
gave a better name
AlbertSnows Jul 18, 2024
fdc937c
adding typing
AlbertSnows Jul 18, 2024
ed05a13
adding more state handling
AlbertSnows Jul 18, 2024
7c1ffe4
Trigger GitHub Actions workflow
AlbertSnows Jul 22, 2024
9ee8a78
Testing
k-burt-uch Jul 18, 2024
174424f
fix(PXP-11364): Fix failing dbgap sync unit test
k-burt-uch Jul 22, 2024
532dcc5
Trigger GitHub Actions workflow
AlbertSnows Jul 22, 2024
48ee3e0
Merge branch 'master' into fix/handle_none_string_v2
AlbertSnows Jul 23, 2024
8c4a9b7
moving to separate unit test
AlbertSnows Jul 23, 2024
5e51743
Merge branch 'master' into fix/handle_none_string_v2
AlbertSnows Jul 23, 2024
491140f
Merge branch 'master' into fix/handle_none_string_v2
AlbertSnows Jul 31, 2024
d9bc79f
Merge branch 'master' into fix/handle_none_string_v2
AlbertSnows Aug 7, 2024
a06ecaf
more documentation
AlbertSnows Aug 9, 2024
65edbd9
Merge branch 'master' into fix/handle_none_string_v2
Avantol13 Aug 9, 2024
2b63f57
more documentation
AlbertSnows Aug 9, 2024
95db2db
Merge remote-tracking branch 'origin/fix/handle_none_string_v2' into …
AlbertSnows Aug 12, 2024
8e60f67
cleaning up references in test fence create
AlbertSnows Aug 12, 2024
528feac
Merge branch 'master' into fix/handle_none_string_v2
AlbertSnows Aug 12, 2024
06cebff
documentation + minor rename
AlbertSnows Aug 12, 2024
af9c5b1
fix name, add description
AlbertSnows Sep 3, 2024
e2d5fd7
Merge branch 'master' into fix/handle_none_string_v2
AlbertSnows Sep 3, 2024
a7ec57a
Merge branch 'master' into fix/handle_none_string_v2
Avantol13 Oct 18, 2024
d44dd5a
Merge branch 'master' into fix/handle_none_string_v2
Avantol13 Oct 21, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fence/config-default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -558,7 +558,7 @@ dbGaP:
#
# If `parse_consent_code` is true, then a user will be given access to the exact
# same consent codes in the child studies
parent_to_child_studies_mapping:
parent_to_child_studies_mapping: {}
# 'phs001194': ['phs000571', 'phs001843']
# A consent of "c999" can indicate access to that study's "exchange area data"
# and when a user has access to one study's exchange area data, they
Expand Down
126 changes: 107 additions & 19 deletions fence/config.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import collections
import os
from yaml import safe_load as yaml_load
import urllib.parse
Expand All @@ -6,6 +7,7 @@
from gen3config import Config

from cdislogging import get_logger
from typing import List, Dict, Any

logger = get_logger(__name__)

Expand Down Expand Up @@ -150,28 +152,114 @@ def post_process(self):
logger.warning(
f"IdP '{idp_id}' is using multifactor_auth_claim_info '{mfa_info['claim']}', which is neither AMR or ACR. Unable to determine if a user used MFA. Fence will continue and assume they have not used MFA."
)
self._validate_dbgap_config(self._configs["dbGaP"])

self._validate_parent_child_studies(self._configs["dbGaP"])
@staticmethod
def _get_parent_studies_safely(dbgap_config):
"""
This will get a list of parent id's from the dbgap config's mapping property
or return and empty array if the entry doesn't exist
Args:
dbgap_config: { parent_to_child_studies_mapping: { k1: v1, ... } }
Returns:
[k1, k2, ...]
"""
study_mapping = dbgap_config.get('parent_to_child_studies_mapping', {})
# study mapping could be 'None'
safe_studies = list(study_mapping.keys()) if isinstance(study_mapping, dict) else []
return safe_studies

# TODO(fix-it): These functions should go in utils.py, but cannot be migrated until
# the variable DEFAULT_BACKOFF_SETTINGS is migrated here
@staticmethod
def _validate_parent_child_studies(dbgap_configs):
if isinstance(dbgap_configs, list):
configs = dbgap_configs
else:
configs = [dbgap_configs]

all_parent_studies = set()
for dbgap_config in configs:
parent_studies = dbgap_config.get(
"parent_to_child_studies_mapping", {}
).keys()
conflicts = parent_studies & all_parent_studies
if len(conflicts) > 0:
raise Exception(
f"{conflicts} are duplicate parent study ids found in parent_to_child_studies_mapping for "
f"multiple dbGaP configurations."
)
all_parent_studies.update(parent_studies)
def _some(predicate, iterable, found_nothing_value=None):
""" Returns the first element that satisfies the predicate, else fail value
Expects predicate to be a boolean lambda and iterable to be a collection
"""
for item in iterable:
result = predicate(item)
if result:
return item
return found_nothing_value

@staticmethod
def _coerce_to_array(unknown_data_structure, exception_message="Unrecognized data structure, aborting!"):
"""
Given a dubious data structure, coerces it into a list
depending on the data type.
Currently, handles list, dict, and None
Args:
exception_message: any string for more context
unknown_data_structure: either a list, dict, or None; fails otherwise

Returns: a list of some kind depending on the provided data structure

note: fails if the unknown data structure is any other type
"""
# TODO: these three functions can be move out on their own when migrated to utils.py
identity = lambda v: v
wrap_in_array = lambda v: [v]
empty_array = lambda _: []
# END TODO
data_is = lambda data_type: isinstance(unknown_data_structure, data_type)
type_to_coercion = {
list: identity,
dict: wrap_in_array,
type(None): empty_array,
}
data_type_of_parameter = FenceConfig._some(data_is, list(type_to_coercion.keys()), TypeError)
if data_type_of_parameter is TypeError:
raise ValueError(exception_message + f" | Type: {type(unknown_data_structure)}")
return type_to_coercion[data_type_of_parameter](unknown_data_structure)

@staticmethod
def _find_duplicates(collection):
"""

Args:
collection: any data type accepted by the Counter object

Returns:
a set of items that were found more than once in the collection

note: if collection is a dictionary, counter will treat it as a
duplicates mapping!
"""
item_with_occurrences = collections.Counter(collection)
duplicates = {item
for item, occurrences in item_with_occurrences.items()
if occurrences > 1}
return duplicates

# END TODO


@staticmethod
def _validate_parent_child_studies(dbgap_configs: List[Dict[str, Any]]) -> None:
"""
Given a list of dictionaries that contain a parent -> child study mapping,
this function will check that all parents are unique and not duplicated
Args:
dbgap_configs: [ { parent_to_child_studies_mapping: { k1: v1, ... } }, ... ]

Note: This function will throw if duplicates are found.
"""
safe_list_of_parent_studies = []
for dbgap_config in dbgap_configs:
safe_parent_studies = FenceConfig._get_parent_studies_safely(dbgap_config)
safe_list_of_parent_studies.extend(safe_parent_studies)

duplicates = FenceConfig._find_duplicates(safe_list_of_parent_studies)
if len(duplicates) > 0:
raise Exception(
f"{duplicates} are duplicate parent study ids found in parent_to_child_studies_mapping for "
f"multiple dbGaP configurations.")

@staticmethod
def _validate_dbgap_config(dbgap_configs):
error_message = "The dbgap configuration is not a recognized data structure, aborting!"
corrected_dbgap_configs = FenceConfig._coerce_to_array(dbgap_configs, error_message)
FenceConfig._validate_parent_child_studies(corrected_dbgap_configs)


config = FenceConfig(DEFAULT_CFG_PATH)
69 changes: 67 additions & 2 deletions fence/scripting/fence_create.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,9 @@
from fence.config import config
from fence.sync.sync_users import UserSyncer
from fence.utils import (
create_client,
get_valid_expiration,
generate_client_credentials,
get_SQLAlchemyDriver,
get_SQLAlchemyDriver, logger,
)
from sqlalchemy.orm.attributes import flag_modified
from gen3authz.client.arborist.client import ArboristClient
Expand Down Expand Up @@ -1824,3 +1823,69 @@ def access_token_polling_job(
with driver.session as db_session:
loop = asyncio.get_event_loop()
loop.run_until_complete(job.update_tokens(db_session))


def create_client(
DB,
username=None,
urls=[],
name="",
description="",
auto_approve=False,
is_admin=False,
grant_types=None,
confidential=True,
arborist=None,
policies=None,
allowed_scopes=None,
expires_in=None,
):
client_id, client_secret, hashed_secret = generate_client_credentials(confidential)
if arborist is not None:
arborist.create_client(client_id, policies)
driver = get_SQLAlchemyDriver(DB)
auth_method = "client_secret_basic" if confidential else "none"

allowed_scopes = allowed_scopes or config["CLIENT_ALLOWED_SCOPES"]
if not set(allowed_scopes).issubset(set(config["CLIENT_ALLOWED_SCOPES"])):
raise ValueError(
"Each allowed scope must be one of: {}".format(
config["CLIENT_ALLOWED_SCOPES"]
)
)

if "openid" not in allowed_scopes:
allowed_scopes.append("openid")
logger.warning('Adding required "openid" scope to list of allowed scopes.')

with driver.session as s:
user = None
if username:
user = query_for_user(session=s, username=username)
if not user:
user = User(username=username, is_admin=is_admin)
s.add(user)

if s.query(Client).filter(Client.name == name).first():
if arborist is not None:
arborist.delete_client(client_id)
raise Exception("client {} already exists".format(name))

client = Client(
client_id=client_id,
client_secret=hashed_secret,
user=user,
redirect_uris=urls,
allowed_scopes=" ".join(allowed_scopes),
description=description,
name=name,
auto_approve=auto_approve,
grant_types=grant_types,
is_confidential=confidential,
token_endpoint_auth_method=auth_method,
expires_in=expires_in,
)
s.add(client)
s.commit()

return client_id, client_secret
49 changes: 48 additions & 1 deletion fence/scripting/google_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
"""
import traceback

import requests
from gen3cirrus.google_cloud.iam import GooglePolicyMember
from gen3cirrus import GoogleCloudManager
from gen3cirrus.google_cloud.errors import GoogleAPIError
Expand Down Expand Up @@ -37,7 +38,7 @@
from fence import utils
from fence.config import config
from fence.models import User
from fence.errors import Unauthorized
from fence.errors import Unauthorized, NotFound

logger = get_logger(__name__)

Expand Down Expand Up @@ -445,6 +446,52 @@ def _get_user_email_list_from_google_project_with_owner_role(project_id):
)


def send_email(from_email, to_emails, subject, text, smtp_domain):
"""
Send email to group of emails using mail gun api.

https://app.mailgun.com/

Args:
from_email(str): from email
to_emails(list): list of emails to receive the messages
text(str): the text message
smtp_domain(dict): smtp domain server

{
"smtp_hostname": "smtp.mailgun.org",
"default_login": "[email protected]",
"api_url": "https://api.mailgun.net/v3/mailgun.planx-pla.net",
"smtp_password": "password", # pragma: allowlist secret
"api_key": "api key" # pragma: allowlist secret
}

Returns:
Http response

Exceptions:
KeyError

"""
if smtp_domain not in config["GUN_MAIL"] or not config["GUN_MAIL"].get(
smtp_domain
).get("smtp_password"):
raise NotFound(
"SMTP Domain '{}' does not exist in configuration for GUN_MAIL or "
"smtp_password was not provided. "
"Cannot send email.".format(smtp_domain)
)

api_key = config["GUN_MAIL"][smtp_domain].get("api_key", "")
email_url = config["GUN_MAIL"][smtp_domain].get("api_url", "") + "/messages"

return requests.post(
email_url,
auth=("api", api_key),
data={"from": from_email, "to": to_emails, "subject": subject, "text": text},
)


def _send_emails_informing_service_account_removal(
to_emails, invalid_service_account_reasons, invalid_project_reasons, project_id
):
Expand Down
Loading
Loading