Skip to content

Commit b8585a8

Browse files
authored
Merge pull request #187 from uc-cdis/fix/BRH-541
Fix/BRH-541
2 parents f3506f2 + 144252a commit b8585a8

File tree

9 files changed

+138
-30
lines changed

9 files changed

+138
-30
lines changed
2.78 KB
Binary file not shown.
-5 Bytes
Binary file not shown.
0 Bytes
Binary file not shown.

docs/_build/html/_modules/gen3/auth.html

Lines changed: 67 additions & 13 deletions
Large diffs are not rendered by default.

docs/_build/html/searchindex.js

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

docs/_build/html/tools/indexing.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,7 @@ <h1>Indexing Tools<a class="headerlink" href="#indexing-tools" title="Permalink
380380

381381
<dl class="py function">
382382
<dt class="sig sig-object py" id="gen3.tools.indexing.verify_manifest.async_verify_object_manifest">
383-
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1686160040.2199411.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition"></a></dt>
383+
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1686340166.129575.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition"></a></dt>
384384
<dd><p>Verify all file object records into a manifest csv</p>
385385
<dl class="field-list simple">
386386
<dt class="field-odd">Parameters<span class="colon">:</span></dt>

docs/_build/html/tools/metadata.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ <h1>Metadata Tools<a class="headerlink" href="#metadata-tools" title="Permalink
101101

102102
<dl class="py function">
103103
<dt class="sig sig-object py" id="gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest">
104-
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1686160040.6163409.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition"></a></dt>
104+
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1686340166.4061348.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition"></a></dt>
105105
<dd><p>Ingest all metadata records into a manifest csv</p>
106106
<dl class="field-list simple">
107107
<dt class="field-odd">Parameters<span class="colon">:</span></dt>

gen3/auth.py

Lines changed: 67 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ def __init__(
185185
client_credentials=None,
186186
client_scopes=None,
187187
):
188+
logging.debug("Initatializing auth..")
188189
self.endpoint = remove_trailing_whitespace_and_slashes_in_url(endpoint)
189190
# note - `_refresh_token` is not actually a JWT refresh token - it's a
190191
# gen3 api key with a token as the "api_key" property
@@ -280,6 +281,9 @@ def __init__(
280281
)
281282
):
282283
try:
284+
logging.debug(
285+
"Switch to using WTS and set external WTS host url.."
286+
)
283287
self._use_wts = True
284288
self._external_wts_host = (
285289
endpoint_from_token(self._refresh_token["api_key"])
@@ -479,23 +483,72 @@ def get_access_token_from_wts(self, endpoint=None):
479483
in the given namespace. If idp is not set, then default to "local"
480484
"""
481485
# attempt to get a token from the workspace-token-service
486+
logging.debug("getting access token from wts..")
482487
auth_url = get_wts_endpoint(self._wts_namespace) + "/token/"
483-
if self._wts_idp and self._wts_idp != "local":
484-
auth_url += "?idp={}".format(self._wts_idp)
485488

486-
try:
487-
resp = requests.get(auth_url)
488-
if (resp and resp.status_code == 200) or (not self._external_wts_host):
489-
return _handle_access_token_response(resp, "token")
490-
except Exception as e:
491-
if not self._external_wts_host:
492-
raise e
493-
else:
494-
# Try to obtain token from external wts
495-
pass
489+
# If non "local" idp value exists, append to auth url
490+
# If user specified endpoint value, then first attempt to determine idp value.
491+
if self.endpoint or (self._wts_idp and self._wts_idp != "local"):
492+
# If user supplied endpoint value and not idp, figure out the idp value
493+
if self.endpoint:
494+
logging.debug(
495+
"First try to use the local WTS to figure out idp name for the supplied endpoint.."
496+
)
497+
try:
498+
provider_List = get_wts_idps(self._wts_namespace)
499+
matchProviders = list(
500+
filter(
501+
lambda provider: provider["base_url"] == endpoint,
502+
provider_List["providers"],
503+
)
504+
)
505+
if len(matchProviders) == 1:
506+
logging.debug("Found matching idp from local WTS.")
507+
self._wts_idp = matchProviders[0]["idp"]
508+
elif len(matchProviders) > 1:
509+
raise ValueError(
510+
"Multiple idps matched with endpoint value provided."
511+
)
512+
else:
513+
logging.debug("Could not find matching idp from local WTS.")
514+
except Exception as e:
515+
logging.debug(
516+
"Exception occured when making network call to local WTS."
517+
)
518+
if not self._external_wts_host:
519+
raise e
520+
else:
521+
logging.debug("Since external WTS host exists, continuing on..")
522+
pass
523+
524+
if self._wts_idp and self._wts_idp != "local":
525+
auth_url += "?idp={}".format(self._wts_idp)
526+
527+
# If endpoint value exists, only get WTS token if idp value has been successfully determined
528+
# Otherwise skip to querying external WTS
529+
# This is to prevent local WTS from supplying an incorrect token to user
530+
if (
531+
not self._external_wts_host
532+
or not self.endpoint
533+
or (self.endpoint and self._wts_idp != "local")
534+
):
535+
try:
536+
logging.debug("Try to get access token from local WTS..")
537+
logging.debug(f"{auth_url=}")
538+
resp = requests.get(auth_url)
539+
if (resp and resp.status_code == 200) or (not self._external_wts_host):
540+
return _handle_access_token_response(resp, "token")
541+
except Exception as e:
542+
if not self._external_wts_host:
543+
raise e
544+
else:
545+
# Try to obtain token from external wts
546+
logging.debug("Could get obtain token from Local WTS.")
547+
pass
496548

497549
# local workspace wts call failed, try using a network call
498550
# First get access token with WTS host
551+
logging.debug("Trying to get access token from external WTS Host..")
499552
wts_token = get_access_token_with_key(self._refresh_token)
500553
auth_url = self._external_wts_host + "token/"
501554

@@ -523,6 +576,7 @@ def get_access_token_from_wts(self, endpoint=None):
523576

524577
if len(matchProviders) == 1:
525578
self._wts_idp = matchProviders[0]["idp"]
579+
logging.debug("Succesfully determined idp value: {}".format(self._wts_idp))
526580
else:
527581
idp_list = "\n "
528582

@@ -559,7 +613,7 @@ def get_access_token_from_wts(self, endpoint=None):
559613
+ idp_list
560614
+ "Query /wts/external_oidc/ for more information."
561615
)
562-
616+
logging.debug("Finally getting access token..")
563617
auth_url += "?idp={}".format(self._wts_idp)
564618
header = {"Authorization": "Bearer " + wts_token}
565619
resp = requests.get(auth_url, headers=header)

pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[tool.poetry]
22
name = "gen3"
33
homepage = "https://gen3.org/"
4-
version = "4.19.0"
4+
version = "4.19.1"
55
description = "Gen3 CLI and Python SDK"
66
authors = ["Center for Translational Data Science at the University of Chicago <[email protected]>"]
77
license = "Apache-2.0"

0 commit comments

Comments
 (0)