Skip to content

Commit 3175163

Browse files
Avantol13Avantol13-machine-user
Avantol13
and
Avantol13-machine-user
authored
chore(deps): update deps, docs, and use common utils function (#137)
* chore(deps): update deps, docs, and use common utils function * Apply automatic documentation changes * chore(version): bump * Apply automatic documentation changes Co-authored-by: Alexander VT <[email protected]>
1 parent 026ba23 commit 3175163

File tree

14 files changed

+194
-161
lines changed

14 files changed

+194
-161
lines changed

‎.secrets.baseline‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"files": "(docs\\/_build|poetry.lock)|^.secrets.baseline$",
44
"lines": null
55
},
6-
"generated_at": "2022-06-07T20:32:38Z",
6+
"generated_at": "2022-06-10T15:20:47Z",
77
"plugins_used": [
88
{
99
"name": "AWSKeyDetector"
@@ -62,19 +62,19 @@
6262
{
6363
"hashed_secret": "7c35c215b326b9463b669b657c1ff9873ff53d9a",
6464
"is_verified": false,
65-
"line_number": 202,
65+
"line_number": 203,
6666
"type": "Hex High Entropy String"
6767
},
6868
{
6969
"hashed_secret": "0d515eaf06062d52e8c80abb4d3b713a65396d30",
7070
"is_verified": false,
71-
"line_number": 207,
71+
"line_number": 208,
7272
"type": "Hex High Entropy String"
7373
},
7474
{
7575
"hashed_secret": "b4cff7c2af45cdfe66195ec574a7b8832f8621ea",
7676
"is_verified": false,
77-
"line_number": 214,
77+
"line_number": 215,
7878
"type": "Hex High Entropy String"
7979
}
8080
],
@@ -350,49 +350,49 @@
350350
{
351351
"hashed_secret": "96c9184fb19c9c1618ccf44d141f8029a739891c",
352352
"is_verified": false,
353-
"line_number": 426,
353+
"line_number": 424,
354354
"type": "Hex High Entropy String"
355355
},
356356
{
357357
"hashed_secret": "e1da93616713812cb50e0ac845b1e9e305d949f1",
358358
"is_verified": false,
359-
"line_number": 562,
359+
"line_number": 559,
360360
"type": "Hex High Entropy String"
361361
},
362362
{
363363
"hashed_secret": "47f42f4c34fddab383b817e689dc0fb75af81266",
364364
"is_verified": false,
365-
"line_number": 586,
365+
"line_number": 583,
366366
"type": "Hex High Entropy String"
367367
},
368368
{
369369
"hashed_secret": "300d95dd5d30ab6928ffda6c08c6a129a23e5b39",
370370
"is_verified": false,
371-
"line_number": 610,
371+
"line_number": 607,
372372
"type": "Hex High Entropy String"
373373
},
374374
{
375375
"hashed_secret": "f9e664db75c7f23a299b0b055c10e08d47073e93",
376376
"is_verified": false,
377-
"line_number": 678,
377+
"line_number": 675,
378378
"type": "Hex High Entropy String"
379379
},
380380
{
381381
"hashed_secret": "7c35c215b326b9463b669b657c1ff9873ff53d9a",
382382
"is_verified": false,
383-
"line_number": 819,
383+
"line_number": 816,
384384
"type": "Hex High Entropy String"
385385
},
386386
{
387387
"hashed_secret": "0d515eaf06062d52e8c80abb4d3b713a65396d30",
388388
"is_verified": false,
389-
"line_number": 823,
389+
"line_number": 820,
390390
"type": "Hex High Entropy String"
391391
},
392392
{
393393
"hashed_secret": "b4cff7c2af45cdfe66195ec574a7b8832f8621ea",
394394
"is_verified": false,
395-
"line_number": 828,
395+
"line_number": 825,
396396
"type": "Hex High Entropy String"
397397
}
398398
],

‎docs/_build/doctrees/environment.pickle‎

0 Bytes
Binary file not shown.

‎docs/_build/doctrees/tools/indexing.doctree‎

-5 Bytes
Binary file not shown.

‎docs/_build/doctrees/tools/metadata.doctree‎

5 Bytes
Binary file not shown.

‎docs/_build/html/searchindex.js‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/_build/html/tools/indexing.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,7 @@ <h1>Indexing Tools<a class="headerlink" href="#indexing-tools" title="Permalink
340340

341341
<dl class="py function">
342342
<dt class="sig sig-object py" id="gen3.tools.indexing.verify_manifest.async_verify_object_manifest">
343-
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1654807944.3028057.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition"></a></dt>
343+
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1654874771.190345.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition"></a></dt>
344344
<dd><p>Verify all file object records into a manifest csv</p>
345345
<dl class="field-list simple">
346346
<dt class="field-odd">Parameters<span class="colon">:</span></dt>

‎docs/_build/html/tools/metadata.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ <h1>Metadata Tools<a class="headerlink" href="#metadata-tools" title="Permalink
104104

105105
<dl class="py function">
106106
<dt class="sig sig-object py" id="gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest">
107-
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1654807944.560571.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition"></a></dt>
107+
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1654874771.4957037.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition"></a></dt>
108108
<dd><p>Ingest all metadata records into a manifest csv</p>
109109
<dl class="field-list simple">
110110
<dt class="field-odd">Parameters<span class="colon">:</span></dt>

‎docs/howto/diirmIndexing.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,16 @@ import asyncio
1818

1919
from gen3.tools import indexing
2020
from gen3.tools.indexing.verify_manifest import manifest_row_parsers
21+
from gen3.utils import get_or_create_event_loop_for_thread
2122

2223
logging.basicConfig(filename="output.log", level=logging.DEBUG)
2324
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
2425

2526
COMMONS = "https://{{insert-commons-here}}/"
2627

2728
def main():
28-
loop = asyncio.new_event_loop()
29-
asyncio.set_event_loop(loop)
29+
loop = get_or_create_event_loop_for_thread()
30+
3031

3132
loop.run_until_complete(
3233
indexing.async_download_object_manifest(
@@ -83,8 +84,8 @@ def main():
8384
# override default parsers
8485
manifest_row_parsers["file_size"] = _get_file_size
8586

86-
loop = asyncio.new_event_loop()
87-
asyncio.set_event_loop(loop)
87+
loop = get_or_create_event_loop_for_thread()
88+
8889

8990
loop.run_until_complete(
9091
indexing.async_verify_object_manifest(

‎docs/howto/metadataTools.md‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import asyncio
3030
from gen3.auth import Gen3Auth
3131
from gen3.tools import metadata
3232
from gen3.tools.metadata.ingest_manifest import manifest_row_parsers
33+
from gen3.utils import get_or_create_event_loop_for_thread
3334

3435
logging.basicConfig(filename="output.log", level=logging.DEBUG)
3536
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
@@ -39,8 +40,8 @@ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
3940
MANIFEST = "dbgap_extract_guid.tsv"
4041

4142
def main():
42-
loop = asyncio.new_event_loop()
43-
asyncio.set_event_loop(loop)
43+
loop = get_or_create_event_loop_for_thread()
44+
4445

4546
auth = Gen3Auth(refresh_file="credentials.json")
4647

@@ -256,8 +257,8 @@ MANIFEST = "dbgap_extract.tsv"
256257

257258

258259
def main():
259-
loop = asyncio.new_event_loop()
260-
asyncio.set_event_loop(loop)
260+
loop = get_or_create_event_loop_for_thread()
261+
261262

262263
auth = Gen3Auth(refresh_file="credentials.json")
263264

@@ -565,8 +566,8 @@ def get_dbgap_merged_metadata_manifest():
565566
"partial_match_or_exact_match": "partial_match",
566567
}
567568

568-
loop = asyncio.new_event_loop()
569-
asyncio.set_event_loop(loop)
569+
loop = get_or_create_event_loop_for_thread()
570+
570571

571572
job_output = loop.run_until_complete(
572573
jobs.async_run_job_and_wait(job_name=DBGAP_METADATA_JOB, job_input=job_input)
@@ -591,8 +592,8 @@ def metadata_ingest():
591592
"metadata_source": "dbgap",
592593
}
593594

594-
loop = asyncio.new_event_loop()
595-
asyncio.set_event_loop(loop)
595+
loop = get_or_create_event_loop_for_thread()
596+
596597

597598
job_output = loop.run_until_complete(
598599
jobs.async_run_job_and_wait(job_name=INGEST_METADATA_JOB, job_input=job_input)
@@ -633,8 +634,8 @@ COMMONS = "https://{{insert-commons-here}}/"
633634

634635

635636
def main():
636-
loop = asyncio.new_event_loop()
637-
asyncio.set_event_loop(loop)
637+
loop = get_or_create_event_loop_for_thread()
638+
638639

639640
loop.run_until_complete(
640641
metadata.async_verify_metadata_manifest(

‎docs/reference/sdkClasses.md‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ This contains an auth wrapper for supporting JWT based authentication with `requ
1717
By default - the `Gen3Auth` constructor looks for an api key
1818
in `~/.gen3/credentials.json`. You may override that path
1919
via the `GEN3_API_KEY` environment varialbe, or by passing a
20-
`refresh_file` parameter.
20+
`refresh_file` parameter.
2121

2222
When working in a Gen3 Workspace, all parameters are optional and the `Gen3Auth` instance should be initialized as follows:
2323

@@ -52,6 +52,7 @@ import asyncio
5252
from gen3.index import Gen3Index
5353
from gen3.auth import Gen3Auth
5454
from gen3.jobs import Gen3Jobs, DBGAP_METADATA_JOB, INGEST_METADATA_JOB
55+
from gen3.utils import get_or_create_event_loop_for_thread
5556

5657
# An API Key downloaded from the above commons' "Profile" page
5758
API_KEY_FILEPATH = "credentials.json"
@@ -69,8 +70,7 @@ def metadata_ingest():
6970
"metadata_source": "dbgaptest",
7071
}
7172

72-
loop = asyncio.new_event_loop()
73-
asyncio.set_event_loop(loop)
73+
loop = get_or_create_event_loop_for_thread()
7474

7575
job_output = loop.run_until_complete(
7676
jobs.async_run_job_and_wait(job_name=INGEST_METADATA_JOB, job_input=job_input)
@@ -93,8 +93,7 @@ def main():
9393
"partial_match_or_exact_match": "partial_match",
9494
}
9595

96-
loop = asyncio.new_event_loop()
97-
asyncio.set_event_loop(loop)
96+
loop = get_or_create_event_loop_for_thread()
9897

9998
job_output = loop.run_until_complete(
10099
jobs.async_run_job_and_wait(job_name=DBGAP_METADATA_JOB, job_input=job_input)
@@ -138,8 +137,7 @@ def example_async_run_job():
138137
auth = Gen3Auth(refresh_file=API_KEY_FILEPATH)
139138
jobs = Gen3Jobs(auth_provider=auth)
140139

141-
loop = asyncio.new_event_loop()
142-
asyncio.set_event_loop(loop)
140+
loop = get_or_create_event_loop_for_thread()
143141

144142
job_output = loop.run_until_complete(
145143
jobs.async_run_job_and_wait(job_name=DBGAP_METADATA_JOB, job_input=JOB_INPUT)

0 commit comments

Comments
 (0)