feat(dois): dbGaP Parsing for DOI metadata automation (#183)

* feat(doi): rough support for DOI creation against DataCite API

* Apply automatic documentation changes

* feat(doi): add tests, refactor some support for various fields that are lists

* feat(doi): get DOI core classes working, add tests, various fixes and updates for discovery metadata

* Apply automatic documentation changes

* chore(poetry): bump version

* Apply automatic documentation changes

* fix(logging): correct log to reflect actual HTTP method

* Apply automatic documentation changes

* fix(doi): fix PUT endpoint to use DOI identifier in URL

* Apply automatic documentation changes

* chore(cli): various cleanup/clarification on help messages

* feat(dois): refactor external API locations, add new dbGaP support, add dbgap->DOI automation

* fix(merge): fix an issue with metadata merging

* feat(dois): correct typos, add docs, clearer logs

* feat(doi): formatting and docs updates

* feat(dois): ability to specify to use the GUID in the landing page URL

* Apply automatic documentation changes

* chore(version): bump

* chore(discovery): refactor to reduce code duplication

* Apply automatic documentation changes

* Update gen3/external/nih/dbgap_doi.py

Co-authored-by: Pauline Ribeyre <[email protected]>

* fix(discovery_doi): updates after review, make support more generic and modular

* Apply automatic documentation changes

* feat(tests): add tests, refactor for clarity, fix issue with update

* Apply automatic documentation changes

* feat(dois): add support for excluding specific datasets from discovery DOI minting

* Apply automatic documentation changes

* fix(dois): add backoff, handle alternateIDs from DataCite correctly, update docs

* Apply automatic documentation changes

* fix(doi): fix test that was expecting old behavior

* Apply automatic documentation changes

* Update gen3/discovery_dois.py

Co-authored-by: Pauline Ribeyre <[email protected]>

* Apply automatic documentation changes

---------

Co-authored-by: Alexander VT <[email protected]>
Co-authored-by: Pauline Ribeyre <[email protected]>

Author: 3 people

docs/_build/doctrees/environment.pickle

@@ -725,9 +725,7 @@ <h1>Source code for gen3.metadata</h1><div class="highlight"><pre>
 
         <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;hitting: </span><span class="si">{</span><span class="n">url_with_params</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;data: </span><span class="si">{</span><span class="n">data</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
-        <span class="n">response</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">put</span><span class="p">(</span>
-            <span class="n">url_with_params</span><span class="p">,</span> <span class="n">json</span><span class="o">=</span><span class="n">data</span><span class="p">,</span> <span class="n">merge</span><span class="o">=</span><span class="n">merge</span><span class="p">,</span> <span class="n">auth</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_auth_provider</span>
-        <span class="p">)</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">url_with_params</span><span class="p">,</span> <span class="n">json</span><span class="o">=</span><span class="n">data</span><span class="p">,</span> <span class="n">auth</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_auth_provider</span><span class="p">)</span>
         <span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
 
         <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span></div>

docs/_build/doctrees/tools/indexing.doctree

@@ -380,7 +380,7 @@ <h1>Indexing Tools<a class="headerlink" href="#indexing-tools" title="Permalink
 
 <dl class="py function">
 <dt class="sig sig-object py" id="gen3.tools.indexing.verify_manifest.async_verify_object_manifest">
-<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1686340166.129575.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition">¶</a></dt>
+<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1688591583.648493.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition">¶</a></dt>
 <dd><p>Verify all file object records into a manifest csv</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>

docs/_build/doctrees/tools/metadata.doctree

@@ -101,7 +101,7 @@ <h1>Metadata Tools<a class="headerlink" href="#metadata-tools" title="Permalink
 
 <dl class="py function">
 <dt class="sig sig-object py" id="gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest">
-<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1686340166.4061348.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition">¶</a></dt>
+<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1688591583.9268854.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition">¶</a></dt>
 <dd><p>Ingest all metadata records into a manifest csv</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>

docs/_build/html/_modules/gen3/metadata.html

@@ -128,7 +128,7 @@ def test_manual_single_doi(publish_dois=False):
   # Setup
   gen3_auth = Gen3Auth()
   datacite = DataCite(
-    api=DataCite.TEST_URL,
+    use_prod=False,
     auth_provider=HTTPBasicAuth(
       os.environ.get("DATACITE_USERNAME"),
       os.environ.get("DATACITE_PASSWORD"),
@@ -162,7 +162,11 @@ def test_manual_single_doi(publish_dois=False):
   }
 
   # Create/Mint the DOI in DataCite
-  doi = DigitalObjectIdentifier(root_url=COMMONS_DISCOVERY_PAGE, **doi_metadata)
+  # The default url generated is "root_url" + identifier
+  # If your Discovery metadata records don't use the DOI as the GUID,
+  # you may need to supply the URL yourself like below
+  url = COMMONS_DISCOVERY_PAGE.rstrip("/") + f"/{gen3_metadata_guid}"
+  doi = DigitalObjectIdentifier(url=url, **doi_metadata)
 
   if publish_dois:
     logging.info(f"Publishing DOI `{identifier}`...")
@@ -212,6 +216,28 @@ DOI url.
 ```json
 "discoveryConfig": {
     // ...
+    "features": {
+      // ...
+      "search": {
+        "searchBar": {
+          "enabled": true,
+          "searchableTextFields": [
+            "doi_titles",
+            "doi_version_information",
+            "doi_citation",
+            "doi_creators",
+            "doi_publisher",
+            "doi_identifier",
+            "doi_alternateIdentifiers",
+            "doi_contributors",
+            "doi_descriptions",
+            "doi_publication_year",
+            "doi_resolveable_link",
+            "doi_fundingReferences",
+            "doi_relatedIdentifiers"
+          ]
+        },
+    // ...
     "detailView": {
       // ...
       "tabs": [
@@ -224,7 +250,7 @@ DOI url.
                   {
                     "type": "block",
                     "label": "",
-                    "sourceField": "disclaimer",
+                    "sourceField": "doi_disclaimer",
                     "default": ""
                   },
                   {
@@ -245,6 +271,12 @@ DOI url.
                     "sourceField": "doi_is_available",
                     "default": "None"
                   },
+                  {
+                    "type": "text",
+                    "label": "Creators:",
+                    "sourceField": "doi_creators",
+                    "default": "Not specified"
+                  },
                   {
                     "type": "text",
                     "label": "Citation:",
@@ -308,6 +340,18 @@ DOI url.
                     "label": "Version:",
                     "sourceField": "doi_version_information",
                     "default": "Not specified"
+                  },
+                  {
+                    "type": "text",
+                    "label": "Contributors:",
+                    "sourceField": "doi_contributors",
+                    "default": "Not specified"
+                  },
+                  {
+                    "type": "text",
+                    "label": "Related Identifiers:",
+                    "sourceField": "doi_relatedIdentifiers",
+                    "default": "Not specified"
                   }
                 ]
               },
@@ -327,12 +371,17 @@ DOI url.
         // ...
 ```
 
-#### Work in Progress. Script to automate dbGaP scraping for updating datasets and minting DOIs
+#### Automate DOI creation for Datasets
+
+Automates the pulling of current datasets from Discovery, getting identifiers,
+scraping various APIs for DOI related metadata, and then going through
+the DOI creation loop to mint the DOI in Datacite and persist the metadata back in
+Gen3.
 
-- TODO: Push DOI from submitted to registered
+See below for a full example using the dbGaP `DbgapMetadataInterface`.
 
-See below for a full example of DOI metadata gathering, minting, and persisting
-into Gen3.
+More interfaces may exist in the future for doing this by querying non-dbGaP
+sources.
 
 ```python
 import os
@@ -341,7 +390,7 @@ from requests.auth import HTTPBasicAuth
 from cdislogging import get_logger
 
 from gen3.auth import Gen3Auth
-from gen3.discovery_dois import mint_dois_for_dbgap_discovery_datasets
+from gen3.discovery_dois import mint_dois_for_discovery_datasets, DbgapMetadataInterface
 from gen3.utils import get_random_alphanumeric
 
 logging = get_logger("__name__", log_level="info")
@@ -355,33 +404,54 @@ DOI_ACCESS_INFORMATION = "You can find information about how to access this reso
 DOI_ACCESS_INFORMATION_LINK = "https://example.com/more/info"
 DOI_CONTACT = "https://example.com/contact/"
 
+def mint_discovery_dois():
+    auth = Gen3Auth()
 
-def get_doi_identifier():
-    return (
-        PREFIX + "/EXAMPLE-" + get_random_alphanumeric(4) + "-" + get_random_alphanumeric(4)
-    )
+    # this alternate ID is some globally unique ID other than the GUID that
+    # will be needed to get DOI metadata (like the phsid for dbGaP)
+    metadata_field_for_alternate_id = "dbgap_accession"
 
+    # you can choose to exclude certain Discovery Metadata datasets based on
+    # their GUID or alternate ID (this means they won't get additional DOI metadata
+    # or have DOIs minted, they'll be skipped)
+    exclude_datasets=["MetadataGUID_to_exclude", "AlternateID_to_exclude", "..."]
 
-def main():
-    auth = Gen3Auth()
-    dbgap_phsid_field = "dbgap_accession"
+    # When this is True, you CANNOT REVERT THIS ACTION. A published DOI
+    # cannot be deleted. It is recommended to test with "Draft" state DOIs first
+    # (which is the default when publish_dois is not True).
+    publish_dois = False
 
-    mint_dois_for_dbgap_discovery_datasets(
+    mint_dois_for_discovery_datasets(
         gen3_auth=auth,
         datacite_auth=HTTPBasicAuth(
             os.environ.get("DATACITE_USERNAME"),
             os.environ.get("DATACITE_PASSWORD"),
         ),
-        dbgap_phsid_field=dbgap_phsid_field,
+        metadata_field_for_alternate_id=metadata_field_for_alternate_id,
         get_doi_identifier_function=get_doi_identifier,
-        publisher=PUBLISHER,
+        metadata_interface=DbgapMetadataInterface,
+        doi_publisher=PUBLISHER,
         commons_discovery_page=COMMONS_DISCOVERY_PAGE,
         doi_disclaimer=DOI_DISCLAIMER,
         doi_access_information=DOI_ACCESS_INFORMATION,
         doi_access_information_link=DOI_ACCESS_INFORMATION_LINK,
         doi_contact=DOI_CONTACT,
+        publish_dois=publish_dois,
+        datacite_use_prod=False,
+        exclude_datasets=["MetadataGUID_to_exclude", "AlternateID_to_exclude", "..."]
     )
 
+
+def get_doi_identifier():
+    return (
+        PREFIX + "/EXAMPLE-" + get_random_alphanumeric(4) + "-" + get_random_alphanumeric(4)
+    )
+
+
+def main():
+    mint_discovery_dois()
+
+
 if __name__ == "__main__":
     main()
 

docs/_build/html/searchindex.js

@@ -34,18 +34,18 @@ def get(self):
     "--auth",
     "auth_config",
     default=os.getenv("GEN3_API_KEY", None),
-    help="""authentication source:
-    "idp://wts/<idp>" is an identity provider in a Gen3 workspace,
-    "accesstoken:///<token>" is an access token,
-    otherwise a path to an api key or basename of key under ~/.gen3/;
-    default value is "credentials" if ~/.gen3/credentials.json exists, otherwise "idp://wts/local"
+    help="""authentication source, by default expects an API key in "~/.gen3/credentials.json".
+    Has special support for token service: "idp://wts/<idp>", and raw access tokens
+    "accesstoken:///<token>",
+    otherwise a path to an API key or basename of key under ~/.gen3/ can be used.
+    Default value is "credentials" if ~/.gen3/credentials.json exists, otherwise "idp://wts/local"
     """,
 )
 @click.option(
     "--endpoint",
     "endpoint",
     default=os.getenv("GEN3_ENDPOINT", "default"),
-    help="commons hostname - optional if API Key given",
+    help="commons hostname - optional if API Key given in `auth`",
 )
 @click.option(
     "-v",

docs/_build/html/tools/indexing.html

@@ -72,7 +72,7 @@ def wts_list():
 
 @click.group()
 def auth():
-    """Gen3 sdk auth commands"""
+    """Commands for authentication and authorization"""
     pass