Merge pull request #248 from uc-cdis/chore/allow-commons-url-pull-object

Chore/allow commons-url pull object

Author: george42-ctds

docs/_build/doctrees/environment.pickle

@@ -718,7 +718,11 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
 
 
 <span class="k">def</span><span class="w"> </span><span class="nf">resolve_objects_drs_hostname</span><span class="p">(</span>
-    <span class="n">object_ids</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Downloadable</span><span class="p">],</span> <span class="n">resolved_drs_prefix_cache</span><span class="p">:</span> <span class="nb">dict</span><span class="p">,</span> <span class="n">mds_url</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">endpoint</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">object_ids</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Downloadable</span><span class="p">],</span>
+    <span class="n">resolved_drs_prefix_cache</span><span class="p">:</span> <span class="nb">dict</span><span class="p">,</span>
+    <span class="n">mds_url</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+    <span class="n">endpoint</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+    <span class="n">commons_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Given a list of object_ids go through list and resolve + cache any unknown hosts</span>
 
@@ -729,7 +733,9 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
 <span class="sd">        hostname (str): Hostname to main Gen3 environment</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">for</span> <span class="n">entry</span> <span class="ow">in</span> <span class="n">object_ids</span><span class="p">:</span>
-        <span class="k">if</span> <span class="n">endpoint</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">entry</span><span class="o">.</span><span class="n">hostname</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">commons_url</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">entry</span><span class="o">.</span><span class="n">hostname</span> <span class="o">=</span> <span class="n">commons_url</span>
+        <span class="k">elif</span> <span class="n">endpoint</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">entry</span><span class="o">.</span><span class="n">hostname</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
             <span class="n">entry</span><span class="o">.</span><span class="n">hostname</span> <span class="o">=</span> <span class="n">endpoint</span>
         <span class="k">if</span> <span class="n">entry</span><span class="o">.</span><span class="n">hostname</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
             <span class="c1"># if resolution fails the entry hostname will still be None</span>
@@ -864,7 +870,8 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
         <span class="n">auth</span><span class="p">:</span> <span class="n">Gen3Auth</span><span class="p">,</span>
         <span class="n">download_list</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Downloadable</span><span class="p">],</span>
         <span class="n">show_progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-        <span class="n">endpoint</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="n">endpoint</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">commons_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="p">):</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Initialize the DownloadManager so that is ready to start downloading.</span>
@@ -879,6 +886,7 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
 
         <span class="bp">self</span><span class="o">.</span><span class="n">hostname</span> <span class="o">=</span> <span class="n">hostname</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span> <span class="o">=</span> <span class="n">endpoint</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">commons_url</span> <span class="o">=</span> <span class="n">commons_url</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">access_token</span> <span class="o">=</span> <span class="n">auth</span><span class="o">.</span><span class="n">get_access_token</span><span class="p">()</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="n">Gen3Metadata</span><span class="p">(</span><span class="n">auth</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">wts_endpoints</span> <span class="o">=</span> <span class="n">wts_external_oidc</span><span class="p">(</span><span class="n">hostname</span><span class="p">)</span>
@@ -908,7 +916,8 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
             <span class="n">object_list</span><span class="p">,</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">resolved_compact_drs</span><span class="p">,</span>
             <span class="n">mds_url</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;http://</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">hostname</span><span class="si">}</span><span class="s2">/mds/aggregate/info&quot;</span><span class="p">,</span>
-            <span class="n">endpoint</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span>
+            <span class="n">endpoint</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">,</span>
+            <span class="n">commons_url</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">commons_url</span><span class="p">,</span>
         <span class="p">)</span>
         <span class="n">progress_bar</span> <span class="o">=</span> <span class="p">(</span>
             <span class="n">tqdm</span><span class="p">(</span><span class="n">desc</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;Resolving objects&quot;</span><span class="p">,</span> <span class="n">total</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">object_list</span><span class="p">))</span>
@@ -1203,7 +1212,7 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
         <span class="n">auth</span><span class="o">=</span><span class="n">auth</span><span class="p">,</span>
         <span class="n">download_list</span><span class="o">=</span><span class="n">object_list</span><span class="p">,</span>
         <span class="n">show_progress</span><span class="o">=</span><span class="n">show_progress</span><span class="p">,</span>
-        <span class="n">endpoint</span><span class="o">=</span><span class="n">hostname</span>
+        <span class="n">endpoint</span><span class="o">=</span><span class="n">hostname</span><span class="p">,</span>
     <span class="p">)</span>
 
     <span class="n">out_dir_path</span> <span class="o">=</span> <span class="n">ensure_dirpath_exists</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">output_dir</span><span class="p">))</span>
@@ -1224,6 +1233,7 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
     <span class="n">show_progress</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
     <span class="n">unpack_packages</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
     <span class="n">delete_unpacked_packages</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+    <span class="n">commons_url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    A convenience function used to download a single DRS object.</span>
@@ -1251,7 +1261,8 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
         <span class="n">auth</span><span class="o">=</span><span class="n">auth</span><span class="p">,</span>
         <span class="n">download_list</span><span class="o">=</span><span class="n">object_list</span><span class="p">,</span>
         <span class="n">show_progress</span><span class="o">=</span><span class="n">show_progress</span><span class="p">,</span>
-        <span class="n">endpoint</span><span class="o">=</span><span class="n">hostname</span>
+        <span class="n">endpoint</span><span class="o">=</span><span class="n">hostname</span><span class="p">,</span>
+        <span class="n">commons_url</span><span class="o">=</span><span class="n">commons_url</span><span class="p">,</span>
     <span class="p">)</span>
 
     <span class="n">out_dir_path</span> <span class="o">=</span> <span class="n">ensure_dirpath_exists</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">output_dir</span><span class="p">))</span>
@@ -1447,6 +1458,7 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
     <span class="n">show_progress</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
     <span class="n">unpack_packages</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
     <span class="n">delete_unpacked_packages</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+    <span class="n">commons_url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    A convenience function used to download a single DRS object.</span>
@@ -1469,6 +1481,7 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
         <span class="n">show_progress</span><span class="p">,</span>
         <span class="n">unpack_packages</span><span class="p">,</span>
         <span class="n">delete_unpacked_packages</span><span class="p">,</span>
+        <span class="n">commons_url</span><span class="p">,</span>
     <span class="p">)</span>
 
 

docs/_build/doctrees/tools/drs_pull.doctree

@@ -54,7 +54,7 @@
 </div></blockquote>
 <dl class="py class">
 <dt class="sig sig-object py" id="gen3.tools.download.drs_download.DownloadManager">
-<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.download.drs_download.</span></span><span class="sig-name descname"><span class="pre">DownloadManager</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">hostname</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../auth.html#gen3.auth.Gen3Auth" title="gen3.auth.Gen3Auth"><span class="pre">Gen3Auth</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">download_list</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">List</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#gen3.tools.download.drs_download.Downloadable" title="gen3.tools.download.drs_download.Downloadable"><span class="pre">Downloadable</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">show_progress</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">endpoint</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/download/drs_download.html#DownloadManager"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.download.drs_download.DownloadManager" title="Link to this definition">¶</a></dt>
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.download.drs_download.</span></span><span class="sig-name descname"><span class="pre">DownloadManager</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">hostname</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../auth.html#gen3.auth.Gen3Auth" title="gen3.auth.Gen3Auth"><span class="pre">Gen3Auth</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">download_list</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">List</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#gen3.tools.download.drs_download.Downloadable" title="gen3.tools.download.drs_download.Downloadable"><span class="pre">Downloadable</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">show_progress</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">endpoint</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">commons_url</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/download/drs_download.html#DownloadManager"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.download.drs_download.DownloadManager" title="Link to this definition">¶</a></dt>
 <dd><p>Class to assist in downloading a list of Downloadable object which at a minimum is a json manifest
 of DRS object ids. The methods of interest are download and user_access.</p>
 <dl class="py method">