Merge pull request #249 from uc-cdis/fix/external-file-retriever

Fix/external file retriever

Author: mfshao

docs/_build/doctrees/environment.pickle

@@ -5,7 +5,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>gen3.file &#8212; Gen3 SDK  documentation</title>
-    <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=d1102ebc" />
+    <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=5ecbeea2" />
     <link rel="stylesheet" type="text/css" href="../../_static/alabaster.css?v=12dfc556" />
     <script src="../../_static/documentation_options.js?v=5929fcd5"></script>
     <script src="../../_static/doctools.js?v=9a2dae69"></script>
@@ -30,24 +30,24 @@
           <div class="body" role="main">
 
   <h1>Source code for gen3.file</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">json</span>
-<span class="kn">import</span> <span class="nn">requests</span>
-<span class="kn">import</span> <span class="nn">json</span>
-<span class="kn">import</span> <span class="nn">asyncio</span>
-<span class="kn">import</span> <span class="nn">aiohttp</span>
-<span class="kn">import</span> <span class="nn">aiofiles</span>
-<span class="kn">import</span> <span class="nn">time</span>
-<span class="kn">from</span> <span class="nn">tqdm</span> <span class="kn">import</span> <span class="n">tqdm</span>
-<span class="kn">from</span> <span class="nn">types</span> <span class="kn">import</span> <span class="n">SimpleNamespace</span> <span class="k">as</span> <span class="n">Namespace</span>
-<span class="kn">import</span> <span class="nn">os</span>
-<span class="kn">import</span> <span class="nn">requests</span>
-<span class="kn">from</span> <span class="nn">pathlib</span> <span class="kn">import</span> <span class="n">Path</span>
-
-<span class="kn">from</span> <span class="nn">cdislogging</span> <span class="kn">import</span> <span class="n">get_logger</span>
-
-<span class="kn">from</span> <span class="nn">gen3.index</span> <span class="kn">import</span> <span class="n">Gen3Index</span>
-<span class="kn">from</span> <span class="nn">gen3.utils</span> <span class="kn">import</span> <span class="n">DEFAULT_BACKOFF_SETTINGS</span><span class="p">,</span> <span class="n">raise_for_status_and_print_error</span>
-<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">parse_qsl</span><span class="p">,</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">urlparse</span><span class="p">,</span> <span class="n">urlunparse</span>
+<span></span><span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">requests</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">asyncio</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">aiohttp</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">aiofiles</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">tqdm</span><span class="w"> </span><span class="kn">import</span> <span class="n">tqdm</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SimpleNamespace</span> <span class="k">as</span> <span class="n">Namespace</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">requests</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">cdislogging</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_logger</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">gen3.index</span><span class="w"> </span><span class="kn">import</span> <span class="n">Gen3Index</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">gen3.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">DEFAULT_BACKOFF_SETTINGS</span><span class="p">,</span> <span class="n">raise_for_status_and_print_error</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse_qsl</span><span class="p">,</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">urlparse</span><span class="p">,</span> <span class="n">urlunparse</span>
 
 <span class="n">logging</span> <span class="o">=</span> <span class="n">get_logger</span><span class="p">(</span><span class="s2">&quot;__name__&quot;</span><span class="p">)</span>
 
@@ -57,7 +57,7 @@ <h1>Source code for gen3.file</h1><div class="highlight"><pre>
 
 <div class="viewcode-block" id="Gen3File">
 <a class="viewcode-back" href="../../file.html#gen3.file.Gen3File">[docs]</a>
-<span class="k">class</span> <span class="nc">Gen3File</span><span class="p">:</span>
+<span class="k">class</span><span class="w"> </span><span class="nc">Gen3File</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;For interacting with Gen3 file management features.</span>
 
 <span class="sd">    A class for interacting with the Gen3 file download services.</span>
@@ -75,15 +75,15 @@ <h1>Source code for gen3.file</h1><div class="highlight"><pre>
 
 <span class="sd">    &quot;&quot;&quot;</span>
 
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">endpoint</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">auth_provider</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">endpoint</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">auth_provider</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
         <span class="c1"># auth_provider legacy interface required endpoint as 1st arg</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">_auth_provider</span> <span class="o">=</span> <span class="n">auth_provider</span> <span class="ow">or</span> <span class="n">endpoint</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">_endpoint</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_auth_provider</span><span class="o">.</span><span class="n">endpoint</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">unsuccessful_downloads</span> <span class="o">=</span> <span class="p">[]</span>
 
 <div class="viewcode-block" id="Gen3File.get_presigned_url">
 <a class="viewcode-back" href="../../file.html#gen3.file.Gen3File.get_presigned_url">[docs]</a>
-    <span class="k">def</span> <span class="nf">get_presigned_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">guid</span><span class="p">,</span> <span class="n">protocol</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">def</span><span class="w"> </span><span class="nf">get_presigned_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">guid</span><span class="p">,</span> <span class="n">protocol</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Generates a presigned URL for a file.</span>
 
 <span class="sd">        Retrieves a presigned url for a file giving access to a file for a limited time.</span>
@@ -111,7 +111,7 @@ <h1>Source code for gen3.file</h1><div class="highlight"><pre>
 
 <div class="viewcode-block" id="Gen3File.delete_file">
 <a class="viewcode-back" href="../../file.html#gen3.file.Gen3File.delete_file">[docs]</a>
-    <span class="k">def</span> <span class="nf">delete_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">guid</span><span class="p">):</span>
+    <span class="k">def</span><span class="w"> </span><span class="nf">delete_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">guid</span><span class="p">):</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        This method is DEPRECATED. Use delete_file_locations() instead.</span>
 <span class="sd">        Delete all locations of a stored data file and remove its record from indexd</span>
@@ -130,7 +130,7 @@ <h1>Source code for gen3.file</h1><div class="highlight"><pre>
 
 <div class="viewcode-block" id="Gen3File.delete_file_locations">
 <a class="viewcode-back" href="../../file.html#gen3.file.Gen3File.delete_file_locations">[docs]</a>
-    <span class="k">def</span> <span class="nf">delete_file_locations</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">guid</span><span class="p">):</span>
+    <span class="k">def</span><span class="w"> </span><span class="nf">delete_file_locations</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">guid</span><span class="p">):</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Delete all locations of a stored data file and remove its record from indexd</span>
 
@@ -147,7 +147,7 @@ <h1>Source code for gen3.file</h1><div class="highlight"><pre>
 
 <div class="viewcode-block" id="Gen3File.upload_file">
 <a class="viewcode-back" href="../../file.html#gen3.file.Gen3File.upload_file">[docs]</a>
-    <span class="k">def</span> <span class="nf">upload_file</span><span class="p">(</span>
+    <span class="k">def</span><span class="w"> </span><span class="nf">upload_file</span><span class="p">(</span>
         <span class="bp">self</span><span class="p">,</span> <span class="n">file_name</span><span class="p">,</span> <span class="n">authz</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">protocol</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">expires_in</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">bucket</span><span class="o">=</span><span class="kc">None</span>
     <span class="p">):</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
@@ -193,7 +193,7 @@ <h1>Source code for gen3.file</h1><div class="highlight"><pre>
         <span class="k">return</span> <span class="n">data</span></div>
 
 
-    <span class="k">def</span> <span class="nf">_ensure_dirpath_exists</span><span class="p">(</span><span class="n">path</span><span class="p">:</span> <span class="n">Path</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Path</span><span class="p">:</span>
+    <span class="k">def</span><span class="w"> </span><span class="nf">_ensure_dirpath_exists</span><span class="p">(</span><span class="n">path</span><span class="p">:</span> <span class="n">Path</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Path</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Utility to create a directory if missing.</span>
 <span class="sd">        Returns the path so that the call can be inlined in another call</span>
 <span class="sd">        Args:</span>
@@ -211,7 +211,7 @@ <h1>Source code for gen3.file</h1><div class="highlight"><pre>
 
 <div class="viewcode-block" id="Gen3File.download_single">
 <a class="viewcode-back" href="../../file.html#gen3.file.Gen3File.download_single">[docs]</a>
-    <span class="k">def</span> <span class="nf">download_single</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">object_id</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+    <span class="k">def</span><span class="w"> </span><span class="nf">download_single</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">object_id</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Download a single file using its GUID.</span>
 
@@ -271,7 +271,7 @@ <h1>Source code for gen3.file</h1><div class="highlight"><pre>
 
 <div class="viewcode-block" id="Gen3File.upload_file_to_guid">
 <a class="viewcode-back" href="../../file.html#gen3.file.Gen3File.upload_file_to_guid">[docs]</a>
-    <span class="k">def</span> <span class="nf">upload_file_to_guid</span><span class="p">(</span>
+    <span class="k">def</span><span class="w"> </span><span class="nf">upload_file_to_guid</span><span class="p">(</span>
         <span class="bp">self</span><span class="p">,</span> <span class="n">guid</span><span class="p">,</span> <span class="n">file_name</span><span class="p">,</span> <span class="n">protocol</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">expires_in</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">bucket</span><span class="o">=</span><span class="kc">None</span>
     <span class="p">):</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>