Gen3Index fixes (#161)

Author: Avantol13

docs/_build/doctrees/environment.pickle

@@ -508,30 +508,39 @@ <h1>Source code for gen3.index</h1><div class="highlight"><pre>
                 <span class="n">urls</span> <span class="o">=</span> <span class="p">[]</span>
 
             <span class="n">json</span> <span class="o">=</span> <span class="p">{</span>
-                <span class="s2">&quot;urls&quot;</span><span class="p">:</span> <span class="n">urls</span><span class="p">,</span>
                 <span class="s2">&quot;form&quot;</span><span class="p">:</span> <span class="s2">&quot;object&quot;</span><span class="p">,</span>
                 <span class="s2">&quot;hashes&quot;</span><span class="p">:</span> <span class="n">hashes</span><span class="p">,</span>
                 <span class="s2">&quot;size&quot;</span><span class="p">:</span> <span class="n">size</span><span class="p">,</span>
-                <span class="s2">&quot;file_name&quot;</span><span class="p">:</span> <span class="n">file_name</span><span class="p">,</span>
-                <span class="s2">&quot;metadata&quot;</span><span class="p">:</span> <span class="n">metadata</span><span class="p">,</span>
-                <span class="s2">&quot;urls_metadata&quot;</span><span class="p">:</span> <span class="n">urls_metadata</span><span class="p">,</span>
-                <span class="s2">&quot;baseid&quot;</span><span class="p">:</span> <span class="n">baseid</span><span class="p">,</span>
-                <span class="s2">&quot;acl&quot;</span><span class="p">:</span> <span class="n">acl</span><span class="p">,</span>
-                <span class="s2">&quot;authz&quot;</span><span class="p">:</span> <span class="n">authz</span><span class="p">,</span>
-                <span class="s2">&quot;version&quot;</span><span class="p">:</span> <span class="n">version</span><span class="p">,</span>
+                <span class="s2">&quot;urls&quot;</span><span class="p">:</span> <span class="n">urls</span> <span class="ow">or</span> <span class="p">[],</span>
             <span class="p">}</span>
-
             <span class="k">if</span> <span class="n">did</span><span class="p">:</span>
                 <span class="n">json</span><span class="p">[</span><span class="s2">&quot;did&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">did</span>
+            <span class="k">if</span> <span class="n">file_name</span><span class="p">:</span>
+                <span class="n">json</span><span class="p">[</span><span class="s2">&quot;file_name&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_name</span>
+            <span class="k">if</span> <span class="n">metadata</span><span class="p">:</span>
+                <span class="n">json</span><span class="p">[</span><span class="s2">&quot;metadata&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">metadata</span>
+            <span class="k">if</span> <span class="n">baseid</span><span class="p">:</span>
+                <span class="n">json</span><span class="p">[</span><span class="s2">&quot;baseid&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">baseid</span>
+            <span class="k">if</span> <span class="n">acl</span><span class="p">:</span>
+                <span class="n">json</span><span class="p">[</span><span class="s2">&quot;acl&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">acl</span>
+            <span class="k">if</span> <span class="n">urls_metadata</span><span class="p">:</span>
+                <span class="n">json</span><span class="p">[</span><span class="s2">&quot;urls_metadata&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">urls_metadata</span>
+            <span class="k">if</span> <span class="n">version</span><span class="p">:</span>
+                <span class="n">json</span><span class="p">[</span><span class="s2">&quot;version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">version</span>
+            <span class="k">if</span> <span class="n">authz</span><span class="p">:</span>
+                <span class="n">json</span><span class="p">[</span><span class="s2">&quot;authz&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">authz</span>
+
+            <span class="c1"># aiohttp only allows basic auth with their built in auth, so we</span>
+            <span class="c1"># need to manually add JWT auth header</span>
+            <span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;Authorization&quot;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">auth</span><span class="o">.</span><span class="n">_get_auth_value</span><span class="p">()}</span>
 
             <span class="k">async</span> <span class="k">with</span> <span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
                 <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">url</span><span class="si">}</span><span class="s2">/index/&quot;</span><span class="p">,</span>
                 <span class="n">json</span><span class="o">=</span><span class="n">json</span><span class="p">,</span>
-                <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;content-type&quot;</span><span class="p">:</span> <span class="s2">&quot;application/json&quot;</span><span class="p">},</span>
+                <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
                 <span class="n">ssl</span><span class="o">=</span><span class="n">_ssl</span><span class="p">,</span>
-                <span class="n">auth</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">auth</span><span class="p">,</span>
             <span class="p">)</span> <span class="k">as</span> <span class="n">response</span><span class="p">:</span>
-                <span class="n">raise_for_status_and_print_error</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+                <span class="k">assert</span> <span class="n">response</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="mi">200</span><span class="p">,</span> <span class="k">await</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
                 <span class="n">response</span> <span class="o">=</span> <span class="k">await</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
 
         <span class="k">return</span> <span class="n">response</span></div>
@@ -750,6 +759,8 @@ <h1>Source code for gen3.index</h1><div class="highlight"><pre>
         <span class="n">acl</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
         <span class="n">authz</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
         <span class="n">urls_metadata</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+        <span class="n">_ssl</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
     <span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Asynchronous function to update a record in indexd.</span>
@@ -771,21 +782,38 @@ <h1>Source code for gen3.index</h1><div class="highlight"><pre>
                 <span class="s2">&quot;authz&quot;</span><span class="p">:</span> <span class="n">authz</span><span class="p">,</span>
                 <span class="s2">&quot;urls_metadata&quot;</span><span class="p">:</span> <span class="n">urls_metadata</span><span class="p">,</span>
             <span class="p">}</span>
-            <span class="n">record</span> <span class="o">=</span> <span class="k">await</span> <span class="n">async_get_record</span><span class="p">(</span><span class="n">guid</span><span class="p">)</span>
+            <span class="n">record</span> <span class="o">=</span> <span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">async_get_record</span><span class="p">(</span><span class="n">guid</span><span class="p">)</span>
             <span class="n">revision</span> <span class="o">=</span> <span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;rev&quot;</span><span class="p">)</span>
 
             <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">updatable_attrs</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
                 <span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                     <span class="n">record</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
 
+            <span class="k">del</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;created_date&quot;</span><span class="p">]</span>
+            <span class="k">del</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;rev&quot;</span><span class="p">]</span>
+            <span class="k">del</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;updated_date&quot;</span><span class="p">]</span>
+            <span class="k">del</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;version&quot;</span><span class="p">]</span>
+            <span class="k">del</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;uploader&quot;</span><span class="p">]</span>
+            <span class="k">del</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;form&quot;</span><span class="p">]</span>
+            <span class="k">del</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;urls_metadata&quot;</span><span class="p">]</span>
+            <span class="k">del</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;baseid&quot;</span><span class="p">]</span>
+            <span class="k">del</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;size&quot;</span><span class="p">]</span>
+            <span class="k">del</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;hashes&quot;</span><span class="p">]</span>
+            <span class="k">del</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;did&quot;</span><span class="p">]</span>
+
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;PUT-ing record: </span><span class="si">{</span><span class="n">record</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+
+            <span class="c1"># aiohttp only allows basic auth with their built in auth, so we</span>
+            <span class="c1"># need to manually add JWT auth header</span>
+            <span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;Authorization&quot;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">auth</span><span class="o">.</span><span class="n">_get_auth_value</span><span class="p">()}</span>
+
             <span class="k">async</span> <span class="k">with</span> <span class="n">session</span><span class="o">.</span><span class="n">put</span><span class="p">(</span>
-                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">url</span><span class="si">}</span><span class="s2">/index/</span><span class="si">{</span><span class="n">guid</span><span class="si">}</span><span class="s2">/rev=</span><span class="si">{</span><span class="n">revision</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
+                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">url</span><span class="si">}</span><span class="s2">/index/</span><span class="si">{</span><span class="n">guid</span><span class="si">}</span><span class="s2">?rev=</span><span class="si">{</span><span class="n">revision</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
                 <span class="n">json</span><span class="o">=</span><span class="n">record</span><span class="p">,</span>
-                <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;content-type&quot;</span><span class="p">:</span> <span class="s2">&quot;application/json&quot;</span><span class="p">},</span>
+                <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
                 <span class="n">ssl</span><span class="o">=</span><span class="n">_ssl</span><span class="p">,</span>
-                <span class="n">auth</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">auth</span><span class="p">,</span>
             <span class="p">)</span> <span class="k">as</span> <span class="n">response</span><span class="p">:</span>
-                <span class="n">raise_for_status_and_print_error</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+                <span class="k">assert</span> <span class="n">response</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="mi">200</span><span class="p">,</span> <span class="k">await</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
                 <span class="n">response</span> <span class="o">=</span> <span class="k">await</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
 
         <span class="k">return</span> <span class="n">response</span></div>

docs/_build/doctrees/file.doctree

@@ -224,7 +224,6 @@ <h1>Source code for gen3.tools.indexing.download_manifest</h1><div class="highli
         <span class="n">input_record_chunks</span> <span class="o">=</span> <span class="p">(</span>
             <span class="s2">&quot;|||&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">,</span> <span class="n">record_chunks</span><span class="p">[</span><span class="n">x</span><span class="p">]))</span> <span class="k">if</span> <span class="n">record_chunks</span> <span class="k">else</span> <span class="s2">&quot;|||&quot;</span>
         <span class="p">)</span>
-        <span class="nb">print</span><span class="p">(</span><span class="n">input_record_chunks</span><span class="p">)</span>
 
         <span class="c1"># write record_checksum chunks to temporary files since the size can overload</span>
         <span class="c1"># command line arguments</span>

docs/_build/doctrees/indexing.doctree

@@ -91,6 +91,20 @@ <h1>Gen3 File Class<a class="headerlink" href="#gen3-file-class" title="Permalin
 </dl>
 </dd></dl>
 
+<dl class="py method">
+<dt class="sig sig-object py" id="gen3.file.Gen3File.download_single">
+<span class="sig-name descname"><span class="pre">download_single</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">object_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/gen3/file.html#Gen3File.download_single"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.file.Gen3File.download_single" title="Permalink to this definition">¶</a></dt>
+<dd><p>Download a single file using its GUID.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>object_id</strong> (<em>str</em>) – The file’s unique ID</p></li>
+<li><p><strong>path</strong> (<em>str</em>) – Path to store the downloaded file at</p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
 <dl class="py method">
 <dt class="sig sig-object py" id="gen3.file.Gen3File.get_presigned_url">
 <span class="sig-name descname"><span class="pre">get_presigned_url</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">guid</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">protocol</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/gen3/file.html#Gen3File.get_presigned_url"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.file.Gen3File.get_presigned_url" title="Permalink to this definition">¶</a></dt>
@@ -199,6 +213,7 @@ <h3>Navigation</h3>
 <li class="toctree-l2"><a class="reference internal" href="#gen3.file.Gen3File"><code class="docutils literal notranslate"><span class="pre">Gen3File</span></code></a><ul>
 <li class="toctree-l3"><a class="reference internal" href="#gen3.file.Gen3File.delete_file"><code class="docutils literal notranslate"><span class="pre">Gen3File.delete_file()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="#gen3.file.Gen3File.delete_file_locations"><code class="docutils literal notranslate"><span class="pre">Gen3File.delete_file_locations()</span></code></a></li>
+<li class="toctree-l3"><a class="reference internal" href="#gen3.file.Gen3File.download_single"><code class="docutils literal notranslate"><span class="pre">Gen3File.download_single()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="#gen3.file.Gen3File.get_presigned_url"><code class="docutils literal notranslate"><span class="pre">Gen3File.get_presigned_url()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="#gen3.file.Gen3File.upload_file"><code class="docutils literal notranslate"><span class="pre">Gen3File.upload_file()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="#gen3.file.Gen3File.upload_file_to_guid"><code class="docutils literal notranslate"><span class="pre">Gen3File.upload_file_to_guid()</span></code></a></li>