Merge pull request #309 from bsipocz/CI_execute_for_rendering

CI: execute for rendering 0434f1e

_sources/forced_photometry/multiband_photometry.md

@@ -108,7 +108,10 @@ from photometry import Band
 from photometry import lookup_img_pair
 #from prepare_prf import prepare_prf
 
-# temporarily let the notebook start without tractor as dependency
+# This code is to parse cloud access information; currently in `code_src`, eventually will be part of pyvo
+import fornax
+
+# temporarily let the notebook start without tractor as a dependency
 try:
     from find_nconfsources import find_nconfsources
 except ImportError:
@@ -181,24 +184,6 @@ print("Number of objects: ", len(cosmos_table))
 
 Details here may change as the prototype code is being added to the appropriate libraries, as well as the data holding to the appropriate NGAP storage as opposed to IRSA resources.
 
-```{code-cell} ipython3
-# Temporary solution, remove when the fornax API is added to the image
-# This relies on the assumption that https://github.com/nasa-fornax/fornax-cloud-access-API is being cloned to this environment.
-# If it's not, then run a ``git clone https://github.com/nasa-fornax/fornax-cloud-access-API --depth=1`` from a terminal at the highest directory root.
-# You may need to update the fork if you forked it in the past
-
-import os
-if not os.path.exists('../../fornax-cloud-access-API'):
-    ! git clone https://github.com/nasa-fornax/fornax-cloud-access-API --depth=1 ../../fornax-cloud-access-API
-```
-
-```{code-cell} ipython3
-sys.path.append('../../fornax-cloud-access-API')
-
-import pyvo
-import fornax
-```
-
 ```{code-cell} ipython3
 # Getting the COSMOS address from the registry to follow PyVO user case approach. We could hardwire it.
 image_services = pyvo.regsearch(servicetype='sia')
@@ -796,10 +781,10 @@ nway_write_header('data/multiband_phot.fits', 'OPT', float((2*rad_in_arcmin/60)*
 ```
 
 ```{code-cell} ipython3
-# call nway
-!nway.py 'data/Chandra/COSMOS_chandra.fits' :ERROR_RADIUS 'data/multiband_phot.fits' 0.1 --out=data/Chandra/chandra_multiband.fits --radius 15 --prior-completeness 0.9
+%%bash
 
-#!/opt/conda/bin/nway.py 'data/Chandra/COSMOS_chandra.fits' :ERROR_RADIUS 'data/multiband_phot.fits' 0.1 --out=data/Chandra/chandra_multiband.fits --radius 15 --prior-completeness 0.9
+# call nway
+nway.py 'data/Chandra/COSMOS_chandra.fits' :ERROR_RADIUS 'data/multiband_phot.fits' 0.1 --out=data/Chandra/chandra_multiband.fits --radius 15 --prior-completeness 0.9
 ```
 
 ```{code-cell} ipython3

_sources/light_curves/ML_AGNzoo.md

@@ -97,15 +97,14 @@ colors = [
 custom_cmap = LinearSegmentedColormap.from_list("custom_theme", colors[1:])
 ```
 
-
-
-
 ## 1) Loading data
 Here we load a parquet file of light curves generated using the light_curve_generator notebook in this same GitHub repo. With that light_curve_generator notebook, you can build your favorite sample from different sources in the literature and grab the data from archives of interest. This sample contains both spatial coordinates and categorical labels for each AGN. The labels are generated by a bitwise addition of a set of binary indicators. Each binary indicator corresponds to the AGN's membership in various categories, such as being an SDSS_QSO or a WISE_Variable. For example, an AGN that is both an SDSS_QSO, a WISE_Variable, and also shows 'Turn_on' characteristics, would have a label calculated by combining these specific binary indicators using bitwise addition.
 
 ```{code-cell} ipython3
+%%bash
+
 # To download the data file containing the light curves from Googledrive
-!gdown 1gb2vWn0V2unstElGTTrHIIWIftHbXJvz -O ./data/df_lc_020724.parquet.gzip
+gdown 1gb2vWn0V2unstElGTTrHIIWIftHbXJvz -O ./data/df_lc_020724.parquet.gzip
 ```
 
 ```{code-cell} ipython3
@@ -166,7 +165,7 @@ for (_, band), times in df_lc.reset_index().groupby(["objectid", "band"]).time:
     cadence[band].append(len(times))
     if times.max() - times.min() > 0:
         timerange[band].append(np.round(times.max() - times.min(), 1))
-        
+
 plt.figure(figsize=(20,4))
 plt.title(r'Time range and cadence covered in each in each waveband averaged over this sample:')
 for el in cadence.keys():
@@ -572,7 +571,7 @@ for index, f in enumerate(fzr):
         if label not in labc:
             labc[label] = []  # Initialize the list for this label if it's not already in labc
         labc[label].append(index)  # Append the current index to the list of indices for this label
-        
+
 sm = sompy.SOMFactory.build(data, mapsize=[msz0,msz1], mapshape='planar', lattice='rect', initialization='pca')
 sm.train(n_job=4, shared_memory = 'no')
 ```

forced_photometry/multiband_photometry.html

@@ -566,7 +566,10 @@ <h2>Non-standard Dependencies<a class="headerlink" href="#non-standard-dependenc
 <span class="kn">from</span> <span class="nn">photometry</span> <span class="kn">import</span> <span class="n">lookup_img_pair</span>
 <span class="c1">#from prepare_prf import prepare_prf</span>
 
-<span class="c1"># temporarily let the notebook start without tractor as dependency</span>
+<span class="c1"># This code is to parse cloud access information; currently in `code_src`, eventually will be part of pyvo</span>
+<span class="kn">import</span> <span class="nn">fornax</span>
+
+<span class="c1"># temporarily let the notebook start without tractor as a dependency</span>
 <span class="k">try</span><span class="p">:</span>
     <span class="kn">from</span> <span class="nn">find_nconfsources</span> <span class="kn">import</span> <span class="n">find_nconfsources</span>
 <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
@@ -653,30 +656,6 @@ <h3>Use the fornax cloud access API to obtain the IRAC data from the IRSA S3 buc
 <p>Details here may change as the prototype code is being added to the appropriate libraries, as well as the data holding to the appropriate NGAP storage as opposed to IRSA resources.</p>
 <div class="cell docutils container">
 <div class="cell_input docutils container">
-<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Temporary solution, remove when the fornax API is added to the image</span>
-<span class="c1"># This relies on the assumption that https://github.com/nasa-fornax/fornax-cloud-access-API is being cloned to this environment.</span>
-<span class="c1"># If it&#39;s not, then run a ``git clone https://github.com/nasa-fornax/fornax-cloud-access-API --depth=1`` from a terminal at the highest directory root.</span>
-<span class="c1"># You may need to update the fork if you forked it in the past</span>
-
-<span class="kn">import</span> <span class="nn">os</span>
-<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="s1">&#39;../../fornax-cloud-access-API&#39;</span><span class="p">):</span>
-    <span class="err">!</span> <span class="n">git</span> <span class="n">clone</span> <span class="n">https</span><span class="p">:</span><span class="o">//</span><span class="n">github</span><span class="o">.</span><span class="n">com</span><span class="o">/</span><span class="n">nasa</span><span class="o">-</span><span class="n">fornax</span><span class="o">/</span><span class="n">fornax</span><span class="o">-</span><span class="n">cloud</span><span class="o">-</span><span class="n">access</span><span class="o">-</span><span class="n">API</span> <span class="o">--</span><span class="n">depth</span><span class="o">=</span><span class="mi">1</span> <span class="o">../../</span><span class="n">fornax</span><span class="o">-</span><span class="n">cloud</span><span class="o">-</span><span class="n">access</span><span class="o">-</span><span class="n">API</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="cell docutils container">
-<div class="cell_input docutils container">
-<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;../../fornax-cloud-access-API&#39;</span><span class="p">)</span>
-
-<span class="kn">import</span> <span class="nn">pyvo</span>
-<span class="kn">import</span> <span class="nn">fornax</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="cell docutils container">
-<div class="cell_input docutils container">
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Getting the COSMOS address from the registry to follow PyVO user case approach. We could hardwire it.</span>
 <span class="n">image_services</span> <span class="o">=</span> <span class="n">pyvo</span><span class="o">.</span><span class="n">regsearch</span><span class="p">(</span><span class="n">servicetype</span><span class="o">=</span><span class="s1">&#39;sia&#39;</span><span class="p">)</span>
 <span class="n">irsa_cosmos</span> <span class="o">=</span> <span class="p">[</span><span class="n">s</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">image_services</span> <span class="k">if</span> <span class="s1">&#39;irsa&#39;</span> <span class="ow">in</span> <span class="n">s</span><span class="o">.</span><span class="n">ivoid</span> <span class="ow">and</span> <span class="s1">&#39;cosmos&#39;</span> <span class="ow">in</span> <span class="n">s</span><span class="o">.</span><span class="n">ivoid</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
@@ -1343,10 +1322,10 @@ <h3>4b. Run <code class="docutils literal notranslate"><span class="pre">nway</s
 </div>
 <div class="cell docutils container">
 <div class="cell_input docutils container">
-<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># call nway</span>
-<span class="err">!</span><span class="n">nway</span><span class="o">.</span><span class="n">py</span> <span class="s1">&#39;data/Chandra/COSMOS_chandra.fits&#39;</span> <span class="p">:</span><span class="n">ERROR_RADIUS</span> <span class="s1">&#39;data/multiband_phot.fits&#39;</span> <span class="mf">0.1</span> <span class="o">--</span><span class="n">out</span><span class="o">=</span><span class="n">data</span><span class="o">/</span><span class="n">Chandra</span><span class="o">/</span><span class="n">chandra_multiband</span><span class="o">.</span><span class="n">fits</span> <span class="o">--</span><span class="n">radius</span> <span class="mi">15</span> <span class="o">--</span><span class="n">prior</span><span class="o">-</span><span class="n">completeness</span> <span class="mf">0.9</span>
+<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="o">%%</span><span class="n">bash</span>
 
-<span class="c1">#!/opt/conda/bin/nway.py &#39;data/Chandra/COSMOS_chandra.fits&#39; :ERROR_RADIUS &#39;data/multiband_phot.fits&#39; 0.1 --out=data/Chandra/chandra_multiband.fits --radius 15 --prior-completeness 0.9</span>
+<span class="c1"># call nway</span>
+<span class="n">nway</span><span class="o">.</span><span class="n">py</span> <span class="s1">&#39;data/Chandra/COSMOS_chandra.fits&#39;</span> <span class="p">:</span><span class="n">ERROR_RADIUS</span> <span class="s1">&#39;data/multiband_phot.fits&#39;</span> <span class="mf">0.1</span> <span class="o">--</span><span class="n">out</span><span class="o">=</span><span class="n">data</span><span class="o">/</span><span class="n">Chandra</span><span class="o">/</span><span class="n">chandra_multiband</span><span class="o">.</span><span class="n">fits</span> <span class="o">--</span><span class="n">radius</span> <span class="mi">15</span> <span class="o">--</span><span class="n">prior</span><span class="o">-</span><span class="n">completeness</span> <span class="mf">0.9</span>
 </pre></div>
 </div>
 </div>

light_curves/ML_AGNzoo.html

@@ -532,8 +532,10 @@ <h2>1) Loading data<a class="headerlink" href="#loading-data" title="Link to thi
 <p>Here we load a parquet file of light curves generated using the light_curve_generator notebook in this same GitHub repo. With that light_curve_generator notebook, you can build your favorite sample from different sources in the literature and grab the data from archives of interest. This sample contains both spatial coordinates and categorical labels for each AGN. The labels are generated by a bitwise addition of a set of binary indicators. Each binary indicator corresponds to the AGN’s membership in various categories, such as being an SDSS_QSO or a WISE_Variable. For example, an AGN that is both an SDSS_QSO, a WISE_Variable, and also shows ‘Turn_on’ characteristics, would have a label calculated by combining these specific binary indicators using bitwise addition.</p>
 <div class="cell docutils container">
 <div class="cell_input docutils container">
-<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># To download the data file containing the light curves from Googledrive</span>
-<span class="err">!</span><span class="n">gdown</span> <span class="mi">1</span><span class="n">gb2vWn0V2unstElGTTrHIIWIftHbXJvz</span> <span class="o">-</span><span class="n">O</span> <span class="o">./</span><span class="n">data</span><span class="o">/</span><span class="n">df_lc_020724</span><span class="o">.</span><span class="n">parquet</span><span class="o">.</span><span class="n">gzip</span>
+<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="o">%%</span><span class="n">bash</span>
+
+<span class="c1"># To download the data file containing the light curves from Googledrive</span>
+<span class="n">gdown</span> <span class="mi">1</span><span class="n">gb2vWn0V2unstElGTTrHIIWIftHbXJvz</span> <span class="o">-</span><span class="n">O</span> <span class="o">./</span><span class="n">data</span><span class="o">/</span><span class="n">df_lc_020724</span><span class="o">.</span><span class="n">parquet</span><span class="o">.</span><span class="n">gzip</span>
 </pre></div>
 </div>
 </div>
@@ -606,7 +608,7 @@ <h3>1.1) What is in this sample<a class="headerlink" href="#what-is-in-this-samp
     <span class="n">cadence</span><span class="p">[</span><span class="n">band</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">times</span><span class="p">))</span>
     <span class="k">if</span> <span class="n">times</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">-</span> <span class="n">times</span><span class="o">.</span><span class="n">min</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
         <span class="n">timerange</span><span class="p">[</span><span class="n">band</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="n">times</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">-</span> <span class="n">times</span><span class="o">.</span><span class="n">min</span><span class="p">(),</span> <span class="mi">1</span><span class="p">))</span>
-        
+
 <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span><span class="mi">4</span><span class="p">))</span>
 <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;Time range and cadence covered in each in each waveband averaged over this sample:&#39;</span><span class="p">)</span>
 <span class="k">for</span> <span class="n">el</span> <span class="ow">in</span> <span class="n">cadence</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
@@ -1029,7 +1031,7 @@ <h3>3.2) Reduced dimensions on a SOM grid<a class="headerlink" href="#reduced-di
         <span class="k">if</span> <span class="n">label</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">labc</span><span class="p">:</span>
             <span class="n">labc</span><span class="p">[</span><span class="n">label</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>  <span class="c1"># Initialize the list for this label if it&#39;s not already in labc</span>
         <span class="n">labc</span><span class="p">[</span><span class="n">label</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">index</span><span class="p">)</span>  <span class="c1"># Append the current index to the list of indices for this label</span>
-        
+
 <span class="n">sm</span> <span class="o">=</span> <span class="n">sompy</span><span class="o">.</span><span class="n">SOMFactory</span><span class="o">.</span><span class="n">build</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">mapsize</span><span class="o">=</span><span class="p">[</span><span class="n">msz0</span><span class="p">,</span><span class="n">msz1</span><span class="p">],</span> <span class="n">mapshape</span><span class="o">=</span><span class="s1">&#39;planar&#39;</span><span class="p">,</span> <span class="n">lattice</span><span class="o">=</span><span class="s1">&#39;rect&#39;</span><span class="p">,</span> <span class="n">initialization</span><span class="o">=</span><span class="s1">&#39;pca&#39;</span><span class="p">)</span>
 <span class="n">sm</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="n">n_job</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">shared_memory</span> <span class="o">=</span> <span class="s1">&#39;no&#39;</span><span class="p">)</span>
 </pre></div>