Merge pull request #121 from ncats/develop

Major Version Update- BTEP Presentations

.hooks/startup.sh

@@ -11,4 +11,4 @@ set -e
 
 pip install scikit-learn streamlit-extras squidpy split-file-reader st-pages dill pympler objsize
 mamba install -y -q natsort "foundry-transforms-lib-python>=0.578.0"
-mamba install -y gcc_linux-64 gxx_linux-64 && mamba install -y hnswlib
+mamba install -y gcc_linux-64 gxx_linux-64 && mamba install -y python-annoy && mamba install -y hnswlib

.streamlit/pages.toml

@@ -34,33 +34,33 @@ name = "Main"
 icon = ""
 
 [[pages]]
-name = "Phenotyping 🖊️"
+name = "Phenotype Clustering Workflow ✨"
 icon = ""
 is_section = true
 
 [[pages]]
-path = "pages/multiaxial_gating.py"
-name = "Raw Intensities"
+path = "pages/05a_Pheno_Cluster.py"
+name = "Phenotype Clustering"
 icon = ""
 
 [[pages]]
-path = "pages/02_phenotyping.py"
-name = "Thresholded Intensities"
+path = "pages/05b_Pheno_Cluster.py"
+name = "Clusters Differential Expression"
 icon = ""
 
 [[pages]]
-name = "Phenotype Clustering Workflow ✨"
+name = "Phenotyping 🖊️"
 icon = ""
 is_section = true
 
 [[pages]]
-path = "pages/05a_Pheno_Cluster.py"
-name = "Phenotype Clustering"
+path = "pages/multiaxial_gating.py"
+name = "Raw Intensities"
 icon = ""
 
 [[pages]]
-path = "pages/05b_Pheno_Cluster.py"
-name = "Clusters Differential Expression"
+path = "pages/02_phenotyping.py"
+name = "Thresholded Intensities"
 icon = ""
 
 [[pages]]
@@ -128,12 +128,17 @@ path = "pages/memory_analyzer.py"
 name = "Memory Analyzer"
 icon = ""
 
-# [[pages]]
-# name = "In Development"
-# icon = ""
-# is_section = true
+[[pages]]
+name = "Radial Profiles"
+icon = ""
+is_section = true
+
+[[pages]]
+path = "pages/radial_profiles_app.py"
+name = "Calculations"
+icon = ""
 
-# [[pages]]
-# path = "pages/macro_radial_density.py"
-# name = "Macro Radial Density"
-# icon = ""
+[[pages]]
+path = "pages/radial_profiles_app_plotting_aggregated_results.py"
+name = "Plots"
+icon = ""

Multiplex_Analysis_Web_Apps-orig.py

@@ -0,0 +1,45 @@
+'''
+Top level Streamlit Application for MAWA
+'''
+
+# Import relevant libraries
+import os
+import streamlit as st
+
+# Import relevant libraries
+import nidap_dashboard_lib as ndl   # Useful functions for dashboards connected to NIDAP
+import app_top_of_page as top
+import streamlit_dataframe_editor as sde
+
+def main():
+    '''
+    Define the single, main function
+    '''
+
+    # Set a wide layout
+    st.set_page_config(layout="wide")
+
+    input_path = './input'
+    if not os.path.exists(input_path):
+        os.makedirs(input_path)
+
+    output_path = './output'
+    if not os.path.exists(output_path):
+        os.makedirs(output_path)
+
+    # Run streamlit-dataframe-editor library initialization tasks at the top of the page
+    st.session_state = sde.initialize_session_state(st.session_state)
+
+    # Run Top of Page (TOP) functions
+    st.session_state = top.top_of_page_reqs(st.session_state)
+
+    # Markdown text
+    intro_markdown = ndl.read_markdown_file('markdown/MAWA_WelcomePage.md')
+    st.markdown(intro_markdown, unsafe_allow_html=True)
+
+    # Run streamlit-dataframe-editor library finalization tasks at the bottom of the page
+    st.session_state = sde.finalize_session_state(st.session_state)
+
+# Call the main function
+if __name__ == '__main__':
+    main()

Multiplex_Analysis_Web_Apps.py

@@ -1,45 +1,194 @@
-'''
-Top level Streamlit Application for MAWA
-'''
-
-# Import relevant libraries
-import os
 import streamlit as st
-
-# Import relevant libraries
+import os
+from streamlit_extras.app_logo import add_logo
+import streamlit_session_state_management
 import nidap_dashboard_lib as ndl   # Useful functions for dashboards connected to NIDAP
-import app_top_of_page as top
-import streamlit_dataframe_editor as sde
+import streamlit_utils
+import numpy as np
+import subprocess
+import platform_io
+import install_missing_packages
 
-def main():
+install_missing_packages.live_package_installation()
+
+# Note if any of the following imports having "  # slow" are not commented out, there is a delay in running the forking test
+from pages2 import data_import_and_export
+from pages2 import datafile_format_unifier
+from pages2 import open_file
+from pages2 import robust_scatter_plotter
+from pages2 import multiaxial_gating
+from pages2 import thresholded_phenotyping  # slow due to things ultimately importing umap
+from pages2 import adaptive_phenotyping
+from pages2 import Pheno_Cluster_a  # "slow" for forking test initialization
+from pages2 import Pheno_Cluster_b  # "slow" for forking test initialization
+from pages2 import Tool_parameter_selection
+from pages2 import Run_workflow
+from pages2 import Display_individual_ROI_heatmaps
+from pages2 import Display_average_heatmaps
+from pages2 import Display_average_heatmaps_per_annotation
+from pages2 import Display_ROI_P_values_overlaid_on_slides
+from pages2 import Neighborhood_Profiles  # slow due to things ultimately importing umap
+from pages2 import UMAP_Analyzer  # slow due to things ultimately importing umap
+from pages2 import Clusters_Analyzer  # slow due to things ultimately importing umap
+from pages2 import memory_analyzer
+from pages2 import radial_bins_plots
+from pages2 import radial_profiles_analysis
+from pages2 import preprocessing
+from pages2 import results_transfer
+# from pages2 import forking_test
+
+
+def welcome_page():
+    # Markdown text
+    intro_markdown = ndl.read_markdown_file('markdown/MAWA_WelcomePage.md')
+    st.markdown(intro_markdown, unsafe_allow_html=True)
+
+
+def platform_is_nidap():
+    '''
+    Check if the Streamlit application is operating on NIDAP
+    '''
+    return np.any(['nidap.nih.gov' in x for x in subprocess.run('conda config --show channels', shell=True, capture_output=True).stdout.decode().split('\n')[1:-1]])
+
+
+def check_for_platform(session_state):
     '''
-    Define the single, main function
+    Set the platform parameters based on the platform the Streamlit app is running on
     '''
+    # Initialize the platform object
+    if 'platform' not in session_state:
+        session_state['platform'] = platform_io.Platform(platform=('nidap' if platform_is_nidap() else 'local'))
+    return session_state
+
+
+def main():
 
-    # Set a wide layout
     st.set_page_config(layout="wide")
 
+    # Use the new st.naviation()/st.Page() API to create a multi-page app
+    pg = st.navigation({
+        'Home 🏠':
+            [
+                st.Page(welcome_page, title="Welcome", url_path='home')
+            ],
+        'File Handling 🗄️':
+            [
+                st.Page(data_import_and_export.main, title="Data Import and Export", url_path='data_import_and_export'),
+                st.Page(datafile_format_unifier.main, title="Datafile Unification", url_path='datafile_unification'),
+                st.Page(open_file.main, title="Open File", url_path='open_file')
+            ],
+        'Coordinate Scatter Plotter 🌟':
+            [
+                st.Page(robust_scatter_plotter.main, title="Coordinate Scatter Plotter", url_path='coordinate_scatter_plotter')
+            ],
+        'Phenotyping 🧬':
+            [
+                st.Page(multiaxial_gating.main, title="Using Raw Intensities", url_path='using_raw_intensities'),
+                st.Page(thresholded_phenotyping.main, title="Using Thresholded Intensities", url_path='using_thresholded_intensities'),
+                st.Page(adaptive_phenotyping.main, title="Adaptive Phenotyping", url_path='adaptive_phenotyping')
+            ],
+         'Phenotype Clustering Workflow ✨':
+             [
+                 st.Page(Pheno_Cluster_a.main, title="Unsupervised Phenotype Clustering", url_path='unsupervised_phenotype_clustering'),
+                 st.Page(Pheno_Cluster_b.main, title="Differential Intensity", url_path='differential_intensity')
+             ],
+        'Spatial Interaction Tool 🗺️':
+            [
+                st.Page(Tool_parameter_selection.main, title="Tool Parameter Selection", url_path='tool_parameter_selection'),
+                st.Page(Run_workflow.main, title="Run SIT Workflow", url_path='run_sit_workflow'),
+                st.Page(Display_individual_ROI_heatmaps.main, title="Display Individual ROI Heatmaps", url_path='display_individual_roi_heatmaps'),
+                st.Page(Display_average_heatmaps.main, title="Display Average Heatmaps", url_path='display_average_heatmaps'),
+                st.Page(Display_average_heatmaps_per_annotation.main, title="Display Average Heatmaps per Annotation", url_path='display_average_heatmaps_per_annotation'),
+                st.Page(Display_ROI_P_values_overlaid_on_slides.main, title="Display ROI P Values Overlaid on Slides", url_path='display_roi_p_values_overlaid_on_slides')
+            ],
+        'Neighborhood Profiles Workflow 🌳':
+            [
+                st.Page(Neighborhood_Profiles.main, title="Neighborhood Profiles", url_path='neighborhood_profiles'),
+                st.Page(UMAP_Analyzer.main, title="UMAP Differences Analyzer", url_path='umap_differences_analyzer'),
+                st.Page(Clusters_Analyzer.main, title="Clusters Analyzer", url_path='clusters_analyzer')
+            ],
+        'Radial Profiles 🌀':
+            [
+                st.Page(radial_bins_plots.main, title="Radial Bins Plots", url_path='radial_bins_plots'),
+                st.Page(radial_profiles_analysis.main, title="Radial Profiles Analysis", url_path='radial_profiles_analysis')
+            ],
+        'Utilities 🛠️':
+            [
+                st.Page(preprocessing.main, title="Preprocessing", url_path='preprocessing'),
+                st.Page(memory_analyzer.main, title="Memory Analyzer", url_path='memory_analyzer'),
+                st.Page(results_transfer.main, title="Results Transfer", url_path='results_transfer'),
+                # st.Page(forking_test.main, title="Forking Test", url_path='forking_test')
+            ]
+        })
+
+    # Ensure the input/output directories exist
     input_path = './input'
     if not os.path.exists(input_path):
         os.makedirs(input_path)
-
     output_path = './output'
     if not os.path.exists(output_path):
         os.makedirs(output_path)
 
-    # Run streamlit-dataframe-editor library initialization tasks at the top of the page
-    st.session_state = sde.initialize_session_state(st.session_state)
+    # For widget persistence, we need always copy the session state to itself, being careful with widgets that cannot be persisted, like st.data_editor() (where we use the "__do_not_persist" suffix to avoid persisting it)
+    for key in st.session_state.keys():
+        if (not key.endswith('__do_not_persist')) and (not key.startswith('FormSubmitter:')):
+            st.session_state[key] = st.session_state[key]
 
-    # Run Top of Page (TOP) functions
-    st.session_state = top.top_of_page_reqs(st.session_state)
+    # This is needed for the st.dataframe_editor() class (https://github.com/andrew-weisman/streamlit-dataframe-editor) but is also useful for seeing where we are and where we've been
+    st.session_state['current_page_name'] = pg.url_path if pg.url_path != '' else 'Home'
+    if 'previous_page_name' not in st.session_state:
+        st.session_state['previous_page_name'] = st.session_state['current_page_name']
 
-    # Markdown text
-    intro_markdown = ndl.read_markdown_file('markdown/MAWA_WelcomePage.md')
-    st.markdown(intro_markdown, unsafe_allow_html=True)
+    # Add logo to sidebar
+    add_logo('app_images/mawa_logo-width315.png', height=250)
+
+    # Determine whether this is the first time the app has been run
+    if 'app_has_been_run_at_least_once' not in st.session_state:
+        st.session_state['app_has_been_run_at_least_once'] = True
+        first_app_run = True
+    else:
+        first_app_run = False
+
+    # Run session state management in the sidebar
+    streamlit_session_state_management.execute(first_app_run)
+
+    # Initalize session_state values for streamlit processing
+    if 'init' not in st.session_state:
+        st.session_state = ndl.init_session_state(st.session_state)
+
+    # Sidebar organization
+    with st.sidebar:
+        st.write('**:book: [Documentation](https://ncats.github.io/multiplex-analysis-web-apps/)**')
+        with st.expander('Advanced:'):
+            benchmark_button = True
+            if benchmark_button:
+                st.button('Record Benchmarking', on_click = st.session_state.bc.save_run_to_csv)
+            if st.button('Calculate memory used by Python session'):
+                streamlit_utils.write_python_session_memory_usage()
+
+    # Check the platform
+    st.session_state = check_for_platform(st.session_state)
+
+    # Format tooltips
+    tooltip_style = """
+        <style>
+        div[data-baseweb="tooltip"] {
+        width: 250px;
+        }
+        </style>
+    """
+    st.markdown(tooltip_style,unsafe_allow_html=True)
+
+    # On every page, display its title
+    st.title(pg.title)
+
+    # Render the select page
+    pg.run()
+
+    # Update the previous page location
+    st.session_state['previous_page_name'] = st.session_state['current_page_name']
 
-    # Run streamlit-dataframe-editor library finalization tasks at the bottom of the page
-    st.session_state = sde.finalize_session_state(st.session_state)
 
-# Call the main function
+# Needed for rendering pages which use multiprocessing (https://docs.python.org/3/library/multiprocessing.html#the-spawn-and-forkserver-start-methods)
 if __name__ == '__main__':
     main()

SpatialUMAP.py

@@ -220,9 +220,7 @@ def calculate_density_matrix_for_all_images(self, cpu_pool_size = 8):
         with mp.Pool(processes=cpu_pool_size) as pool:
             results = pool.starmap(utils.fast_neighbors_counts_for_block2, kwargs_list)
 
-        print('Finished calculating density matrix for all images. Concatenating results...')
         df_density_matrix = pd.concat(self.get_dataframes(results))
-        print('Finished concatenating counts results.')
         full_array = None
         for ii, phenotype in enumerate(phenotypes):
             cols2Use = [f'{phenotype} in {x}' for x in range_strings]
@@ -231,10 +229,9 @@ def calculate_density_matrix_for_all_images(self, cpu_pool_size = 8):
                 full_array = array_set
             else:
                 full_array = np.dstack((full_array, array_set))
-        print('BBBBB')
+
         full_array_nan = np.isnan(full_array)
         full_array[full_array_nan] = 0
-        print('CCCCC')
 
         # Concatenate the results into a single dataframe
         return full_array
@@ -274,10 +271,11 @@ def __init__(self, dist_bin_um, um_per_px, area_downsample):
         # Mean Densities
         self.dens_df = pd.DataFrame()
         self.prop_df = pd.DataFrame()
-        self.dens_df_mean = pd.DataFrame(data = {'clust_label': ['No Cluster'], 
+        self.dens_df_mean = pd.DataFrame(data = {'clust_label': ['No Cluster'],
                                                  'phenotype': ['Other'], 
                                                  'dist_bin': [25], 
-                                                 'density_mean': [0]})
+                                                 'density_mean': [0],
+                                                 'density_sem': [0]})
         self.dens_df_se = pd.DataFrame()
         self.maxdens_df = pd.DataFrame()
 

app_top_of_page.py

@@ -6,7 +6,7 @@
 import numpy as np
 import platform_io
 import streamlit as st
-from st_pages import show_pages_from_config, add_indentation
+# from st_pages import show_pages_from_config, add_indentation
 from streamlit_extras.app_logo import add_logo
 import streamlit_session_state_management
 import streamlit_utils