From 5ddf55ab5253033d0c17829dc7f259bc4a9265b7 Mon Sep 17 00:00:00 2001
From: cmaceves <44014693+cmaceves@users.noreply.github.com>
Date: Tue, 16 Jul 2019 10:52:15 -0700
Subject: [PATCH 1/4] Update views.py
---
code/views.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/code/views.py b/code/views.py
index ffe0d10..c4a0447 100644
--- a/code/views.py
+++ b/code/views.py
@@ -690,7 +690,6 @@ def processcomparemultivariate():
with open(new_analysis_filename, 'wb') as f:
f.write(r.content)
- #output_png = os.path.join("./tempuploads", str(uuid.uuid4()))
- output_png = ("./tempuploads")
+ output_png = os.path.join("./tempuploads", task_id)
redu_pca.project_new_data(new_analysis_filename, output_png)
- return send_file("./tempuploads/index.html")
+ return send_file("./tempuploads/%s/index.html" %(task_id))
From f9e59a06b1c00b793e2d9e9aa69f1e79c48a8048 Mon Sep 17 00:00:00 2001
From: cmaceves <44014693+cmaceves@users.noreply.github.com>
Date: Wed, 24 Jul 2019 13:41:02 -0700
Subject: [PATCH 2/4] Add files via upload
---
code/redu_pca.py | 49 ++++++++++++++++++++++++++----------------------
1 file changed, 27 insertions(+), 22 deletions(-)
diff --git a/code/redu_pca.py b/code/redu_pca.py
index 51febd5..abfe7cf 100644
--- a/code/redu_pca.py
+++ b/code/redu_pca.py
@@ -10,19 +10,14 @@
from skbio.stats.ordination import OrdinationResults
from emperor import Emperor
import scipy.sparse as sps
-
-
-PATH_TO_COMPONENT_MATRIX = "./component_matrix.csv" #eigenvectors output by calculate_master_projection
-PATH_TO_ORIGINAL_PCA = "./original_pca.csv" #original PCA matrix of the original files
-PATH_TO_ORIGINAL_MAPPING_FILE = "./all_sampleinformation.tsv" #global ReDU metadata
-
+import config
### Given a file input occurrence table, creates the eigen vectors file defined above PATH_TO_COMPONENT_MATRIX, and PCA project of these files PATH_TO_ORIGINAL_PCA
def calculate_master_projection(input_file_occurrences_table, components = 5):
print("made it to original projection function")
#load data from master GNPS occurance table into memory
df_temp = pd.read_csv(input_file_occurrences_table, sep = "\t")
-
+
all_compound_occurances = df_temp["Compound_Name"]
all_file_occurances = df_temp["full_CCMS_path"]
@@ -49,14 +44,21 @@ def calculate_master_projection(input_file_occurrences_table, components = 5):
#convert it into the correct format for the return
sparse_occ_matrix = pd.DataFrame(index = list(unique_compounds), columns = list(unique_sample), data = matrix)
-
- #separate the sparse matrix into a compound list, value matrix, and samples names
- compound_list = list(sparse_occ_matrix.index)
- sample_list = list(sparse_occ_matrix.columns)
- new_matrix = sparse_occ_matrix.values.T #align so that the "features" are column headers
-
- #format the sample list so it can be merged later with metadata
- sample_list = [("f." + item) for item in sample_list]
+ print(sparse_occ_matrix.shape)
+
+ #bring in metadata
+ master_metadata_file = pd.read_csv(config.PATH_TO_ORIGINAL_MAPPING_FILE, "\t")
+ master_sample_list = master_metadata_file["filename"].tolist()
+
+ #reformatting the sample names for comparison
+ sample_list = ["f." + item for item in list(sparse_occ_matrix.columns)]
+ sparse_occ_matrix.columns = sample_list
+
+ #fitlering out samples thta are not contained within the metadata
+ sparse_occ_matrix = sparse_occ_matrix.loc[:, sparse_occ_matrix.columns.isin(master_sample_list)]
+ new_sample_list = sparse_occ_matrix.columns
+ compound_list = list(sparse_occ_matrix.index)
+ new_matrix = sparse_occ_matrix.T.values #align so that the "features" are column headers
#bring sklearn components into play
pca = PCA(n_components = components) #creating the instance
@@ -71,20 +73,20 @@ def calculate_master_projection(input_file_occurrences_table, components = 5):
#place eigenvalues and percent variance on the end of this file so it can be used for emperor projection
df_temp.loc[len(compound_list)] = eigenvalues
df_temp.loc[len(compound_list)+1] = percent_variance
- df_temp.to_csv("./component_matrix.csv")
+ df_temp.to_csv(config.PATH_TO_COMPONENT_MATRIX)
sklearn_output = pca.transform(new_matrix) #using sklearn to transform the output
#saving the "master pca" calculated by this function as a csv
- df_temp = pd.DataFrame(data = sklearn_output, index = sample_list)
- df_temp.to_csv("./original_pca.csv")
+ df_temp = pd.DataFrame(data = sklearn_output, index = new_sample_list)
+ df_temp.to_csv(config.PATH_TO_ORIGINAL_PCA)
### Given a new file occurrence table, creates a projection of the new data along with the old data and saves as a png output
def project_new_data(new_file_occurrence_table, output_file):
new_matrix = np.array([])
file_list = []
- component_matrix = pd.read_csv(PATH_TO_COMPONENT_MATRIX, sep = ",") #read in the eigenvectors
+ component_matrix = pd.read_csv(config.PATH_TO_COMPONENT_MATRIX, sep = ",") #read in the eigenvectors
#grab the eigenvalues, percent explained variance values, and then drop them from the dataframe
eigenvalues = list(component_matrix.iloc[-2,:])[1:]
@@ -134,7 +136,7 @@ def project_new_data(new_file_occurrence_table, output_file):
new_pca_df["type"] = "new"
#load and format the original pca
- original_pca_df = pd.read_csv(PATH_TO_ORIGINAL_PCA, sep = ",")
+ original_pca_df = pd.read_csv(config.PATH_TO_ORIGINAL_PCA, sep = ",")
original_pca_df.rename(columns = {'Unnamed: 0': 'SampleID'}, inplace = True)
original_pca_df.set_index(['SampleID'], inplace=True)
original_pca_df["type"] = "OG"
@@ -159,17 +161,20 @@ def emperor_output(sklearn_output, full_file_list, eigenvalues, percent_variance
ores = OrdinationResults(long_method_name = "principal component analysis", short_method_name = "pcoa", eigvals = eigvals, samples = samples, proportion_explained = p_explained)
#this first part is for the global metadata file
- global_metadata = pd.read_csv(PATH_TO_ORIGINAL_MAPPING_FILE, sep = "\t")
+ global_metadata = pd.read_csv(config.PATH_TO_ORIGINAL_MAPPING_FILE, sep = "\t")
+ global_metadata_headers = global_metadata.columns.tolist()
global_metadata.rename(columns = {'filename': 'SampleID'}, inplace = True)
global_metadata["type"] = "Global Data"
global_metadata.set_index("SampleID", inplace = True)
-
#this part is for the user uploaded metadata file
metadata_uploaded = pd.DataFrame({"SampleID": new_files, "type":["Your Data"] * len(new_files)})
+ for item in global_metadata_headers:
+ metadata_uploaded[item] = ["Your Data"] * len(new_files)
metadata_uploaded.set_index("SampleID", inplace = True)
common = pd.concat([global_metadata, metadata_uploaded])
+
#so you need to align the metadata and the files contained within the ordination file BEFORE feeding it into the Emperor thing otherwise it doesn't like to output results
final_metadata, unused = common.align(samples, join = "right", axis = 0)
From bda36fa57430d20b94ea8e74db72c7217ba30e42 Mon Sep 17 00:00:00 2001
From: cmaceves <44014693+cmaceves@users.noreply.github.com>
Date: Wed, 21 Aug 2019 13:29:06 -0700
Subject: [PATCH 3/4] placing the redu gif logo on the homepage
---
code/templates/homepage.html | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/code/templates/homepage.html b/code/templates/homepage.html
index 9447c54..df95e8c 100644
--- a/code/templates/homepage.html
+++ b/code/templates/homepage.html
@@ -6,7 +6,7 @@
-
Reanalysis of Data User Interface for MS2 (ReDU-MS2)
+
@@ -27,7 +27,7 @@ Analyze Your Data
@@ -44,7 +44,7 @@ Chemicals were annotated via GNPS library matching (default parameters) in all samples. This options provides the ability to see how many times a chemical annotation was observed, the files in which in which it was observed, and a link to the sample information associated with the chemical annotation.
+
+ ((total_compounds)) total unique chemicals were annotated in all files across ReDU.
+ This encompasses ((total_identifications)) MS2 identifications were annotated via GNPS library matching (default parameters) in all samples with repeated identifications per file removed.
+ This options provides the ability to see how many times a chemical annotation was observed, the files in which in which it was observed, and a link to the sample information associated with the chemical annotation.
@@ -117,7 +120,7 @@