Merge master into ALS-4461

.github/release.yml

@@ -0,0 +1,18 @@
+changelog:
+  exclude:
+    labels:
+      - ignore-for-release
+  categories:
+    - title: Breaking Changes
+      labels:
+        - breaking-change
+    - title: Features
+      labels:
+        - feature 
+    - title: Enhancements 
+      labels:
+        - enhancement
+    - title: Bug Fixes
+      labels:
+        - bug
+

client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java

@@ -4,9 +4,13 @@
 
 import edu.harvard.hms.dbmi.avillach.hpds.data.query.Filter.DoubleFilter;
 import edu.harvard.hms.dbmi.avillach.hpds.data.query.Filter.FloatFilter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class Query {
 
+	private static final Logger log = LoggerFactory.getLogger(Query.class);
+
 	public Query() {
 
 	}
@@ -180,7 +184,7 @@ public String toString() {
 			break;
 		default:
 			//no logic here; all enum values should be present above
-			System.out.println("Formatting not supported for type " + expectedResultType);
+			log.warn("Formatting not supported for type {}", expectedResultType);
 		}
 
 		writePartFormat("Required Fields", requiredFields, builder, false);

data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/BucketIndexBySample.java

@@ -101,7 +101,7 @@ public BucketIndexBySample(VariantStore variantStore, String storageDir) throws
 									}
 								});
 							} catch (IOException e) {
-								e.printStackTrace();
+								throw new UncheckedIOException(e);
 							}
 
 							// For each patient set the patientBucketCharMask entry to 0 or 1 if they have a variant in the bucket.
@@ -124,18 +124,15 @@ public BucketIndexBySample(VariantStore variantStore, String storageDir) throws
 		// populate patientBucketMasks with bucketMasks for each patient 
 		patientBucketMasks = new FileBackedJavaIndexedStorage(Integer.class, BigInteger.class, new File(storageFileStr));
 
-		//the process to write out the bucket masks takes a very long time.  
-		//Lets spin up another thread that occasionally logs progress
 		int[] processedPatients = new int[1];
-
 		patientIds.parallelStream().forEach((patientId)->{
 			try {
 				BigInteger patientMask = new BigInteger(new String(patientBucketCharMasks[patientIds.indexOf(patientId)]),2);
 				patientBucketMasks.put(patientId, patientMask);
 			}catch(NumberFormatException e) {
 				log.error("NFE caught for " + patientId, e);
 			} catch (IOException e) {
-				e.printStackTrace();
+				throw new UncheckedIOException(e);
 			}
 			processedPatients[0] += 1;
 			int processedPatientsCount = processedPatients[0];
@@ -168,9 +165,9 @@ public Collection<String> filterVariantSetForPatientSet(Set<String> variantSet,
 			try {
 				return patientBucketMasks.get(patientNum);
 			} catch (IOException e) {
-				e.printStackTrace();
+				throw new UncheckedIOException(e);
 			}
-			return _defaultMask;
+			//return _defaultMask;
 		}).collect(Collectors.toList());
 		for(BigInteger patientMask : patientBucketmasksForSet) {
 			patientBucketMask = patientMask.or(patientBucketMask);

data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/CompressedIndex.java

@@ -1,10 +1,6 @@
 package edu.harvard.hms.dbmi.avillach.hpds.data.genotype;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
+import java.io.*;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -22,13 +18,16 @@
 import com.google.common.collect.Range;
 
 import edu.harvard.hms.dbmi.avillach.hpds.storage.FileBackedByteIndexedStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class CompressedIndex implements Serializable {
 
 	/**
 	 * 
 	 */
 	private static final long serialVersionUID = 5089713203903957829L;
+	private static final Logger log = LoggerFactory.getLogger(CompressedIndex.class);
 	public Float min = Float.MAX_VALUE, max = Float.MIN_VALUE;
 	private HashMap<Range<Float>, byte[]> compressedRangeMap;
 	private int valueCount;
@@ -61,7 +60,7 @@ public TreeMap<Float, TreeSet<String>> buildContinuousValuesMap(FileBackedByteIn
 							continuousValueMap.put(DoubleValue, currentValues);
 							setMinAndMax(DoubleValue);
 						}catch(NumberFormatException e3) {
-							System.out.println("Unable to parse value : " + value.trim());
+							log.info("Unable to parse value : " + value.trim());
 						}
 					}
 				}
@@ -114,8 +113,7 @@ public void buildIndex(TreeMap<Float, TreeSet<String>> continuousValueMap) {
 								gzos.close();
 								compressed = baos.toByteArray();
 							} catch (IOException e) {
-								// TODO Auto-generated catch block
-								e.printStackTrace();
+								throw new UncheckedIOException(e);
 							}
 							return compressed;
 						})
@@ -174,11 +172,9 @@ private TreeMap<Double, TreeSet<String>> retrieveRangeMap(Range<Double> range) {
 				){
 			continousValueMap = (TreeMap<Double, TreeSet<String>>)ois.readObject();
 		} catch (IOException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
+			throw new UncheckedIOException(e);
 		} catch (ClassNotFoundException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
+			throw new RuntimeException(e);
 		}
 		return continousValueMap;
 	}

data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantStore.java

@@ -61,25 +61,6 @@ public void writeInstance(String genomicDirectory) {
 		}
 	}
 
-	public ArrayList<String> listVariants() {
-		ArrayList<String> allVariants = new ArrayList<>();
-		for (String key : variantMaskStorage.keySet()) {
-			FileBackedJsonIndexStorage<Integer, ConcurrentHashMap<String, VariantMasks>> storage = variantMaskStorage
-					.get(key);
-			storage.keys().stream().forEach((Integer bucket) -> {
-				try {
-					for (String variant : storage.get(bucket).keySet()) {
-						allVariants.add(variant);
-					}
-				} catch (IOException e) {
-					// TODO Auto-generated catch block
-					e.printStackTrace();
-				}
-			});
-		}
-		return allVariants;
-	}
-
 	public Map<String, int[]> countVariants() {
 		HashMap<String, Integer> countOffsetMap = new HashMap<String, Integer>();
 		TreeMap<String, int[]> counts = new TreeMap<>();
@@ -108,8 +89,7 @@ public Map<String, int[]> countVariants() {
 								|| masks.heterozygousNoCallMask != null || masks.homozygousNoCallMask != null ? 1 : 0;
 					}));
 				} catch (IOException e) {
-					// TODO Auto-generated catch block
-					e.printStackTrace();
+					throw new UncheckedIOException(e);
 				}
 			});
 		}
@@ -159,8 +139,7 @@ public void open() {
 				try {
 					fbbis.open();
 				} catch (FileNotFoundException e) {
-					// TODO Auto-generated catch block
-					e.printStackTrace();
+					throw new UncheckedIOException(e);
 				}
 			}
 		}));

etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/HideAnnotationCategoryValue.java

@@ -1,7 +1,6 @@
 package edu.harvard.hms.dbmi.avillach.hpds.data.genotype.util;
 
 import java.io.FileInputStream;
-import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.ObjectInputStream;
@@ -20,40 +19,27 @@
 public class HideAnnotationCategoryValue {
 	protected static LoadingCache<String, PhenoCube<?>> store;
 
-	protected static TreeMap<String, ColumnMeta> metaStoreSource;
-
 	protected static TreeSet<Integer> allIds;
 
-	public static void main(String[] args) throws ClassNotFoundException, FileNotFoundException, IOException {
+	public static void main(String[] args) throws ClassNotFoundException, IOException {
 		String infoStoreToModify = args[0];
 		String valueToScrub = args[1];
 
 		String infoStoreFilename = "/opt/local/hpds/all/" + infoStoreToModify.trim();
-		try (
-				FileInputStream fis = new FileInputStream(infoStoreFilename);
-				GZIPInputStream gis = new GZIPInputStream(fis);
-				ObjectInputStream ois = new ObjectInputStream(gis)
-				){
-			FileBackedByteIndexedInfoStore infoStore = (FileBackedByteIndexedInfoStore) ois.readObject();
-			infoStore.getAllValues().keys().remove(valueToScrub);
-			try(
-					FileOutputStream fos = new FileOutputStream(infoStoreFilename);
-					GZIPOutputStream gos = new GZIPOutputStream(fos);
-					ObjectOutputStream oos = new ObjectOutputStream(gos);
-					){
-				oos.writeObject(infoStore);
-				oos.flush();oos.close();
-			}
-		} catch (FileNotFoundException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		} catch (IOException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		} catch (ClassNotFoundException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
+
+		FileInputStream fis = new FileInputStream(infoStoreFilename);
+		GZIPInputStream gis = new GZIPInputStream(fis);
+		ObjectInputStream ois = new ObjectInputStream(gis);
+
+		FileBackedByteIndexedInfoStore infoStore = (FileBackedByteIndexedInfoStore) ois.readObject();
+		infoStore.getAllValues().keys().remove(valueToScrub);
+
+		FileOutputStream fos = new FileOutputStream(infoStoreFilename);
+		GZIPOutputStream gos = new GZIPOutputStream(fos);
+		ObjectOutputStream oos = new ObjectOutputStream(gos);
+
+		oos.writeObject(infoStore);
+		oos.flush();oos.close();
 	}
 
 }

etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/DumpSourceCSV.java

@@ -1,12 +1,6 @@
 package edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.util;
 
-import java.io.ByteArrayInputStream;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.RandomAccessFile;
+import java.io.*;
 import java.util.ArrayList;
 import java.util.Set;
 import java.util.TreeMap;
@@ -68,10 +62,10 @@ public static void main(String[] args) throws ClassNotFoundException, FileNotFou
 					cubeLines.add(line);
 				}
 				writer.printRecords(cubeLines);
-			}catch(ExecutionException e) {
-				e.printStackTrace();
+			} catch(ExecutionException e) {
+				throw new RuntimeException(e);
 			} catch (IOException e) {
-				e.printStackTrace();
+				throw new UncheckedIOException(e);
 			}
 		});
 		writer.flush();
@@ -90,7 +84,6 @@ protected static Object[] loadMetadata() {
 			Set<Integer> allIds = (TreeSet<Integer>) objectInputStream.readObject();
 			return new Object[] {metastoreScrubbed, allIds};
 		} catch (IOException | ClassNotFoundException e) {
-			e.printStackTrace();
 			throw new RuntimeException("Could not load metastore");
 		} 
 	}

etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/FixCategoricalConcepts.java

@@ -125,7 +125,7 @@ private static void processRecord(final PhenoCube[] currentConcept, List<String>
 				store.allIds.add(Integer.parseInt(record.get(PATIENT_NUM)));
 			}
 		} catch (ExecutionException e) {
-			e.printStackTrace();
+			log.error("Error processing record", e);
 		}
 	}
 
@@ -174,10 +174,9 @@ protected static Object[] loadMetadata() {
 			Set<Integer> allIds = (TreeSet<Integer>) objectInputStream.readObject();
 			return new Object[] {metastoreScrubbed, allIds};
 		} catch (IOException | ClassNotFoundException e) {
-			e.printStackTrace();
 			log.warn("************************************************");
 			log.warn("************************************************");
-			log.warn("Could not load metastore");
+			log.warn("Could not load metastore", e);
 			log.warn("If you meant to include phenotype data of any kind, please check that the file /opt/local/source/columnMeta.javabin exists and is readable by the service.");
 			log.warn("************************************************");
 			log.warn("************************************************");

etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RekeyDataset.java

@@ -100,10 +100,9 @@ protected static Object[] loadMetadata() {
 			Set<Integer> allIds = (TreeSet<Integer>) objectInputStream.readObject();
 			return new Object[] {metastoreScrubbed, allIds};
 		} catch (IOException | ClassNotFoundException e) {
-			e.printStackTrace();
 			log.warn("************************************************");
 			log.warn("************************************************");
-			log.warn("Could not load metastore");
+			log.warn("Could not load metastore", e);
 			log.warn("If you meant to include phenotype data of any kind, please check that the file /opt/local/source/columnMeta.javabin exists and is readable by the service.");
 			log.warn("************************************************");
 			log.warn("************************************************");

etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RemapIds.java

@@ -124,7 +124,7 @@ private static void processRecord(final PhenoCube[] currentConcept, List<String>
 				store.allIds.add(patientId);
 			}
 		} catch (ExecutionException e) {
-			e.printStackTrace();
+			log.error("Error processing record", e);
 		}
 	}
 
@@ -177,10 +177,9 @@ protected static Object[] loadMetadata() {
 			Set<Integer> allIds = (TreeSet<Integer>) objectInputStream.readObject();
 			return new Object[] {metastoreScrubbed, allIds};
 		} catch (IOException | ClassNotFoundException e) {
-			e.printStackTrace();
 			log.warn("************************************************");
 			log.warn("************************************************");
-			log.warn("Could not load metastore");
+			log.warn("Could not load metastore", e);
 			log.warn("If you meant to include phenotype data of any kind, please check that the file /opt/local/source/columnMeta.javabin exists and is readable by the service.");
 			log.warn("************************************************");
 			log.warn("************************************************");

etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RemoveConceptFromMetadata.java

@@ -78,7 +78,6 @@ protected static TreeMap<String, ColumnMeta> removeMetadata(Path filePath) {
 
 			return metastore;
 		} catch (IOException | ClassNotFoundException e) {
-			e.printStackTrace();
 			throw new RuntimeException("Could not load metastore");
 		} 
 	}

etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RenameCategories.java

@@ -49,8 +49,7 @@ public static void main(String[] args) throws ClassNotFoundException, FileNotFou
 			try {
 				loadingStoreTarget.store.put(key, store.get(key));
 			} catch (ExecutionException e) {
-				// TODO Auto-generated catch block
-				e.printStackTrace();
+				throw new RuntimeException(e);
 			}
 		});
 	}
@@ -67,7 +66,6 @@ protected static Object[] loadMetadata() {
 			Set<Integer> allIds = (TreeSet<Integer>) objectInputStream.readObject();
 			return new Object[] {metastoreScrubbed, allIds};
 		} catch (IOException | ClassNotFoundException e) {
-			e.printStackTrace();
 			throw new RuntimeException("Could not load metastore");
 		} 
 	}

etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/MultialleleCounter.java

@@ -1,9 +1,6 @@
 package edu.harvard.hms.dbmi.avillach.hpds.etl.genotype;
 
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.ObjectInputStream;
+import java.io.*;
 import java.util.ArrayList;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
@@ -43,9 +40,8 @@ public static void main(String[] args) throws ClassNotFoundException, IOExceptio
                         }
                     }
                 } catch (IOException e) {
-                    // TODO Auto-generated catch block
-                    e.printStackTrace();
-                }
+					throw new UncheckedIOException(e);
+				}
                 System.out.println("Completed bucket : " + offsetBucket);
             });
         }