Merge release branch into feature branch

hms-dbmi · Oct 13, 2023 · 3fc4b64 · 3fc4b64
2 parents ebced5a + 0218493
commit 3fc4b64
Show file tree

Hide file tree

Showing 35 changed files with 935 additions and 553 deletions.
diff --git a/.github/workflows/github-actions-deploy-snapshots.yml b/.github/workflows/github-actions-deploy-snapshots.yml
@@ -18,4 +18,4 @@ jobs:
       - name: Build with Maven
         run: mvn --update-snapshots deploy
         env:
-          GITHUB_TOKEN: ${{ github.token }}
+          GITHUB_TOKEN: ${{ github.token }}
diff --git a/.github/workflows/label-checker.yml b/.github/workflows/label-checker.yml
@@ -0,0 +1,21 @@
+name: Label Checker
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+      - unlabeled
+
+jobs:
+
+  check_labels:
+    name: Check labels
+    runs-on: ubuntu-latest
+    steps:
+      - uses: docker://agilepathway/pull-request-label-checker:latest
+        with:
+          one_of: breaking-change,enhancement,bug,documentation,ignore-for-release
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+
diff --git a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java
@@ -37,6 +37,7 @@ public Query(Query query) {
 	private List<String> fields = new ArrayList<>();
 	private List<String> requiredFields = new ArrayList<>();
 	private List<String> anyRecordOf = new ArrayList<>();
+	private List<List<String>> anyRecordOfMulti = new ArrayList<>();
 	private Map<String, DoubleFilter> numericFilters = new HashMap<>();
 	private Map<String, String[]> categoryFilters = new HashMap<>();
 	private List<VariantInfoFilter> variantInfoFilters = new ArrayList<>();
@@ -62,6 +63,14 @@ public List<String> getRequiredFields() {
 	public List<String> getAnyRecordOf() {
 		return anyRecordOf;
 	}
+	public List<List<String>> getAnyRecordOfMulti() {
+		return anyRecordOfMulti;
+	}
+	public List<List<String>> getAllAnyRecordOf() {
+		List<List<String>> anyRecordOfMultiCopy = new ArrayList<>(anyRecordOfMulti);
+		anyRecordOfMultiCopy.add(anyRecordOf);
+		return anyRecordOfMultiCopy;
+	}
 
 	public Map<String, DoubleFilter> getNumericFilters() {
 		return numericFilters;
@@ -98,6 +107,9 @@ public void setRequiredFields(Collection<String> requiredFields) {
 	public void setAnyRecordOf(Collection<String> anyRecordOf) {
 		this.anyRecordOf = anyRecordOf != null ? new ArrayList<>(anyRecordOf) : new ArrayList<>();
 	}
+	public void setAnyRecordOfMulti(Collection<List<String>> anyRecordOfMulti) {
+		this.anyRecordOfMulti = anyRecordOfMulti != null ? new ArrayList<>(anyRecordOfMulti) : new ArrayList<>();
+	}
 
 	public void setNumericFilters(Map<String, DoubleFilter> numericFilters) {
 		this.numericFilters = numericFilters != null ? new HashMap<>(numericFilters) : new HashMap<>();
@@ -191,7 +203,7 @@ public String toString() {
 		writePartFormat("Numeric filters", numericFilters, builder);
 		writePartFormat("Category filters", categoryFilters, builder);
 		writePartFormat("Variant Info filters", variantInfoFilters, builder, false);
-		writePartFormat("Any-Record-Of filters", anyRecordOf, builder, true);
+		writePartFormat("Any-Record-Of filters", getAllAnyRecordOf(), builder, true);
 
 		return builder.toString();
 	}
@@ -234,7 +246,7 @@ private static void showTopLevelValues(Collection varList, StringBuilder builder
 
 				Integer count = countMap.get(firstLevel);
 				if(count == null) {
-					count = new Integer(1);
+					count = 1;
 				} else {
 					count = count + 1;
 				}

diff --git a/common/pom.xml b/common/pom.xml
@@ -21,5 +21,9 @@
 			<groupId>com.google.guava</groupId>
 			<artifactId>guava</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>org.codehaus.jackson</groupId>
+			<artifactId>jackson-jaxrs</artifactId>
+		</dependency>
 	</dependencies>
 </project>
diff --git a/...rc/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedByteIndexedStorage.java b/...rc/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedByteIndexedStorage.java
@@ -1,44 +1,52 @@
 package edu.harvard.hms.dbmi.avillach.hpds.storage;
 
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.RandomAccessFile;
-import java.io.Serializable;
+import java.io.*;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
 
-import org.apache.commons.io.output.ByteArrayOutputStream;
-
-public class FileBackedByteIndexedStorage <K, V extends Serializable> implements Serializable {
+public abstract class FileBackedByteIndexedStorage <K, V extends Serializable> implements Serializable {
 	private static final long serialVersionUID = -7297090745384302635L;
-	private transient RandomAccessFile storage;
-	private ConcurrentHashMap<K, Long[]> index;
-	private File storageFile;
-	private boolean completed = false;
-	private Long maxStorageSize;  //leave this in to not break serialization
+	protected transient RandomAccessFile storage;
+	protected ConcurrentHashMap<K, Long[]> index;
+	protected File storageFile;
+	protected boolean completed = false;
+
 
 	public FileBackedByteIndexedStorage(Class<K> keyClass, Class<V> valueClass, File storageFile) throws FileNotFoundException {
 		this.index = new ConcurrentHashMap<K, Long[]>();
 		this.storageFile = storageFile;
 		this.storage = new RandomAccessFile(this.storageFile, "rw");
 	}
 
+	public void updateStorageDirectory(File storageDirectory) {
+		if (!storageDirectory.isDirectory()) {
+			throw new IllegalArgumentException("storageDirectory is not a directory");
+		}
+		String currentStoreageFilename = storageFile.getName();
+		storageFile = new File(storageDirectory, currentStoreageFilename);
+	}
+
 	public Set<K> keys(){
 		return index.keySet();
 	}
 
-	public void put(K key, V value) throws IOException {
+	public void put(K key, V value) {
 		if(completed) {
 			throw new RuntimeException("A completed FileBackedByteIndexedStorage cannot be modified.");
 		}
-		Long[] recordIndex = store(value);
+		Long[] recordIndex;
+		try (ByteArrayOutputStream out = writeObject(value)) {
+			recordIndex = new Long[2];
+			synchronized (storage) {
+				storage.seek(storage.length());
+				recordIndex[0] = storage.getFilePointer();
+				storage.write(out.toByteArray());
+				recordIndex[1] = storage.getFilePointer() - recordIndex[0];
+			}
+		} catch (IOException e) {
+			throw new UncheckedIOException(e);
+		}
 		index.put(key, recordIndex);
 	}
 
@@ -63,60 +71,44 @@ public void complete() {
 		this.completed = true;
 	}
 
-	public boolean isComplete() {
-		return this.completed;
-	}
-
-	private Long[] store(V value) throws IOException {
-
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		ObjectOutputStream oos = new ObjectOutputStream(new GZIPOutputStream(out));
-		oos.writeObject(value);
-		oos.flush();
-		oos.close();
-
-		Long[] recordIndex = new Long[2];
-		synchronized(storage) {
-			storage.seek(storage.length());
-			recordIndex[0] = storage.getFilePointer();
-			storage.write(out.toByteArray());	
-			recordIndex[1] = storage.getFilePointer() - recordIndex[0];
-//			maxStorageSize = storage.getFilePointer();
-		}
-		return recordIndex;
-	}
-
-	public V get(K key) throws IOException {
-		if(this.storage==null) {
+	public V get(K key) {
+		try {
+			// todo: make this class immutable and remove this lock/check altogether
 			synchronized(this) {
-				this.open();
-			}
-		}
-		Long[] offsetsInStorage = index.get(key);
-		if(offsetsInStorage != null) {
-			Long offsetInStorage = index.get(key)[0];
-			int offsetLength = index.get(key)[1].intValue();
-			if(offsetInStorage != null && offsetLength>0) {
-				byte[] buffer = new byte[offsetLength];
-				synchronized(storage) {
-					storage.seek(offsetInStorage);
-					storage.readFully(buffer);					
+				if(this.storage==null) {
+					this.open();
 				}
-				ObjectInputStream in = new ObjectInputStream(new GZIPInputStream(new ByteArrayInputStream(buffer)));
-
-				try {
-					V readObject = (V) in.readObject();
+			}
+			Long[] offsetsInStorage = index.get(key);
+			if(offsetsInStorage != null) {
+				Long offsetInStorage = index.get(key)[0];
+				int offsetLength = index.get(key)[1].intValue();
+				if(offsetInStorage != null && offsetLength>0) {
+					byte[] buffer = new byte[offsetLength];
+					synchronized(storage) {
+						storage.seek(offsetInStorage);
+						storage.readFully(buffer);
+					}
+					V readObject = readObject(buffer);
 					return readObject;
-				} catch (ClassNotFoundException e) {
-					throw new RuntimeException("This should never happen.");
-				} finally {
-					in.close();
+				}else {
+					return null;
 				}
-			}else {
+			} else {
 				return null;
 			}
-		} else {
-			return null;
+		} catch (IOException e) {
+			throw new UncheckedIOException(e);
 		}
 	}
+
+	protected abstract V readObject(byte[] buffer);
+
+	protected abstract ByteArrayOutputStream writeObject(V value) throws IOException;
+
+	public V getOrELse(K key, V defaultValue) {
+		V result = get(key);
+		return result == null ? defaultValue : result;
+	}
+
 }
diff --git a/...rc/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJavaIndexedStorage.java b/...rc/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJavaIndexedStorage.java
@@ -0,0 +1,32 @@
+package edu.harvard.hms.dbmi.avillach.hpds.storage;
+
+import java.io.*;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+public class FileBackedJavaIndexedStorage <K, V extends Serializable> extends FileBackedByteIndexedStorage<K, V> {
+    public FileBackedJavaIndexedStorage(Class<K> keyClass, Class<V> valueClass, File storageFile) throws FileNotFoundException {
+        super(keyClass, valueClass, storageFile);
+    }
+
+    protected ByteArrayOutputStream writeObject(V value) throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        ObjectOutputStream oos = new ObjectOutputStream(new GZIPOutputStream(out));
+        oos.writeObject(value);
+        oos.flush();
+        oos.close();
+        return out;
+    }
+
+    @Override
+    protected V readObject(byte[] buffer) {
+        try (ObjectInputStream in = new ObjectInputStream(new GZIPInputStream(new ByteArrayInputStream(buffer)));) {
+            V readObject = (V) in.readObject();
+            return readObject;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        } catch (ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/.../src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJsonIndexStorage.java b/.../src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJsonIndexStorage.java
@@ -0,0 +1,40 @@
+package edu.harvard.hms.dbmi.avillach.hpds.storage;
+
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.type.TypeReference;
+
+import java.io.*;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+public abstract class FileBackedJsonIndexStorage <K, V extends Serializable> extends FileBackedByteIndexedStorage<K, V> {
+    private static final long serialVersionUID = -1086729119489479152L;
+
+    protected transient ObjectMapper objectMapper = new ObjectMapper();
+
+    public FileBackedJsonIndexStorage(File storageFile) throws FileNotFoundException {
+        super(null, null, storageFile);
+    }
+
+    protected ByteArrayOutputStream writeObject(V value) throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        objectMapper.writeValue(new GZIPOutputStream(out), value);
+        return out;
+    }
+
+    protected V readObject(byte[] buffer) {
+        try {
+            return objectMapper.readValue(new GZIPInputStream(new ByteArrayInputStream(buffer)), getTypeReference());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    // Required to populate the objectMapper on deserialization
+    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+        in.defaultReadObject();
+        objectMapper = new ObjectMapper();
+    }
+
+    public abstract TypeReference<V> getTypeReference();
+}