Added parentJob to create schema call

molgenis · Oct 10, 2024 · fde8ce8 · fde8ce8
1 parent 108c9f7
commit fde8ce8
Show file tree

Hide file tree

Showing 8 changed files with 61 additions and 20 deletions.
diff --git a/...nis-emx2-graphql/src/main/java/org/molgenis/emx2/graphql/GraphqlDatabaseFieldFactory.java b/...nis-emx2-graphql/src/main/java/org/molgenis/emx2/graphql/GraphqlDatabaseFieldFactory.java
@@ -62,11 +62,14 @@ public GraphQLFieldDefinition.Builder createMutation(Database database, TaskServ
             GraphQLArgument.newArgument()
                 .name(Constants.INCLUDE_DEMO_DATA)
                 .type(Scalars.GraphQLBoolean))
+        .argument(
+            GraphQLArgument.newArgument().name(Constants.PARENT_JOB).type(Scalars.GraphQLString))
         .dataFetcher(
             dataFetchingEnvironment -> {
               String name = dataFetchingEnvironment.getArgument(NAME);
               String description = dataFetchingEnvironment.getArgument(DESCRIPTION);
               String template = dataFetchingEnvironment.getArgument(Constants.TEMPLATE);
+              String parentTaskId = dataFetchingEnvironment.getArgument(Constants.PARENT_JOB);
               Boolean includeDemoData =
                   dataFetchingEnvironment.getArgument(Constants.INCLUDE_DEMO_DATA);
 
@@ -76,6 +79,8 @@ public GraphQLFieldDefinition.Builder createMutation(Database database, TaskServ
               Schema schema = database.createSchema(name, description);
               if (template != null) {
                 Task task = DataModels.getImportTask(schema, template, includeDemoData);
+                Task parentTask = taskService.getTask(parentTaskId);
+                task.setParentTask(parentTask);
                 String id = taskService.submit(task);
                 result.setTaskId(id);
               } else {

diff --git a/backend/molgenis-emx2-tasks/src/main/java/org/molgenis/emx2/tasks/ScriptTask.java b/backend/molgenis-emx2-tasks/src/main/java/org/molgenis/emx2/tasks/ScriptTask.java
@@ -8,6 +8,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
@@ -74,6 +75,8 @@ public void run() {
 
         // paste the script to a file into temp dir
         Path tempScriptFile = Files.createFile(tempDir.resolve("script.py"));
+        script =
+            script.replace("${jobId}", this.getId()); // todo: move to somewhere more appropriate
         Files.writeString(tempScriptFile, this.script);
         Path requirementsFile = Files.createFile(tempDir.resolve("requirements.txt"));
         Files.writeString(requirementsFile, this.dependencies != null ? this.dependencies : "");
@@ -113,6 +116,13 @@ public void run() {
             .put(
                 "OUTPUT_FILE",
                 tempOutputFile.toAbsolutePath().toString()); // in case of an output file
+
+        // todo: remove me, temp local pyclient for testing
+        String projectPath = System.getProperty("user.dir");
+        String pythonPath = Paths.get(projectPath, "tools", "pyclient", "src").toString();
+
+        builder.environment().put("PYTHONPATH", pythonPath);
+
         process = builder.start();
         this.addSubTask("Script started: " + process.info().commandLine().orElse("")).complete();
 

diff --git a/backend/molgenis-emx2-tasks/src/main/java/org/molgenis/emx2/tasks/Task.java b/backend/molgenis-emx2-tasks/src/main/java/org/molgenis/emx2/tasks/Task.java
@@ -42,6 +42,7 @@ public class Task implements Runnable, Iterable<Task> {
   private long startTimeMilliseconds;
   // end time to calculate run time
   private long endTimeMilliseconds;
+  private boolean includeDemoData;
   // subtasks/steps in this task
   private List<Task> subTasks = new ArrayList<>();
   // parent task
@@ -87,10 +88,14 @@ public void addSubTask(Task task) {
     this.subTasks.add(task);
   }
 
-  private void setParentTask(Task parentTask) {
+  public void setParentTask(Task parentTask) {
     this.parentTask = parentTask;
   }
 
+  public Task getParentTask() {
+    return this.parentTask;
+  }
+
   public List<Task> getSubTasks() {
     return Collections.unmodifiableList(subTasks);
   }
@@ -350,7 +355,10 @@ public void handleChange() {
     // currently we only log at setStatus changes to not overload database
     if (this.parentTask != null) {
       this.parentTask.handleChange();
-    } else if (this.changedHandler != null) {
+    }
+    if (this.changedHandler
+        != null) { // todo: do we want this? changed it because task run from scripts can have a
+      // parent and need to be updated in the db
       this.changedHandler.handleChange(this);
     }
   }
@@ -373,4 +381,12 @@ public long getEndTimeMilliseconds() {
   public boolean isRunning() {
     return !status.equals(ERROR) && !status.equals(COMPLETED);
   }
+
+  public boolean isIncludeDemoData() {
+    return includeDemoData;
+  }
+
+  public void setIncludeDemoData(boolean includeDemoData) {
+    this.includeDemoData = includeDemoData;
+  }
 }
diff --git a/backend/molgenis-emx2-tasks/src/main/java/org/molgenis/emx2/tasks/TaskServiceInMemory.java b/backend/molgenis-emx2-tasks/src/main/java/org/molgenis/emx2/tasks/TaskServiceInMemory.java
@@ -4,22 +4,18 @@
 
 import java.net.URL;
 import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 import org.molgenis.emx2.MolgenisException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class TaskServiceInMemory implements TaskService {
   Logger logger = LoggerFactory.getLogger(TaskServiceInMemory.class.getSimpleName());
-  private ExecutorService executorService;
-  private Map<String, Task> tasks = new LinkedHashMap<>();
+  private final ExecutorService executorService;
+  private final Map<String, Task> tasks = new LinkedHashMap<>();
 
   public TaskServiceInMemory() {
-    executorService =
-        new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
+    executorService = Executors.newSingleThreadExecutor();
   }
 
   @Override
@@ -29,8 +25,18 @@ public List<ScriptTask> getScripts() {
 
   @Override
   public String submit(Task task) {
-    tasks.put(task.getId(), task);
-    executorService.submit(task);
+    if (task.getParentTask() != null && tasks.containsKey(task.getParentTask().getId())) {
+      Task parentTask = tasks.get(task.getParentTask().getId());
+      parentTask.addSubTask(task);
+      if (parentTask.getStatus() == RUNNING) {
+        task.run();
+      } else {
+        executorService.submit(task);
+      }
+    } else {
+      tasks.put(task.getId(), task);
+      executorService.submit(task);
+    }
     return task.getId();
   }
 
@@ -71,10 +77,7 @@ public void removeOlderThan(long milliseconds) {
             toBeDeleted.add(key);
           }
         });
-    toBeDeleted.forEach(
-        key -> {
-          tasks.remove(key);
-        });
+    toBeDeleted.forEach(tasks::remove);
   }
 
   @Override

diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/StaticFileMapper.java
@@ -88,6 +88,7 @@ public static void addFileToContext(Context ctx, String path, String mimeType) {
         return;
       }
       if (mimeType == null) {
+        ctx.req().getServletContext().getMimeType(path);
         mimeType = URLConnection.guessContentTypeFromName(path);
         if (mimeType == null) {
           mimeType = "application/octet-stream";

diff --git a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Constants.java b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Constants.java
@@ -25,6 +25,7 @@ public class Constants {
   public static final String IS_CHANGELOG_ENABLED = "isChangelogEnabled";
   public static final String TEMPLATE = "template";
   public static final String INCLUDE_DEMO_DATA = "includeDemoData";
+  public static final String PARENT_JOB = "parentJob";
   public static final String SEMANTICS = "semantics";
   public static final String ROLE = "role";
   public static final String KEY = "key";

diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/client.py b/tools/pyclient/src/molgenis_emx2_pyclient/client.py
@@ -506,7 +506,8 @@ async def export(self, schema: str = None, table: str = None,
     async def create_schema(self, name: str = None,
                       description: str = None,
                       template: str = None,
-                      include_demo_data: bool = False):
+                      include_demo_data: bool = False,
+                      parent_job: str = None):
         """Creates a new schema on the EMX2 server.
 
         :param name: the name of the new schema
@@ -526,7 +527,7 @@ async def create_schema(self, name: str = None,
             raise PyclientException(f"Schema with name {name!r} already exists.")
         query = queries.create_schema()
         variables = self._format_optional_params(name=name, description=description,
-                                                 template=template, include_demo_data=include_demo_data)
+                                                 template=template, include_demo_data=include_demo_data, parent_job=parent_job)
 
         response = self.session.post(
             url=self.api_graphql,
@@ -996,6 +997,8 @@ def _format_optional_params(**kwargs):
             args['name'] = args.pop('name')
         if 'include_demo_data' in args.keys():
             args['includeDemoData'] = args.pop('include_demo_data')
+        if 'parent_job' in args.keys():
+            args['parentJob'] = args.pop('parent_job')
         return args
 
     def _table_in_schema(self, table_name: str, schema_name: str) -> bool:

diff --git a/tools/pyclient/src/molgenis_emx2_pyclient/graphql_queries.py b/tools/pyclient/src/molgenis_emx2_pyclient/graphql_queries.py
@@ -64,13 +64,15 @@ def create_schema():
             $name: String,
             $description: String,
             $template: String,
-            $includeDemoData: Boolean
+            $includeDemoData: Boolean,
+            $parentJob: String
         ) {
             createSchema(
                 name: $name,
                 description: $description,
                 template: $template,
-                includeDemoData: $includeDemoData
+                includeDemoData: $includeDemoData,
+                parentJob: $parentJob
               ) {
                   status
                   message