Merge pull request #587 from saikiranAnnam/revert-586-fix/remove-jypu…

…ter-support

Revert "❌ removed jupyter notebook support"

.vscode/settings.json

@@ -1,5 +1,4 @@
 {
     "java.configuration.updateBuildConfiguration": "automatic",
-    "java.compile.nullAnalysis.mode": "disabled",
-    "java.debug.settings.onBuildFailureProceed": true
+    "java.compile.nullAnalysis.mode": "disabled"
 }

docs/process.md

@@ -50,7 +50,15 @@ Shell scripts can be directly created, saved, executed, and monitored in Geoweav
 
 Python is one of the most popular AI programming languages, and most AI-related packages reside in it. Geoweaver supports Python coding and scripting on top of multiple servers while preserving and maintaining the code in one database. All the historical runs are recorded and served in Geoweaver to prevent future duplicated attempts and significantly improve the reproducibility and reusability of AI programs.
 
-### 3) Build-In Process
+### 3) NoteBook
+
+Geoweaver supports running Jupyter notebooks using the nbconvert command. The notebook and its logout are recorded in the database.
+
+Running notebook in Geoweaver will be equivalent to running `jupyter nbconvert --inplace --allow-errors --to notebook --execute <notebook_file_path>`.
+
+**New notebook snapshots generated by every run will be saved in the Geoweaver database**. No more worries about forgetting the results of your previous experiment configuration! Everything is covered.
+
+### 4) Build-In Process
 
 To help people with limited programming skills, we are developing a set of built-in processes which have fixed program code and expose only input parameters to users. These processes make Geoweaver a powerful AI graphical user interface for diverse user groups to learn and experiment with their AI workflows without coding. Most built-in processes in Geoweaver are developed based on the existing AI python ecosystem like Keras and Scikit-learn. This section is under intensive development, and the first stable version expects users to create a full-stack AI workflow without writing a single line of code.
 

src/main/java/com/gw/database/HostRepository.java

@@ -76,6 +76,31 @@ public interface HostRepository extends CrudRepository<Host, String> {
    */
   @Query(value = "select * from host where confidential = 'FALSE'", nativeQuery = true)
   Collection<Host> findAllPublicHosts();
+
+  /**
+   * Find hosts of type 'jupyter'.
+   *
+   * @return A collection of hosts of type 'jupyter'.
+   */
+  @Query(value = "select * from host where type = 'jupyter'", nativeQuery = true)
+  Collection<Host> findJupyterNotebookHosts();
+
+  /**
+   * Find hosts of type 'jupyterhub'.
+   *
+   * @return A collection of hosts of type 'jupyterhub'.
+   */
+  @Query(value = "select * from host where type = 'jupyterhub'", nativeQuery = true)
+  Collection<Host> findJupyterHubHosts();
+
+  /**
+   * Find hosts of type 'jupyterlab'.
+   *
+   * @return A collection of hosts of type 'jupyterlab'.
+   */
+  @Query(value = "select * from host where type = 'jupyterlab'", nativeQuery = true)
+  Collection<Host> findJupyterLabHosts();
+
   /**
    * Find hosts of type 'gee'.
    *

src/main/java/com/gw/database/ProcessRepository.java

@@ -69,6 +69,12 @@ public interface ProcessRepository extends CrudRepository<GWProcess, String> {
    */
   @Query(value = "select * from gwprocess where lang = 'builtin'", nativeQuery = true)
   Collection<GWProcess> findBuiltinProcess();
-}
-
 
+  /**
+   * Find processes implemented as Jupyter notebooks.
+   *
+   * @return A collection of processes represented as Jupyter notebooks.
+   */
+  @Query(value = "select * from gwprocess where lang = 'jupyter'", nativeQuery = true)
+  Collection<GWProcess> findNotebookProcess();
+}

src/main/java/com/gw/local/LocalSession.java

@@ -67,6 +67,15 @@ public void runBash(
    * @param basedir
    * @param token http session id
    */
+  public void runJupyter(
+      String history_id,
+      String script,
+      String processid,
+      boolean isjoin,
+      String bin,
+      String env,
+      String basedir,
+      String token);
 
   /**
    * Run Python locally

src/main/java/com/gw/local/LocalSessionNixImpl.java

@@ -236,6 +236,117 @@ public void runBash(
     }
   }
 
+  @Override
+  public void runJupyter(
+      String history_id,
+      String notebookjson,
+      String processid,
+      boolean isjoin,
+      String bin,
+      String env,
+      String basedir,
+      String token) {
+
+    this.initHistory(history_id, notebookjson, processid, isjoin, token);
+
+    try {
+
+      log.info("starting command");
+
+      tempfile =
+          bt.normalizedPath(workspace_folder_path)
+              + "/"
+              + history_id
+              + "/gw-"
+              + token
+              + "-"
+              + history.getHistory_id()
+              + ".ipynb";
+
+      bt.writeString2File(notebookjson, tempfile);
+
+      // Get a list of all environment variables
+      Map<String, String> envMap = new HashMap<String, String>(System.getenv());
+
+      ProcessBuilder builder = new ProcessBuilder();
+
+      builder.directory(new File(bt.normalizedPath(workspace_folder_path) + "/" + history_id));
+
+      if (BaseTool.isNull(bin)) {
+
+        builder.command(
+            new String[] {
+              "jupyter",
+              "nbconvert",
+              "--inplace",
+              "--allow-errors",
+              "--to",
+              "notebook",
+              "--execute",
+              tempfile
+            });
+
+      } else {
+
+        builder.command(
+            new String[] {
+              bin,
+              "-m",
+              "jupyter",
+              "nbconvert",
+              "--inplace",
+              "--allow-errors",
+              "--to",
+              "notebook",
+              "--execute",
+              tempfile
+            });
+      }
+
+      builder.redirectErrorStream(true);
+
+      process = builder.start();
+
+      InputStream stdout = process.getInputStream();
+
+      log.info("Local session established");
+
+      input = new BufferedReader(new InputStreamReader(stdout), BaseTool.BUFFER_SIZE);
+
+      sender.init(input, token, history_id, "juyter", tempfile);
+
+      // moved here on 10/29/2018
+      // all SSH sessions must have a output thread
+
+      thread = new Thread(sender);
+
+      thread.setName("SSH Command output thread");
+
+      log.info("starting sending thread");
+
+      thread.start();
+
+      log.info("returning to the client..");
+
+      sender.setProcess(process);
+
+      if (isjoin) {
+
+        process.waitFor();
+      }
+
+    } catch (Exception e) {
+
+      e.printStackTrace();
+
+      this.endWithError(token, e.getLocalizedMessage());
+
+    } finally {
+
+      this.isClose = true;
+    }
+  }
+
   @Override
   public void runPython(
       String history_id,
@@ -322,6 +433,8 @@ public void saveHistory(String logs, String status) {
     history.setIndicator(status);
 
     history_tool.saveHistory(history);
+
+    pt.updateJupyter(history, this.token);
   }
 
   @Override

src/main/java/com/gw/local/LocalSessionOutput.java

@@ -58,10 +58,11 @@ public LocalSessionOutput() {
   /**
    * Initializes the LocalSessionOutput with necessary parameters for running.
    *
-   * @param in         BufferedReader for reading the session's output.
-   * @param token      The session token.
+   * @param in BufferedReader for reading the session's output.
+   * @param token The session token.
    * @param history_id The history ID associated with the session.
-   * @param lang       The programming language used in the session.
+   * @param lang The programming language used in the session.
+   * @param jupyterfilepath The Jupyter file path, if applicable.
    */
   public void init(
       BufferedReader in, String token, String history_id, String lang, String jupyterfilepath) {
@@ -71,7 +72,7 @@ public void init(
     this.run = true;
     this.history_id = history_id;
     this.lang = lang;
-    this.jupyterfilepath = jupyterfilepath; // this is passed null and being used in Windows Impl
+    this.jupyterfilepath = jupyterfilepath;
     refreshLogMonitor();
   }
 
@@ -152,6 +153,34 @@ public void endWithCode(String token, int exitvalue) {
         this.history_id + BaseTool.log_separator + "Exit Code: " + exitvalue);
   }
 
+  /**
+   * Updates the status and logs for a Jupyter execution.
+   *
+   * @param logs The logs generated during execution.
+   * @param status The execution status (e.g., "Done" or "Failed").
+   */
+  public void updateJupyterStatus(String logs, String status) {
+    History h = ht.getHistoryById(this.history_id);
+
+    if (BaseTool.isNull(h)) {
+      h = new History();
+      h.setHistory_id(history_id);
+      log.debug("This is very unlikely");
+    }
+
+    String resultjupyterjson = bt.readStringFromFile(this.jupyterfilepath);
+
+    h.setHistory_input(resultjupyterjson);
+    h.setHistory_output(logs);
+    h.setIndicator(status);
+
+    if ("Done".equals(status) || "Failed".equals(status)) {
+      h.setHistory_end_time(BaseTool.getCurrentSQLDate());
+    }
+
+    ht.saveHistory(h);
+  }
+
   /**
    * Updates the status and logs for an execution.
    *
@@ -245,7 +274,11 @@ public void run() {
                 log.debug("Null output lines exceed 10. Disconnected.");
 
                 // Depending on the language (e.g., "jupyter"), update the status
-                this.updateStatus(logs.toString(), "Done");
+                if ("jupyter".equals(this.lang)) {
+                  this.updateJupyterStatus(logs.toString(), "Done");
+                } else {
+                  this.updateStatus(logs.toString(), "Done");
+                }
                 break;
               } else {
                 startrecorder = -1;
@@ -272,17 +305,23 @@ public void run() {
         } catch (Exception e) {
           e.printStackTrace();
           // Depending on the language, update the status to "Failed"
-          this.updateStatus(logs.toString(), "Failed");
+          if ("jupyter".equals(this.lang)) {
+            this.updateJupyterStatus(logs.toString(), "Failed");
+          } else {
+            this.updateStatus(logs.toString(), "Failed");
+          }
           break;
         } finally {
           // session.saveHistory(logs.toString()); //write the failed record
         }
       }
 
       // Depending on the language (e.g., "jupyter"), update the status to "Done"
-
+      if ("jupyter".equals(this.lang)) {
+        this.updateJupyterStatus(logs.toString(), "Done");
+      } else {
         this.updateStatus(logs.toString(), "Done");
-
+      }
 
       // If the process is available, attempt to stop and get its exit code
       if (!BaseTool.isNull(theprocess)) {