Hack open function to write files in main thread (#1146)

## Things that need doing

- [x] Support `w` mode
- [x] Support `a` mode
- [x] Support `x` mode
- [x] Support creating files when the specified file name does not match
an existing file
- [x] Support `with open(filename) as f` pattern (currently returning
`CustomFile does not support the context manager protocol`)
- [x] Re-enable `pyodide-http` patch
- [x] Think about limiting the number of files the user can create to
avoid overloading the server
- [x] Ensure that file size limit applies to generated files

Author: loiswells97

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Added
 
+- Ability to write to files in `python` (#1146)
 - Support for the `outputPanels` attribute in the `PyodideRunner` (#1157)
 - Downloading project instructions (#1160)
 

cypress/e2e/spec-wc-pyodide.cy.js

@@ -91,6 +91,58 @@ describe("Running the code with pyodide", () => {
       .should("contain", "Hello Lois");
   });
 
+  it("runs a simple program to write to a file", () => {
+    runCode('with open("output.txt", "w") as f:\n\tf.write("Hello world")');
+    cy.get("editor-wc")
+      .shadow()
+      .contains(".files-list-item", "output.txt")
+      .click();
+    cy.get("editor-wc")
+      .shadow()
+      .find(".cm-editor")
+      .should("contain", "Hello world");
+  });
+
+  it("errors when trying to write to an existing file in 'x' mode", () => {
+    runCode('with open("output.txt", "w") as f:\n\tf.write("Hello world")');
+    cy.get("editor-wc")
+      .shadow()
+      .find(".files-list-item")
+      .should("contain", "output.txt");
+    runCode('with open("output.txt", "x") as f:\n\tf.write("Something else")');
+    cy.get("editor-wc")
+      .shadow()
+      .find(".error-message__content")
+      .should(
+        "contain",
+        "FileExistsError: File 'output.txt' already exists on line 1 of main.py",
+      );
+  });
+
+  it("updates the file in the editor when the content is updated programatically", () => {
+    runCode('with open("output.txt", "w") as f:\n\tf.write("Hello world")');
+    cy.get("editor-wc")
+      .shadow()
+      .find("div[class=cm-content]")
+      .invoke(
+        "text",
+        'with open("output.txt", "a") as f:\n\tf.write("Hello again world")',
+      );
+    cy.get("editor-wc")
+      .shadow()
+      .contains(".files-list-item", "output.txt")
+      .click();
+    cy.get("editor-wc")
+      .shadow()
+      .find(".btn--run")
+      .should("not.be.disabled")
+      .click();
+    cy.get("editor-wc")
+      .shadow()
+      .find(".cm-editor")
+      .should("contain", "Hello again world");
+  });
+
   it("runs a simple program with a built-in python module", () => {
     runCode("from math import floor, pi\nprint(floor(pi))");
     cy.get("editor-wc")

src/PyodideWorker.js

@@ -60,12 +60,6 @@ const PyodideWorker = () => {
 
   const runPython = async (python) => {
     stopped = false;
-    await pyodide.loadPackage("pyodide_http");
-
-    await pyodide.runPythonAsync(`
-    import pyodide_http
-    pyodide_http.patch_all()
-  `);
 
     try {
       await withSupportForPackages(python, async () => {
@@ -98,6 +92,52 @@ const PyodideWorker = () => {
     await pyodide.loadPackagesFromImports(python);
 
     checkIfStopped();
+    await pyodide.runPythonAsync(
+      `
+      import basthon
+      import builtins
+      import os
+
+      MAX_FILES = 100
+      MAX_FILE_SIZE = 8500000
+
+      def _custom_open(filename, mode="r", *args, **kwargs):
+          if "x" in mode and os.path.exists(filename):
+              raise FileExistsError(f"File '{filename}' already exists")
+          if ("w" in mode or "a" in mode or "x" in mode) and "b" not in mode:
+              if len(os.listdir()) > MAX_FILES and not os.path.exists(filename):
+                  raise OSError(f"File system limit reached, no more than {MAX_FILES} files allowed")
+              class CustomFile:
+                  def __init__(self, filename):
+                      self.filename = filename
+                      self.content = ""
+
+                  def write(self, content):
+                      self.content += content
+                      if len(self.content) > MAX_FILE_SIZE:
+                          raise OSError(f"File '{self.filename}' exceeds maximum file size of {MAX_FILE_SIZE} bytes")
+                      with _original_open(self.filename, "w") as f:
+                          f.write(self.content)
+                      basthon.kernel.write_file({ "filename": self.filename, "content": self.content, "mode": mode })
+
+                  def close(self):
+                      pass
+
+                  def __enter__(self):
+                      return self
+
+                  def __exit__(self, exc_type, exc_val, exc_tb):
+                      self.close()
+
+              return CustomFile(filename)
+          else:
+              return _original_open(filename, mode, *args, **kwargs)
+
+      # Override the built-in open function
+      builtins.open = _custom_open
+      `,
+      { filename: "__custom_open__.py" },
+    );
     await runPythonFn();
 
     for (let name of imports) {
@@ -337,6 +377,12 @@ const PyodideWorker = () => {
 
         postMessage({ method: "handleVisual", origin, content });
       },
+      write_file: (event) => {
+        const filename = event.toJs().get("filename");
+        const content = event.toJs().get("content");
+        const mode = event.toJs().get("mode");
+        postMessage({ method: "handleFileWrite", filename, content, mode });
+      },
       locals: () => pyodide.runPython("globals()"),
     },
   };
@@ -346,7 +392,7 @@ const PyodideWorker = () => {
     await pyodide.runPythonAsync(`
         # Clear all user-defined variables and modules
         for name in dir():
-            if not name.startswith('_'):
+            if not name.startswith('_') and not name=='basthon':
                 del globals()[name]
       `);
     postMessage({ method: "handleLoaded", stdinBuffer, interruptBuffer });
@@ -364,6 +410,8 @@ const PyodideWorker = () => {
 
     pyodide = await pyodidePromise;
 
+    pyodide.registerJsModule("basthon", fakeBasthonPackage);
+
     await pyodide.runPythonAsync(`
     __old_input__ = input
     def __patched_input__(prompt=False):
@@ -373,6 +421,18 @@ const PyodideWorker = () => {
     __builtins__.input = __patched_input__
     `);
 
+    await pyodide.runPythonAsync(`
+    import builtins
+    # Save the original open function
+    _original_open = builtins.open
+    `);
+
+    await pyodide.loadPackage("pyodide-http");
+    await pyodide.runPythonAsync(`
+        import pyodide_http
+        pyodide_http.patch_all()
+      `);
+
     if (supportsAllFeatures) {
       stdinBuffer =
         stdinBuffer || new Int32Array(new SharedArrayBuffer(1024 * 1024)); // 1 MiB
@@ -416,6 +476,14 @@ const PyodideWorker = () => {
 
     const lines = trace.split("\n");
 
+    // if the third from last line matches /File "__custom_open__\.py", line (\d+)/g then strip off the last three lines
+    if (
+      lines.length > 3 &&
+      /File "__custom_open__\.py", line (\d+)/g.test(lines[lines.length - 3])
+    ) {
+      lines.splice(-3, 3);
+    }
+
     const snippetLine = lines[lines.length - 2]; //    print("hi")invalid
     const caretLine = lines[lines.length - 1]; //                 ^^^^^^^
 
@@ -424,7 +492,9 @@ const PyodideWorker = () => {
       ? [snippetLine.slice(4), caretLine.slice(4)].join("\n")
       : "";
 
-    const matches = [...trace.matchAll(/File "(.*)", line (\d+)/g)];
+    const matches = [
+      ...trace.matchAll(/File "(?!__custom_open__\.py)(.*)", line (\d+)/g),
+    ];
     const match = matches[matches.length - 1];
 
     const path = match ? match[1] : "";

src/components/Editor/EditorPanel/EditorPanel.jsx

@@ -2,7 +2,10 @@
 import "../../../assets/stylesheets/EditorPanel.scss";
 import React, { useRef, useEffect, useContext, useState } from "react";
 import { useSelector, useDispatch } from "react-redux";
-import { updateProjectComponent } from "../../../redux/EditorSlice";
+import {
+  setCascadeUpdate,
+  updateProjectComponent,
+} from "../../../redux/EditorSlice";
 import { useCookies } from "react-cookie";
 import { useTranslation } from "react-i18next";
 import { basicSetup } from "codemirror";
@@ -27,8 +30,10 @@ const MAX_CHARACTERS = 8500000;
 
 const EditorPanel = ({ extension = "html", fileName = "index" }) => {
   const editor = useRef();
+  const editorViewRef = useRef();
   const project = useSelector((state) => state.editor.project);
   const readOnly = useSelector((state) => state.editor.readOnly);
+  const cascadeUpdate = useSelector((state) => state.editor.cascadeUpdate);
   const [cookies] = useCookies(["theme", "fontSize"]);
   const dispatch = useDispatch();
   const { t } = useTranslation();
@@ -40,7 +45,8 @@ const EditorPanel = ({ extension = "html", fileName = "index" }) => {
       updateProjectComponent({
         extension: extension,
         name: fileName,
-        code: content,
+        content,
+        cascadeUpdate: false,
       }),
     );
   };
@@ -74,11 +80,11 @@ const EditorPanel = ({ extension = "html", fileName = "index" }) => {
       window.matchMedia("(prefers-color-scheme:dark)").matches);
   const editorTheme = isDarkMode ? editorDarkTheme : editorLightTheme;
 
-  useEffect(() => {
-    const file = project.components.find(
-      (item) => item.extension === extension && item.name === fileName,
-    );
+  const file = project.components.find(
+    (item) => item.extension === extension && item.name === fileName,
+  );
 
+  useEffect(() => {
     if (!file) {
       return;
     }
@@ -123,6 +129,8 @@ const EditorPanel = ({ extension = "html", fileName = "index" }) => {
       parent: editor.current,
     });
 
+    editorViewRef.current = view;
+
     // 'aria-hidden' to fix keyboard access accessibility error
     view.scrollDOM.setAttribute("aria-hidden", "true");
 
@@ -138,6 +146,23 @@ const EditorPanel = ({ extension = "html", fileName = "index" }) => {
     };
   }, [cookies]);
 
+  useEffect(() => {
+    if (
+      cascadeUpdate &&
+      editorViewRef.current &&
+      file.content !== editorViewRef.current.state.doc.toString()
+    ) {
+      editorViewRef.current.dispatch({
+        changes: {
+          from: 0,
+          to: editorViewRef.current.state.doc.length,
+          insert: file.content,
+        },
+      });
+      dispatch(setCascadeUpdate(false));
+    }
+  }, [file, cascadeUpdate, editorViewRef]);
+
   return (
     <>
       <div className={`editor editor--${settings.fontSize}`} ref={editor}></div>

src/components/Editor/Output/Output.test.js

@@ -26,6 +26,8 @@ describe("Output component", () => {
         project: {
           components: [],
         },
+        focussedFileIndices: [0],
+        openFiles: [["main.py"]],
       },
       auth: {
         user,

src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx

@@ -8,6 +8,8 @@ import {
   setError,
   codeRunHandled,
   setLoadedRunner,
+  updateProjectComponent,
+  addProjectComponent,
 } from "../../../../../redux/EditorSlice";
 import { Tab, Tabs, TabList, TabPanel } from "react-tabs";
 import { useMediaQuery } from "react-responsive";
@@ -51,6 +53,10 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => {
   const projectImages = useSelector((s) => s.editor.project.image_list);
   const projectCode = useSelector((s) => s.editor.project.components);
   const projectIdentifier = useSelector((s) => s.editor.project.identifier);
+  const focussedFileIndex = useSelector(
+    (state) => state.editor.focussedFileIndices,
+  )[0];
+  const openFiles = useSelector((state) => state.editor.openFiles)[0];
   const user = useSelector((s) => s.auth.user);
   const userId = user?.profile?.user;
   const isSplitView = useSelector((s) => s.editor.isSplitView);
@@ -97,6 +103,16 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => {
               data.info,
             );
             break;
+          case "handleFileWrite":
+            const cascadeUpdate =
+              openFiles[focussedFileIndex] === data.filename;
+            handleFileWrite(
+              data.filename,
+              data.content,
+              data.mode,
+              cascadeUpdate,
+            );
+            break;
           case "handleVisual":
             handleVisual(data.origin, data.content);
             break;
@@ -108,7 +124,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => {
         }
       };
     }
-  }, [pyodideWorker]);
+  }, [pyodideWorker, projectCode, openFiles, focussedFileIndex]);
 
   useEffect(() => {
     if (codeRunTriggered && active && output.current) {
@@ -197,6 +213,35 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => {
     disableInput();
   };
 
+  const handleFileWrite = (filename, content, mode, cascadeUpdate) => {
+    const [name, extension] = filename.split(".");
+    const componentToUpdate = projectCode.find(
+      (item) => item.extension === extension && item.name === name,
+    );
+    let updatedContent;
+    if (mode === "w" || mode === "x") {
+      updatedContent = content;
+    } else if (mode === "a") {
+      updatedContent =
+        (componentToUpdate ? componentToUpdate.content + "\n" : "") + content;
+    }
+
+    if (componentToUpdate) {
+      dispatch(
+        updateProjectComponent({
+          extension,
+          name,
+          content: updatedContent,
+          cascadeUpdate,
+        }),
+      );
+    } else {
+      dispatch(
+        addProjectComponent({ name, extension, content: updatedContent }),
+      );
+    }
+  };
+
   const handleVisual = (origin, content) => {
     if (showVisualOutputPanel) {
       setHasVisual(true);