Improve REPL including arrow key/backspace support + unit tests

.github/workflows/rust.yml

@@ -14,9 +14,21 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        # Each feature flag combination
+        features:
+          - ""
+          - "c_stdlib"
+          - "repl"
+          - "c_stdlib repl"
+
     steps:
     - uses: actions/checkout@v3
     - name: Run tests
-      run: cargo test --verbose
-    - name: Run tests C stdlib enabled (no LLVM backend)
-      run: cargo test --features c_stdlib
+      run: |
+        if [ -z "${{ matrix.features }}" ]; then
+          cargo test --verbose
+        else
+          cargo test --verbose --features "${{ matrix.features }}"
+        fi

Cargo.toml

@@ -6,8 +6,10 @@ license = "LGPL-3.0-or-later"
 
 [dependencies]
 pyo3 = { version = "0.20.3", optional = true }
-inkwell = { version = "0.4.0", features = [ "llvm17-0" ], optional = true }
+inkwell = { version = "0.4.0", features = ["llvm17-0"], optional = true }
+crossterm = { version = "0.28.1", optional = true }
 
 [features]
 c_stdlib = ["pyo3"]
 llvm_backend = ["inkwell"]
+repl = ["crossterm"]

README.md

@@ -7,14 +7,17 @@ A starter Python interpreter written in Rust. This is intended as a learning exe
 
 ## Overview
 `memphis` contains a few execution modes, each for learning about a different aspect of interpreter/compiler development:
-1. treewalk (default) - farthest along in development
-1. bytecode VM - foundation complete, missing most Python features
-1. LLVM JIT compiler - very early, contains only a single hardcoded 
-
+1. treewalk (default): farthest along in development.
+1. bytecode VM: foundation complete, but missing many Python features.
+1. LLVM JIT compiler: very _very_ early, contains only a single hardcoded example.
+ 
 See [SUPPORTED.md](docs/SUPPORTED.md) for details on specific features.
 
 ## Design Goals
-- Zero-ish dependencies. The exceptions are `pyo3` for stdlib components implemented in C and `inkwell` for LLVM APIs. This means you can run Python code which does not call the stdlib (limiting, I know) through the treewalk interpreter or bytecode VM using no third-party Rust code.
+- Minimal dependencies. Uses zero dependencies by default, or enable the REPL, Python stdlib, or LLVM backend as needed. This means you can run Python code which does not call the stdlib (limiting, I know) through the treewalk interpreter or bytecode VM using no third-party Rust code. I find this kinda neat and worth preserving.
+  - `pyo3`: Only needed for c_stdlib functionality.
+  - `crossterm`: Only needed for REPL support.
+  - `inkwell`: Only needed if using the LLVM backend.
 - No shortcuts. This is a learning exercise, so try to do things the "right" way, even if it takes a few tries.
 
 ## Installation

docs/DEVELOPING.md

@@ -1,23 +1,32 @@
 # Developing
 ## Local Development
+This project will live and die by `cargo`.
 ```bash
 cargo build
 cargo test
 cargo run examples/test.py
+```
+## Feature Flags
+Feature flags are needed to enable C stdlib or REPL support (or the experimental LLVM backend).
+```bash
 # if examples/test.py depends on stdlib features implemented in C
 cargo run --features c_stdlib examples/test.py
+
+# script to run all combinations of feature flags
+./test_features.sh
 ```
 ## Benchmarking
 To compare runtime, we can build in release mode and use the different engines.
 ```bash
 cargo install --path . --all-features
 hyperfine "memphis examples/loop_perf.py tw" "memphis examples/loop_perf.py vm" "memphis examples/loop_perf.py llvm" --warmup 5
 ```
-
-We require debug symbols to produce a flamegraph.
+### Flamegraph
+This is a cool way to visualize why a bytecode VM is more performant than a treewalk interpreter.
 ```bash
 cargo install flamegraph
 cargo build --all-features
+# we require debug symbols to produce a flamegraph, hence invoking the binary from `target/debug`.
 sudo flamegraph -v -o tw.svg -- target/debug/memphis examples/loop_perf.py tw
 sudo flamegraph -v -o vm.svg -- target/debug/memphis examples/loop_perf.py vm
 sudo flamegraph -v -o llvm.svg -- target/debug/memphis examples/loop_perf.py llvm

src/bytecode_vm/types.rs

@@ -7,7 +7,6 @@ use super::vm::types::{Class, FunctionObject, Method, Object, Reference};
 
 #[derive(Clone, PartialEq, Debug)]
 pub enum Value {
-    Void,
     None,
     Integer(i64),
     String(String),
@@ -22,20 +21,19 @@ pub enum Value {
 
 impl Default for Value {
     fn default() -> Self {
-        Self::Void
+        Self::None
     }
 }
 
 impl Voidable for Value {
-    fn is_void(&self) -> bool {
-        matches!(self, Value::Void)
+    fn is_none(&self) -> bool {
+        matches!(self, Value::None)
     }
 }
 
 impl From<Reference> for Value {
     fn from(value: Reference) -> Self {
         match value {
-            Reference::Void => Value::Void,
             Reference::Int(i) => Value::Integer(i),
             Reference::Bool(i) => Value::Boolean(i),
             // These require a lookup using VM state and must be converted before this function.
@@ -58,7 +56,6 @@ impl From<&Constant> for Value {
 impl Display for Value {
     fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
         match self {
-            Value::Void => unreachable!(),
             Value::None => write!(f, "None"),
             Value::Integer(i) => write!(f, "{}", i),
             Value::String(i) => write!(f, "{}", i),

src/bytecode_vm/vm/mod.rs

@@ -165,7 +165,7 @@ impl VirtualMachine {
             }
         }
 
-        Value::Void
+        Value::None
     }
 
     /// This does not kick off a separate loop; instead, `run_loop` continues execution with the
@@ -200,7 +200,7 @@ impl VirtualMachine {
     pub fn take(&mut self, reference: Reference) -> Value {
         match reference {
             Reference::ObjectRef(index) => {
-                mem::replace(&mut self.object_table[*index], Value::Void)
+                mem::replace(&mut self.object_table[*index], Value::None)
             }
             Reference::ConstantRef(index) => self.constant_pool.get(*index).unwrap().into(),
             _ => reference.into(),
@@ -211,7 +211,7 @@ impl VirtualMachine {
     /// all other types.
     fn create(&mut self, value: Value) -> Reference {
         match value {
-            Value::Integer(_) | Value::Boolean(_) | Value::Void => value.into(),
+            Value::Integer(_) | Value::Boolean(_) => value.into(),
             _ => {
                 let index = Index::new(self.object_table.len());
                 self.object_table.push(value);
@@ -431,7 +431,8 @@ impl VirtualMachine {
                     self.execute_method(method.as_method().clone(), args);
                 }
                 Opcode::ReturnValue => {
-                    let return_value = self.pop().unwrap_or(Reference::Void);
+                    // TODO is reference of 0 a safe default here?
+                    let return_value = self.pop().unwrap_or(Reference::Int(0));
 
                     // Exit the loop if there are no more frames
                     if self.call_stack.is_empty() {

src/bytecode_vm/vm/types.rs

@@ -14,7 +14,6 @@ pub type Namespace = HashMap<String, Reference>;
 /// [`StackValue::ConstantRef`] items reference an immutable object in the constant pool.
 #[derive(Debug, PartialEq, Clone, Copy)]
 pub enum Reference {
-    Void,
     Int(i64),
     Bool(bool),
     ObjectRef(ObjectTableIndex),
@@ -30,7 +29,6 @@ impl Display for Reference {
 impl From<Value> for Reference {
     fn from(value: Value) -> Self {
         match value {
-            Value::Void => Reference::Void,
             Value::Integer(i) => Reference::Int(i),
             Value::Boolean(i) => Reference::Bool(i),
             _ => unimplemented!(),

src/core/mod.rs

@@ -10,9 +10,11 @@ pub use stack::Stack;
 
 use crate::{parser::Parser, types::errors::MemphisError};
 
-/// Return types which Void are used internally, but should never be displayed to the developer.
+/// Return types which None are used internally, but should never be displayed to the developer.
 pub trait Voidable {
-    fn is_void(&self) -> bool;
+    // This is only used in the REPL right now, but it is referenced other places.
+    #[allow(dead_code)]
+    fn is_none(&self) -> bool;
 }
 
 pub trait InterpreterEntrypoint {

src/crosscheck/test_value.rs

@@ -6,7 +6,6 @@ use crate::treewalk::types::ExprResult;
 /// provided the [`From`] trait is implemented.
 #[derive(Clone, Debug, PartialEq)]
 pub enum TestValue {
-    Void,
     None,
     Integer(i64),
     String(String),
@@ -16,7 +15,6 @@ pub enum TestValue {
 impl From<Value> for TestValue {
     fn from(value: Value) -> Self {
         match value {
-            Value::Void => TestValue::Void,
             Value::None => TestValue::None,
             Value::Integer(val) => TestValue::Integer(val),
             Value::String(val) => TestValue::String(val),
@@ -32,7 +30,6 @@ impl From<Value> for TestValue {
 impl From<ExprResult> for TestValue {
     fn from(value: ExprResult) -> Self {
         match value {
-            ExprResult::Void => TestValue::Void,
             ExprResult::None => TestValue::None,
             ExprResult::Integer(_) => {
                 TestValue::Integer(value.as_integer_val().expect("failed to get integer"))

src/init/builder.rs

@@ -51,7 +51,7 @@ impl Builder {
     pub fn text(&mut self, text: &str) -> &mut Self {
         self.init_state();
 
-        // hmm this shouldn't be necessary, especially for VM runs
+        // TODO figure out how to deal with StackFrames for VM mode
         let stack_frame = StackFrame::new_module(LoadedModule::new_virtual(text));
         self.state.clone().unwrap().push_context(stack_frame);
         self.text = Some(text.into());
@@ -74,20 +74,21 @@ impl Builder {
         self
     }
 
-    fn init_state(&mut self) {
-        self.state = match self.state.clone() {
-            Some(s) => Some(s),
-            None => Some(Container::new(State::new())),
-        };
+    fn init_text(&mut self) {
+        if self.text.is_none() {
+            self.text(&String::default());
+        }
     }
 
-    pub fn parser(&mut self) -> Parser {
+    fn init_state(&mut self) {
         if self.state.is_none() {
-            panic!("State never set! Did you forget to call `text` or `path`?");
-        }
-        if self.text.is_none() {
-            panic!("Text never set! Did you forget to call `text` or `path`?");
+            self.state = Some(Container::new(State::default()));
         }
+    }
+
+    pub fn parser(&mut self) -> Parser {
+        self.init_text();
+        self.init_state();
         let lexer = Lexer::new(&self.text.clone().unwrap());
         Parser::new(lexer.tokens(), self.state.clone().unwrap())
     }

src/init/mod.rs

@@ -1,7 +1,13 @@
 mod builder;
 mod memphis;
+#[cfg(feature = "repl")]
 mod repl;
+#[cfg(feature = "repl")]
+mod terminal_io;
 
 pub use builder::Builder;
 pub use memphis::Memphis;
+#[cfg(feature = "repl")]
 pub use repl::Repl;
+#[cfg(feature = "repl")]
+pub use terminal_io::{CrosstermIO, TerminalIO};