intermediate evaluation states now report to the client

toolchain/Cargo.toml

@@ -2,11 +2,9 @@
 members = [
     "chidori-core",
     "chidori-prompt-format",
-    "chidori-im-rs",
     "chidori-static-analysis",
     "chidori-debugger",
     "chidori-tsne",
-    "chidori-optimizer-dsp",
 ]
 resolver = "2"
 

toolchain/book_src/ARCHITECTURE.md

@@ -6,10 +6,8 @@ Chidori consists of the following crates:
 
 - `chidori-core` contains our orchestrator and runtime.
 - `chidori-debugger` contains a UI for visualizing and debugging Chidori executed programs.
-- `chidori-im-rs` contains a fork of im-rs to add support for persisting these data structures to disk.
 - `chidori-prompt-format` implements handlebars-like templating with support for tracing composition
 - `chidori-static-analysis` implements our parsing and extraction of control-flow from Python and TypeScript source code
-- `chidori-optimizer-dsp` (IGNORE) - not yet implemented, in the future we'd like to support DSPy-like optimization
 - `chidori-tsne` (IGNORE) - not yet implemented, in the future we'd like to add support for visualizing embeddings within our debugger
 
 

toolchain/chidori-core/src/cells/code_cell.rs

@@ -38,10 +38,11 @@ pub fn code_cell(execution_state_id: ExecutionNodeId, cell: &CodeCell, range: &T
                             &None,
                             &None,
                         ).await?;
+                        // TODO: don't unwrap the output result here
                         Ok(OperationFnOutput {
                             has_error: false,
                             execution_state: None,
-                            output: result.0.unwrap(),
+                            output: result.0,
                             stdout: result.1,
                             stderr: result.2,
                         })
@@ -90,7 +91,7 @@ pub fn code_cell(execution_state_id: ExecutionNodeId, cell: &CodeCell, range: &T
                                     Ok(OperationFnOutput {
                                         has_error: false,
                                         execution_state: None,
-                                        output: v.0,
+                                        output: Ok(v.0),
                                         stdout: v.1,
                                         stderr: v.2,
                                     }),

toolchain/chidori-core/src/cells/code_gen_cell.rs

@@ -90,7 +90,7 @@ pub fn code_gen_cell(execution_state_id: ExecutionNodeId, cell: &LLMCodeGenCell,
                     Ok(OperationFnOutput {
                         has_error: false,
                         execution_state: state,
-                        output: value,
+                        output: Ok(value),
                         stdout: vec![],
                         stderr: vec![],
                     })

toolchain/chidori-core/src/cells/memory_cell.rs

@@ -60,15 +60,15 @@ pub fn memory_cell(execution_state_id: ExecutionNodeId, cell: &MemoryCell, range
                                     match key.as_str() {
                                         "insert" => {
                                             let (embedded_value, _) = s.dispatch(&embedding_function, value.clone(), None).await?;
-                                            let embedding = rkyv_to_vec_float(embedded_value);
+                                            let embedding = rkyv_to_vec_float(embedded_value.unwrap());
                                             let contents = serialized_value_to_json_value(&value);
                                             let row = vec![(&embedding, contents)];
                                             db.insert("default".to_string(), &row);
                                             sender.send(RkyvSerializedValue::String(String::from("Success"))).unwrap();
                                         }
                                         "search" => {
                                             let (embedded_value, _) = s.dispatch(&embedding_function, value.clone(), None).await?;
-                                            let embedding = rkyv_to_vec_float(embedded_value);
+                                            let embedding = rkyv_to_vec_float(embedded_value.unwrap());
                                             let results = db.search("default".to_string(), embedding, 5);
                                             let mut output = vec![];
                                             for (_, value) in results.iter() {

toolchain/chidori-core/src/cells/mod.rs

@@ -1,6 +1,5 @@
 pub mod template_cell;
 pub mod code_cell;
-pub mod web_cell;
 pub mod llm_prompt_cell;
 pub mod memory_cell;
 pub mod embedding_cell;
@@ -443,7 +442,6 @@ pub enum CellTypes {
     CodeGen(LLMCodeGenCell, TextRange),
     Prompt(LLMPromptCell, TextRange),
     Embedding(LLMEmbeddingCell, TextRange),
-    Web(WebserviceCell, TextRange),
     Template(TemplateCell, TextRange),
     Memory(MemoryCell, TextRange),
 }
@@ -471,7 +469,6 @@ pub fn get_cell_name(cell: &CellTypes) -> &Option<String> {
             LLMPromptCell::Chat { name, .. } => name,
             LLMPromptCell::Completion { .. } => &None,
         },
-        CellTypes::Web(c, _) => &c.name,
         CellTypes::Template(c, _) => &c.name,
         CellTypes::Memory(c, _) => &c.name,
         CellTypes::Embedding(c, _) => &c.name,

toolchain/chidori-core/src/cells/template_cell.rs

@@ -80,7 +80,7 @@ mod test {
             std::collections::HashMap::new()
         );
         let output = op.execute(&ExecutionState::new_with_random_id(), input, None, None).await?;
-        assert_eq!(output.output, crate::execution::primitives::serialized_value::RkyvSerializedValue::String("Hello, !".to_string()));
+        assert_eq!(output.output, Ok(crate::execution::primitives::serialized_value::RkyvSerializedValue::String("Hello, !".to_string())));
         Ok(())
     }
 }