Further integration of functionality into lively frontend

SQUASHED: SQUASHED-AUTO-COMMIT-src-components-tools-astro-plot.js-AUTO-COMMIT-src-components-tools-astro-view-example-transformer.py-AUTO-COMMIT-src-components-tools-astro-view.html-AUTO-COMMIT-src-components-tools-astro-view.js-AUTO-COMMIT-src-components-tools-astro-view.js.l4a,

src/components/tools/astro-plot.js

@@ -58,39 +58,10 @@ export default class AstroPlot extends Morph {
 
   displayContent(str) {
     return str.slice(0, 100).replace('\n', '<br>');
-  } 
+  }
 
-  async initialize() {
-    this.windowTitle = "AstroPlot";
-
-    lively.html.registerKeys(this); // automatically installs handler for some methods
-
-
-    let container = this.get('#embedding_plot')
-    let getExampleData = () => new Promise((resolve, reject) => {
-      d3.csv(
-        'https://raw.githubusercontent.com/plotly/datasets/master/3d-scatter.csv', 
-        (err, rows) => {
-          if (err) reject(err) else resolve(rows)
-        }
-      )
-    });
-
-    const getRealData2 = async () => {
-      const response = await fetch("http://127.0.0.1:5000/dataset/d3-force-main/umap");
-      return await response.json();
-    }
-
-    const getRealData = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/umap")
-      .then(response => response.json());
-    const getClusters = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/clusters")
-      .then(response => response.json());
-
-    //const getRealData = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/umap")
-
-    // const response = await getRealData();
-    const features = await getRealData(); //await response.json()
-    const clusters = await getClusters();
+  async displayData(features, clusters) {
+    if (!clusters) clusters = Array(features.length).fill(0);
 
     const dataframe = {
       _push(el) {
@@ -99,12 +70,12 @@ export default class AstroPlot extends Morph {
           this[key].push(value);
         });
       }
-    }; 
+    };
 
     features.forEach(({ 
       umap_embedding,
       function_name,
-      content,
+      content = "",
       id
     }, i) => dataframe._push({
       x: umap_embedding[0],
@@ -117,7 +88,7 @@ export default class AstroPlot extends Morph {
         content, 
         contentAbbr: `${this.displayContent(content)}...`
       },
-      ids: id
+      ids: id || i
     }));
 
     const data = [
@@ -154,12 +125,35 @@ export default class AstroPlot extends Morph {
       r: 0,
       b: 0,
       t: 0
-      }};
-
+    }};
+
+    let container = this.get('#embedding_plot')
+    container.innerHTML = "";
+
     this.plot = await Plotly.newPlot(container, data, layout, {
       responsive: true,
       displayModeBar: false
     });
+  }
+
+  async initialize() {
+    this.windowTitle = "AstroPlot";
+
+    lively.html.registerKeys(this); // automatically installs handler for some methods
+
+
+    const getRealData = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/umap")
+      .then(response => response.json());
+    const getClusters = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/clusters")
+      .then(response => response.json());
+
+    //const getRealData = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/umap")
+
+    // const response = await getRealData();
+    const features = await getRealData(); //await response.json()
+    const clusters = await getClusters();
+
+    await this.displayData(features, clusters);
  }
 
 /*
@@ -175,6 +169,4 @@ export default class AstroPlot extends Morph {
     // this.someJavaScriptProperty = 42
     // this.appendChild(<div>This is my content</div>)
   }
-
-
 }

src/components/tools/astro-view-example-transformer.py

@@ -1,18 +1,4 @@
-QUERY = """
-    (function_declaration) @function
-    
-    (class_declaration
-        (type_identifier) @class_name
-        (class_body
-            (method_definition
-                name: (property_identifier) @method_name
-                body: (statement_block) @method_body
-            ) @method
-        )
-    ) @class
-"""
-
-class ConcatEmbedding(CodeTransformation):
+class WeightedEmbedding(CodeTransformation):
     def queryAST(self):
         return '''
             (class_declaration
@@ -26,32 +12,6 @@ def queryAST(self):
             ) @class
         '''
 
-    async def mapCaptures(self, query_result, text_embedding, make_query):
-        # (path, query_id, capture_dict) = query_result
-
-        class_name = self.textFromCapture(query_result, 'class_name')
-        method = self.textFromCapture(query_result, 'method')
-
-        concat_embedding = f'''
-            class {class_name} {{
-                {method}
-            }}
-            '''.strip()
-
-        pooler_embedding = await text_embedding(concat_embedding)
-
-        return {
-            'embedded_code': concat_embedding,
-            'embedding': pooler_embedding
-        }
-
-    def reduce(self, df):
-        return df['embedding']
-
-class WeightedEmbedding(CodeTransformation):
-    def queryAST(self):
-        return QUERY
-
     async def mapCaptures(self, query_result, text_embedding, make_query):
         # (id, path, query_id, captures, _) = query_result
 
@@ -69,43 +29,13 @@ async def mapCaptures(self, query_result, text_embedding, make_query):
 
         # return dict with embeddings
         return {
-            "class_embedding": class_embedding, 
-            "method_name_embedding": method_name_embedding, 
-            "method_body_embedding": method_body_embedding
+            "class_embedding": np.array(class_embedding), 
+            "method_name_embedding": np.array(method_name_embedding), 
+            "method_body_embedding": np.array(method_body_embedding)
         }
 
     def reduce(self, df):
-        # weighted sum of embeddings
-        # class_embedding 0.1, method_name_embedding 0.2, method_body_embedding 0.7
-        # multiply whole columns by respective scalar, then add them together
-
-        return df['class_embedding'] + df['method_name_embedding'] + df['method_body_embedding']
-
-class IdentifierEmbedding(CodeTransformation):
-
-    def queryAST(self):
-        return QUERY
-
-    async def mapCaptures(self, query_result, text_embedding, make_query):
-        (id, path, query_id, captures) = query_result
-
-        node = captures['method_body']
-        query = make_query('(identifier) @identifier')
-
-        all_identifiers = query.matches(node)
-        all_identifiers = [captures.get('identifier').text.decode() for q_id, captures in all_identifiers]
-
-        identifier_embeddings = asyncio.gather(*[text_embedding(identifier) for identifier in all_identifiers])
-
-        # return dict with embeddings
-        return {
-            "identifiers": all_identifiers, 
-            "identifers_embeddings": identifier_embeddings, 
-        }
-
-    def reduce(self, df):
-        # weighted sum of embeddings
-        # class_embedding 0.1, method_name_embedding 0.2, method_body_embedding 0.7
-        # multiply whole columns by respective scalar, then add them together
-
-        return df['identifiers']
+        return \
+            df['class_embedding'] * 0.2 + \
+            df['method_name_embedding'] * 0.1 + \
+            df['method_body_embedding'] * 0.7

src/components/tools/astro-view.html

@@ -140,21 +140,30 @@
         <div class="layout-row">
           <input type="text" id="transformerSourcePath" value="">
         </div>
-        <lively-editor class="pane editor" id="transformerSource"></lively-editor>
 
         <div>
+          <button class="toggle" id="runParse" title="Parse files into ASTs">
+            Parse<i class="fa fa-refresh fa-fw"></i>
+          </button>
           <button class="toggle" id="runQuery" title="Query ASTs">
             Query<i class="fa fa-refresh fa-fw"></i>
           </button>
           <button class="toggle" id="runMap" title="Generate Embeddings">
             Map<i class="fa fa-refresh fa-fw"></i>
           </button>
-          <button class="toggle" id="runReduce" title="Apply Weights)">
+          <button class="toggle" id="runReduce" title="Apply Weights">
             Reduce<i class="fa fa-refresh fa-fw"></i>
           </button>
+          <button class="toggle" id="runUmap" title="Map to 3D">
+            UMAP<i class="fa fa-refresh fa-fw"></i>
+          </button>
           <div id="status"/>
         </div>
 
+        <lively-editor class="pane editor" id="transformerSource"></lively-editor>
+
+        <lively-separator></lively-separator>
+
       </div>
     </div>
   </div>

src/components/tools/astro-view.js

@@ -52,10 +52,18 @@ export default class AstroView extends Morph {
   set transformerSourceURL(urlString) { this.transformerSourcePath.value = urlString; }
   onTransformerSourcePathEntered(urlString) { this.loadTransformerSourceFile(urlString); }
 
+  // Plot
+  get astroPlot() { return this.get("#astro-plot"); }
+
+  get api() { return "http://127.0.0.1:5000"; }
+
 
   get astInspector() { return this.get("#ast"); }
 
   get updateButton() { return this.get("#update"); }
+  get runQueryButton() { return this.get('#runQuery'); }
+  get runMapButton() { return this.get('#runMap'); }
+  get runReduceButton() { return this.get('#runReduce'); }
 
   get autoUpdate() { return this._autoUpdate; }
   set autoUpdate(bool) {
@@ -288,11 +296,103 @@ export default class AstroView extends Morph {
   }
 
   async updateTransformer() {
-    this.status = "transformer updated: sending..."
+    this.status = "transformer: sending..."
     try {
-
+      let response = await fetch(`${this.api}/dataset/${this.projectName}/transformer`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          text: this.transformerSource
+        }),
+      })
+      response = await response.json();
+      if (response.error) throw new Error(response.error);
+      this.status = "transformer: using " + response.transformer;
     } catch (e) {
-
+      this.status = "transformer: " + e;
+    }
+  }
+
+  async onRunParse() {
+    this.status = "parser: running..."
+    try {
+      let language = "typescript";
+      let response = await fetch(`${this.api}/dataset/${this.projectName}/parse?language=${language}`, {
+        method: 'POST',
+      })
+      response = await response.json();
+      if (response.error) throw new Error(response.error);
+      this.status = "parser: currently " + response.ASTs + " ASTs in memory ";
+    } catch (e) {
+      this.status = "parser: " + e;
+    }
+  }
+
+  async onRunQuery() {
+    this.status = "query: running..."
+    try {
+      let response = await fetch(`${this.api}/dataset/${this.projectName}/run_query`, {
+        method: 'POST',
+      })
+      response = await response.json();
+      if (response.error) throw new Error(response.error);
+      this.status = "query: matched " + response.matches + " items in " + response.files + " files";
+    } catch (e) {
+      this.status = "query: " + e;
+    }
+  }
+
+  async onRunMap() {
+    this.status = "map: running..."
+    try {
+      let response = await fetch(`${this.api}/dataset/${this.projectName}/run_map`, {
+        method: 'POST',
+      })
+      response = await response.json();
+      if (response.error) throw new Error(response.error);
+      this.status = "map: success. Data columns: " + response.columns;
+    } catch (e) {
+      this.status = "map: " + e;
+    }
+  }
+
+  async onRunReduce() {
+    this.status = "reduce: running..."
+    try {
+      let response = await fetch(`${this.api}/dataset/${this.projectName}/run_reduce`, {
+        method: 'POST',
+      })
+      response = await response.json();
+      if (response.error) throw new Error(response.error);
+      this.status = "reduce: success";
+    } catch (e) {
+      this.status = "reduce: " + e;
+    }
+  }
+
+  async onRunUmap() {
+    this.status = "umap: running..."
+    let data;
+    try {
+      let response = await fetch(`${this.api}/dataset/${this.projectName}/run_umap`, {
+        method: 'POST',
+      })
+      response = await response.json();
+      if (response.error) throw new Error(response.error);
+      data = response.umap;
+      this.status = "umap: success";
+    } catch (e) {
+      this.status = "umap: " + e;
+      return;
+    }
+    debugger;
+
+    try {
+      this.astroPlot.displayData(data)
+    } catch (e) {
+      this.status = "plot: " + e;
     }
   }