diff --git a/src/components/tools/astro-plot.js b/src/components/tools/astro-plot.js index 6eed758d8..01c76d070 100644 --- a/src/components/tools/astro-plot.js +++ b/src/components/tools/astro-plot.js @@ -58,39 +58,10 @@ export default class AstroPlot extends Morph { displayContent(str) { return str.slice(0, 100).replace('\n', '
'); - } + } - async initialize() { - this.windowTitle = "AstroPlot"; - - lively.html.registerKeys(this); // automatically installs handler for some methods - - - let container = this.get('#embedding_plot') - let getExampleData = () => new Promise((resolve, reject) => { - d3.csv( - 'https://raw.githubusercontent.com/plotly/datasets/master/3d-scatter.csv', - (err, rows) => { - if (err) reject(err) else resolve(rows) - } - ) - }); - - const getRealData2 = async () => { - const response = await fetch("http://127.0.0.1:5000/dataset/d3-force-main/umap"); - return await response.json(); - } - - const getRealData = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/umap") - .then(response => response.json()); - const getClusters = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/clusters") - .then(response => response.json()); - - //const getRealData = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/umap") - - // const response = await getRealData(); - const features = await getRealData(); //await response.json() - const clusters = await getClusters(); + async displayData(features, clusters) { + if (!clusters) clusters = Array(features.length).fill(0); const dataframe = { _push(el) { @@ -99,12 +70,12 @@ export default class AstroPlot extends Morph { this[key].push(value); }); } - }; + }; features.forEach(({ umap_embedding, function_name, - content, + content = "", id }, i) => dataframe._push({ x: umap_embedding[0], @@ -117,7 +88,7 @@ export default class AstroPlot extends Morph { content, contentAbbr: `${this.displayContent(content)}...` }, - ids: id + ids: id || i })); const data = [ @@ -154,12 +125,35 @@ export default class AstroPlot extends Morph { r: 0, b: 0, t: 0 - }}; - + }}; + + let container = this.get('#embedding_plot') + container.innerHTML = ""; + this.plot = await Plotly.newPlot(container, data, layout, { responsive: true, displayModeBar: false }); + } + + async initialize() { + this.windowTitle = "AstroPlot"; + + lively.html.registerKeys(this); // automatically installs handler for some methods + + + const getRealData = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/umap") + .then(response => response.json()); + const getClusters = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/clusters") + .then(response => response.json()); + + //const getRealData = async () => await fetch("http://127.0.0.1:5000/dataset/d3-force-main/umap") + + // const response = await getRealData(); + const features = await getRealData(); //await response.json() + const clusters = await getClusters(); + + await this.displayData(features, clusters); } /* @@ -175,6 +169,4 @@ export default class AstroPlot extends Morph { // this.someJavaScriptProperty = 42 // this.appendChild(
This is my content
) } - - } \ No newline at end of file diff --git a/src/components/tools/astro-view-example-transformer.py b/src/components/tools/astro-view-example-transformer.py index e2d6d4ef5..f05719fd7 100644 --- a/src/components/tools/astro-view-example-transformer.py +++ b/src/components/tools/astro-view-example-transformer.py @@ -1,18 +1,4 @@ -QUERY = """ - (function_declaration) @function - - (class_declaration - (type_identifier) @class_name - (class_body - (method_definition - name: (property_identifier) @method_name - body: (statement_block) @method_body - ) @method - ) - ) @class -""" - -class ConcatEmbedding(CodeTransformation): +class WeightedEmbedding(CodeTransformation): def queryAST(self): return ''' (class_declaration @@ -26,32 +12,6 @@ def queryAST(self): ) @class ''' - async def mapCaptures(self, query_result, text_embedding, make_query): - # (path, query_id, capture_dict) = query_result - - class_name = self.textFromCapture(query_result, 'class_name') - method = self.textFromCapture(query_result, 'method') - - concat_embedding = f''' - class {class_name} {{ - {method} - }} - '''.strip() - - pooler_embedding = await text_embedding(concat_embedding) - - return { - 'embedded_code': concat_embedding, - 'embedding': pooler_embedding - } - - def reduce(self, df): - return df['embedding'] - -class WeightedEmbedding(CodeTransformation): - def queryAST(self): - return QUERY - async def mapCaptures(self, query_result, text_embedding, make_query): # (id, path, query_id, captures, _) = query_result @@ -69,43 +29,13 @@ async def mapCaptures(self, query_result, text_embedding, make_query): # return dict with embeddings return { - "class_embedding": class_embedding, - "method_name_embedding": method_name_embedding, - "method_body_embedding": method_body_embedding + "class_embedding": np.array(class_embedding), + "method_name_embedding": np.array(method_name_embedding), + "method_body_embedding": np.array(method_body_embedding) } def reduce(self, df): - # weighted sum of embeddings - # class_embedding 0.1, method_name_embedding 0.2, method_body_embedding 0.7 - # multiply whole columns by respective scalar, then add them together - - return df['class_embedding'] + df['method_name_embedding'] + df['method_body_embedding'] - -class IdentifierEmbedding(CodeTransformation): - - def queryAST(self): - return QUERY - - async def mapCaptures(self, query_result, text_embedding, make_query): - (id, path, query_id, captures) = query_result - - node = captures['method_body'] - query = make_query('(identifier) @identifier') - - all_identifiers = query.matches(node) - all_identifiers = [captures.get('identifier').text.decode() for q_id, captures in all_identifiers] - - identifier_embeddings = asyncio.gather(*[text_embedding(identifier) for identifier in all_identifiers]) - - # return dict with embeddings - return { - "identifiers": all_identifiers, - "identifers_embeddings": identifier_embeddings, - } - - def reduce(self, df): - # weighted sum of embeddings - # class_embedding 0.1, method_name_embedding 0.2, method_body_embedding 0.7 - # multiply whole columns by respective scalar, then add them together - - return df['identifiers'] \ No newline at end of file + return \ + df['class_embedding'] * 0.2 + \ + df['method_name_embedding'] * 0.1 + \ + df['method_body_embedding'] * 0.7 \ No newline at end of file diff --git a/src/components/tools/astro-view.html b/src/components/tools/astro-view.html index daec452a6..f776fabd2 100644 --- a/src/components/tools/astro-view.html +++ b/src/components/tools/astro-view.html @@ -140,21 +140,30 @@
-
+ - +
+ + + +
diff --git a/src/components/tools/astro-view.js b/src/components/tools/astro-view.js index 4d33ff92b..4c2c25372 100644 --- a/src/components/tools/astro-view.js +++ b/src/components/tools/astro-view.js @@ -52,10 +52,18 @@ export default class AstroView extends Morph { set transformerSourceURL(urlString) { this.transformerSourcePath.value = urlString; } onTransformerSourcePathEntered(urlString) { this.loadTransformerSourceFile(urlString); } + // Plot + get astroPlot() { return this.get("#astro-plot"); } + + get api() { return "http://127.0.0.1:5000"; } + get astInspector() { return this.get("#ast"); } get updateButton() { return this.get("#update"); } + get runQueryButton() { return this.get('#runQuery'); } + get runMapButton() { return this.get('#runMap'); } + get runReduceButton() { return this.get('#runReduce'); } get autoUpdate() { return this._autoUpdate; } set autoUpdate(bool) { @@ -288,11 +296,103 @@ export default class AstroView extends Morph { } async updateTransformer() { - this.status = "transformer updated: sending..." + this.status = "transformer: sending..." try { - + let response = await fetch(`${this.api}/dataset/${this.projectName}/transformer`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + text: this.transformerSource + }), + }) + response = await response.json(); + if (response.error) throw new Error(response.error); + this.status = "transformer: using " + response.transformer; } catch (e) { - + this.status = "transformer: " + e; + } + } + + async onRunParse() { + this.status = "parser: running..." + try { + let language = "typescript"; + let response = await fetch(`${this.api}/dataset/${this.projectName}/parse?language=${language}`, { + method: 'POST', + }) + response = await response.json(); + if (response.error) throw new Error(response.error); + this.status = "parser: currently " + response.ASTs + " ASTs in memory "; + } catch (e) { + this.status = "parser: " + e; + } + } + + async onRunQuery() { + this.status = "query: running..." + try { + let response = await fetch(`${this.api}/dataset/${this.projectName}/run_query`, { + method: 'POST', + }) + response = await response.json(); + if (response.error) throw new Error(response.error); + this.status = "query: matched " + response.matches + " items in " + response.files + " files"; + } catch (e) { + this.status = "query: " + e; + } + } + + async onRunMap() { + this.status = "map: running..." + try { + let response = await fetch(`${this.api}/dataset/${this.projectName}/run_map`, { + method: 'POST', + }) + response = await response.json(); + if (response.error) throw new Error(response.error); + this.status = "map: success. Data columns: " + response.columns; + } catch (e) { + this.status = "map: " + e; + } + } + + async onRunReduce() { + this.status = "reduce: running..." + try { + let response = await fetch(`${this.api}/dataset/${this.projectName}/run_reduce`, { + method: 'POST', + }) + response = await response.json(); + if (response.error) throw new Error(response.error); + this.status = "reduce: success"; + } catch (e) { + this.status = "reduce: " + e; + } + } + + async onRunUmap() { + this.status = "umap: running..." + let data; + try { + let response = await fetch(`${this.api}/dataset/${this.projectName}/run_umap`, { + method: 'POST', + }) + response = await response.json(); + if (response.error) throw new Error(response.error); + data = response.umap; + this.status = "umap: success"; + } catch (e) { + this.status = "umap: " + e; + return; + } + debugger; + + try { + this.astroPlot.displayData(data) + } catch (e) { + this.status = "plot: " + e; } } diff --git a/src/components/tools/astro-view.js.l4a b/src/components/tools/astro-view.js.l4a index 312a73d39..a259961e7 100644 --- a/src/components/tools/astro-view.js.l4a +++ b/src/components/tools/astro-view.js.l4a @@ -1 +1 @@ -{"type":"Reference","version":"1e3139ef77e234fa325aa955a04fb8daa1c0fc7e","content":"/*MD # Astro View - AST Token View spelled wrong\n\nMD*/\n\n\nimport Morph from 'src/components/widgets/lively-morph.js';\nimport SyntaxChecker from 'src/client/syntax.js'\n\nimport { uuid as generateUUID, debounce, flatmap, executeAllTestRunners, promisedEvent, loc, range } from 'utils';\n\nexport default class AstroView extends Morph {\n\n static get defaultSourceURL() { return lively4url + \"/src/components/tools/astro-view-example-source.js\"; }\n static get defaultTransformerSourceURL() { return lively4url + \"/src/components/tools/astro-view-example-transformer.py\"; }\n static get defaultWorkspaceURL() { return lively4url + \"/src/components/tools/astro-view-example-workspace.js\"; }\n \n \n\n /*MD ## UI Accessing MD*/\n get container() { return this.get(\"#content\"); }\n \n // Status Text\n get statusLine() { return this.get(\"#status\"); }\n set status(text) { this.statusLine.innerText = text; }\n \n // Source\n get sourceEditor() { return this.get(\"#source\"); } \n get sourceLCM() { return this.sourceEditor.livelyCodeMirror(); }\n get sourceCM() { return this.sourceEditor.currentEditor(); }\n get source() { return this.sourceCM.getValue(); }\n \n // Source Path\n get sourcePath() { return this.get(\"#sourcePath\"); }\n get sourceURL() { return this.sourcePath.value; }\n set sourceURL(urlString) { this.sourcePath.value = urlString; }\n onSourcePathEntered(urlString) { this.loadSourceFile(urlString); }\n \n // Project Name\n get projectNameInput() { return this.get('#projectName'); }\n get projectName() { return this.projectNameInput.value; }\n set projectName(text) { this.projectNameInput.value = text; }\n \n // Transformer Code\n get transformerSourceEditor() { return this.get(\"#transformerSource\"); } \n get transformerSourceLCM() { return this.transformerSourceEditor.livelyCodeMirror(); }\n get transformerSourceCM() { return this.transformerSourceEditor.currentEditor(); }\n get transformerSource() { return this.transformerSourceCM.getValue(); }\n \n // Transformer Code Path\n get transformerSourcePath() { return this.get(\"#transformerSourcePath\"); }\n get transformerSourceURL() { return this.transformerSourcePath.value; }\n set transformerSourceURL(urlString) { this.transformerSourcePath.value = urlString; }\n onTransformerSourcePathEntered(urlString) { this.loadTransformerSourceFile(urlString); }\n \n \n get astInspector() { return this.get(\"#ast\"); }\n \n get updateButton() { return this.get(\"#update\"); }\n \n get autoUpdate() { return this._autoUpdate; }\n set autoUpdate(bool) {\n this.updateButton.classList.toggle(\"on\", bool);\n this.updateButton.querySelector(\"i\").classList.toggle(\"fa-spin\", bool);\n this._autoUpdate = bool;\n }\n onUpdate(evt) {\n if (evt.button === 2) this.autoUpdate = !this.autoUpdate;\n this.update();\n }\n \n log(s) {\n console.log(s)\n }\n\n /*MD ## Initialization MD*/\n\n async loadSourceFile(urlString) {\n console.log(\"LOAD \", urlString);\n this.sourceURL = urlString;\n this.sourceEditor.setURL(lively.paths.normalizePath(urlString, \"\"));\n await this.sourceEditor.loadFile();\n await this.update(); \n }\n \n async loadTransformerSourceFile(urlString) {\n console.log(\"LOAD \", urlString);\n this.transformerSourceURL = urlString;\n this.transformerSourceEditor.setURL(lively.paths.normalizePath(urlString, \"\"));\n await this.transformerSourceEditor.loadFile();\n await this.update(); \n }\n \n async initialize() {\n this.windowTitle = \"Astro View\";\n this.registerButtons();\n\n this.getAllSubmorphs(\"button\").forEach(button => {\n button.addEventListener('contextmenu', e => {\n e.preventDefault();\n e.stopPropagation();\n e.currentTarget.dispatchEvent(new MouseEvent(\"click\", {button: 2}));\n });\n });\n \n await this.sourceEditor.awaitEditor();\n await this.transformerSourceEditor.awaitEditor();\n \n this.sourceEditor.hideToolbar();\n this.astInspector.connectEditor(this.sourceEditor);\n this.sourceLCM.doSave = async () => {\n this.save();\n };\n this.transformerSourceLCM.doSave = async () => {\n this.save();\n };\n \n this.sourceEditor.livelyCodeMirror().editor.on(\"cursorActivity\", (cm) => {\n // #TODO continue here....\n console.log(cm)\n // this.selectPath(pathKeys);\n })\n \n this.debouncedUpdate = this.update::debounce(500);\n this.sourceLCM.addEventListener(\"change\", (() =>\n SyntaxChecker.checkForSyntaxErrors(this.sourceCM))::debounce(200));\n this.sourceLCM.addEventListener(\"change\", () => {\n if (this.autoUpdate) this.debouncedUpdate()\n });\n \n this.debouncedUpdateTransformer = this.updateTransformer::debounce(500);\n this.transformerSourceLCM.addEventListener(\"change\", (() => {\n // SyntaxChecker.checkForSyntaxErrors(this.transformerSourceCM))::debounce(200) \n }));\n this.transformerSourceLCM.addEventListener(\"change\", () => {\n if (this.autoUpdate) this.debouncedUpdateTransformer()\n });\n \n this.sourcePath.addEventListener(\"keyup\", evt => {\n if (evt.code == \"Enter\") this.onSourcePathEntered(this.sourcePath.value);\n });\n this.transformerSourcePath.addEventListener(\"keyup\", evt => {\n if (evt.code == \"Enter\") this.onTransformerSourcePathEntered(this.transformerSourcePath.value);\n });\n\n const source = this.getAttribute(\"source\");\n if (source) this.loadSourceFile(source);\n \n const transformerSource = this.getAttribute(\"transformerSource\");\n if (transformerSource) this.loadTransformerSourceFile(transformerSource);\n \n this.projectName = this.getAttribute(\"projectName\") || \"\";\n \n this.autoUpdate = true;\n\n this.dispatchEvent(new CustomEvent(\"initialize\"));\n }\n\n onEditorCursorActivity(cm) {\n var from = cm.getCursor(true)\n var to = cm.getCursor(false)\n \n this.get(\"#editorInfo\").textContent = `${cm.indexFromPos(from)}-${cm.indexFromPos(to)}`\n }\n \n async updateTokens() { \n let api = \"http://127.0.0.1:5000\";\n let dataset = \"d3-force-main\";\n \n this.status = \"source updated: fetching...\"\n \n try {\n this.tokens = null;\n \n let response = await fetch(`${api}/tokenize`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify({\n text: this.source\n })\n })\n \n let tokens = await response.json();\n // filter new-lines\n tokens = tokens.filter(ea => !ea.value.match(/^[ \\n]+$/));\n \n this.tokens = tokens;\n } catch (e) {\n this.status = `error fetching tokens: ${e}`;\n this.log(`error fetching tokens: ${e}`);\n }\n \n try {\n let response = await fetch(`${api}/dataset/${dataset}/embedding`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify({\n text: this.source\n }),\n })\n \n let embedding = await response.json()\n this.embedding = embedding;\n } catch (e) {\n this.log(`error fetching embedding: ${e}`);\n this.status = `error fetching embedding: ${e}`;\n }\n \n if (this.embedding) {\n let formatted = JSON.stringify(this.embedding)\n this.get('#pool_embedding').innerText = formatted\n this.get('#astro-plot').showFeature(this.embedding)\n }\n\n this.log('fetched tokens', this.tokens)\n \n if (this.tokens) {\n this.get(\"#tokens\").innerHTML = \"\"\n this.tokens.forEach((token) => {\n let tokenView = \n
\n
{token.id}
\n
this.selectToken(tokenView, token)}\n pointerenter={() => this.hoverToken(tokenView, token, true)}\n pointerleave={() => this.hoverToken(tokenView, token, false)}\n >{token.value}
\n
{token.start}-{token.end}
\n
\n this.get(\"#tokens\").appendChild(tokenView)\n })\n } else {\n this.get(\"#tokens\").innerHTML = \"Error fetching tokens\"\n this.status = `error fetching tokens`;\n }\n }\n \n selectToken(view, token) {\n if (this.selectedTokenView) this.selectedTokenView.classList.remove(\"selected\")\n view.classList.add(\"selected\")\n this.selectedTokenView = view\n \n this.get(\"#embeddings\").innerHTML = \"\"\n let rows = []\n \n let tds = Array.from(token.value)\n .map(ea => ea.charCodeAt(0))\n .map(ea => {ea})\n \n rows.push({...tds})\n \n let table = {...rows}
\n \n this.get(\"#embeddings\").appendChild(table)\n \n }\n \n \n hoverToken(view, token, active) {\n if (active) {\n const start = loc(this.sourceCM.posFromIndex(token.start));\n const end = loc(this.sourceCM.posFromIndex(token.end));\n this.hoverMarker = this.sourceCM.markText(start.asCM(), end.asCM(), {css: \"background-color: #fe3\"});\n } else {\n this.hoverMarker.clear();\n this.hoverMarker = null;\n }\n }\n \n /*MD ## Execution MD*/\n \n async update() {\n this.lastSource = this.source\n this.log(\"source code changed, length: \" + this.source.length + \"\")\n \n try {\n var node = await this.astInspector.treeSitterParse(this.source)\n this.treeSitterRootNode = node.rootNode\n this.astInspector.inspect(this.treeSitterRootNode);\n } catch (e) {\n this.astInspector.inspect({Error: e.message});\n }\n \n this.updateTokens();\n }\n \n async updateTransformer() {\n this.status = \"transformer updated: sending...\"\n try {\n \n } catch (e) {\n \n }\n }\n\n async save() {\n if (this.sourceURL) {\n await this.sourceEditor.saveFile();\n }\n if (this.transformerSourceURL) {\n await this.transformerSourceEditor.saveFile();\n }\n this.update();\n }\n\n /*MD ## Lively Integration MD*/\n\n livelyPrepareSave() {\n this.setAttribute('source', this.sourceURL);\n this.setAttribute('transformerSource', this.transformerSourceURL);\n this.setAttribute('projectName', this.projectName);\n \n console.log(\"PREPARE SAVE (AST Explorer)\");\n }\n \n livelyMigrate(other) {\n }\n\n async livelyExample() {\n await this.loadSourceFile(AstroView.defaultSourceURL);\n await this.loadTransformerSourceFile(AstroView.defaultTransformerSourceURL)\n }\n}"} \ No newline at end of file +{"type":"Reference","version":"3623cfe209131c30b24250ef4c8a2fb33f266bb7","content":"/*MD # Astro View - AST Token View spelled wrong\n\nMD*/\n\n\nimport Morph from 'src/components/widgets/lively-morph.js';\nimport SyntaxChecker from 'src/client/syntax.js'\n\nimport { uuid as generateUUID, debounce, flatmap, executeAllTestRunners, promisedEvent, loc, range } from 'utils';\n\nexport default class AstroView extends Morph {\n\n static get defaultSourceURL() { return lively4url + \"/src/components/tools/astro-view-example-source.js\"; }\n static get defaultTransformerSourceURL() { return lively4url + \"/src/components/tools/astro-view-example-transformer.py\"; }\n static get defaultWorkspaceURL() { return lively4url + \"/src/components/tools/astro-view-example-workspace.js\"; }\n \n \n\n /*MD ## UI Accessing MD*/\n get container() { return this.get(\"#content\"); }\n \n // Status Text\n get statusLine() { return this.get(\"#status\"); }\n set status(text) { this.statusLine.innerText = text; }\n \n // Source\n get sourceEditor() { return this.get(\"#source\"); } \n get sourceLCM() { return this.sourceEditor.livelyCodeMirror(); }\n get sourceCM() { return this.sourceEditor.currentEditor(); }\n get source() { return this.sourceCM.getValue(); }\n \n // Source Path\n get sourcePath() { return this.get(\"#sourcePath\"); }\n get sourceURL() { return this.sourcePath.value; }\n set sourceURL(urlString) { this.sourcePath.value = urlString; }\n onSourcePathEntered(urlString) { this.loadSourceFile(urlString); }\n \n // Project Name\n get projectNameInput() { return this.get('#projectName'); }\n get projectName() { return this.projectNameInput.value; }\n set projectName(text) { this.projectNameInput.value = text; }\n \n // Transformer Code\n get transformerSourceEditor() { return this.get(\"#transformerSource\"); } \n get transformerSourceLCM() { return this.transformerSourceEditor.livelyCodeMirror(); }\n get transformerSourceCM() { return this.transformerSourceEditor.currentEditor(); }\n get transformerSource() { return this.transformerSourceCM.getValue(); }\n \n // Transformer Code Path\n get transformerSourcePath() { return this.get(\"#transformerSourcePath\"); }\n get transformerSourceURL() { return this.transformerSourcePath.value; }\n set transformerSourceURL(urlString) { this.transformerSourcePath.value = urlString; }\n onTransformerSourcePathEntered(urlString) { this.loadTransformerSourceFile(urlString); }\n \n // Plot\n get astroPlot() { return this.get(\"#astro-plot\"); }\n \n get api() { return \"http://127.0.0.1:5000\"; }\n \n \n get astInspector() { return this.get(\"#ast\"); }\n \n get updateButton() { return this.get(\"#update\"); }\n get runQueryButton() { return this.get('#runQuery'); }\n get runMapButton() { return this.get('#runMap'); }\n get runReduceButton() { return this.get('#runReduce'); }\n \n get autoUpdate() { return this._autoUpdate; }\n set autoUpdate(bool) {\n this.updateButton.classList.toggle(\"on\", bool);\n this.updateButton.querySelector(\"i\").classList.toggle(\"fa-spin\", bool);\n this._autoUpdate = bool;\n }\n onUpdate(evt) {\n if (evt.button === 2) this.autoUpdate = !this.autoUpdate;\n this.update();\n }\n \n log(s) {\n console.log(s)\n }\n\n /*MD ## Initialization MD*/\n\n async loadSourceFile(urlString) {\n console.log(\"LOAD \", urlString);\n this.sourceURL = urlString;\n this.sourceEditor.setURL(lively.paths.normalizePath(urlString, \"\"));\n await this.sourceEditor.loadFile();\n await this.update(); \n }\n \n async loadTransformerSourceFile(urlString) {\n console.log(\"LOAD \", urlString);\n this.transformerSourceURL = urlString;\n this.transformerSourceEditor.setURL(lively.paths.normalizePath(urlString, \"\"));\n await this.transformerSourceEditor.loadFile();\n await this.update(); \n }\n \n async initialize() {\n this.windowTitle = \"Astro View\";\n this.registerButtons();\n\n this.getAllSubmorphs(\"button\").forEach(button => {\n button.addEventListener('contextmenu', e => {\n e.preventDefault();\n e.stopPropagation();\n e.currentTarget.dispatchEvent(new MouseEvent(\"click\", {button: 2}));\n });\n });\n \n await this.sourceEditor.awaitEditor();\n await this.transformerSourceEditor.awaitEditor();\n \n this.sourceEditor.hideToolbar();\n this.astInspector.connectEditor(this.sourceEditor);\n this.sourceLCM.doSave = async () => {\n this.save();\n };\n this.transformerSourceLCM.doSave = async () => {\n this.save();\n };\n \n this.sourceEditor.livelyCodeMirror().editor.on(\"cursorActivity\", (cm) => {\n // #TODO continue here....\n console.log(cm)\n // this.selectPath(pathKeys);\n })\n \n this.debouncedUpdate = this.update::debounce(500);\n this.sourceLCM.addEventListener(\"change\", (() =>\n SyntaxChecker.checkForSyntaxErrors(this.sourceCM))::debounce(200));\n this.sourceLCM.addEventListener(\"change\", () => {\n if (this.autoUpdate) this.debouncedUpdate()\n });\n \n this.debouncedUpdateTransformer = this.updateTransformer::debounce(500);\n this.transformerSourceLCM.addEventListener(\"change\", (() => {\n // SyntaxChecker.checkForSyntaxErrors(this.transformerSourceCM))::debounce(200) \n }));\n this.transformerSourceLCM.addEventListener(\"change\", () => {\n if (this.autoUpdate) this.debouncedUpdateTransformer()\n });\n \n this.sourcePath.addEventListener(\"keyup\", evt => {\n if (evt.code == \"Enter\") this.onSourcePathEntered(this.sourcePath.value);\n });\n this.transformerSourcePath.addEventListener(\"keyup\", evt => {\n if (evt.code == \"Enter\") this.onTransformerSourcePathEntered(this.transformerSourcePath.value);\n });\n\n const source = this.getAttribute(\"source\");\n if (source) this.loadSourceFile(source);\n \n const transformerSource = this.getAttribute(\"transformerSource\");\n if (transformerSource) this.loadTransformerSourceFile(transformerSource);\n \n this.projectName = this.getAttribute(\"projectName\") || \"\";\n \n this.autoUpdate = true;\n\n this.dispatchEvent(new CustomEvent(\"initialize\"));\n }\n\n onEditorCursorActivity(cm) {\n var from = cm.getCursor(true)\n var to = cm.getCursor(false)\n \n this.get(\"#editorInfo\").textContent = `${cm.indexFromPos(from)}-${cm.indexFromPos(to)}`\n }\n \n async updateTokens() { \n let api = \"http://127.0.0.1:5000\";\n let dataset = \"d3-force-main\";\n \n this.status = \"source updated: fetching...\"\n \n try {\n this.tokens = null;\n \n let response = await fetch(`${api}/tokenize`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify({\n text: this.source\n })\n })\n \n let tokens = await response.json();\n // filter new-lines\n tokens = tokens.filter(ea => !ea.value.match(/^[ \\n]+$/));\n \n this.tokens = tokens;\n } catch (e) {\n this.status = `error fetching tokens: ${e}`;\n this.log(`error fetching tokens: ${e}`);\n }\n \n try {\n let response = await fetch(`${api}/dataset/${dataset}/embedding`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify({\n text: this.source\n }),\n })\n \n let embedding = await response.json()\n this.embedding = embedding;\n } catch (e) {\n this.log(`error fetching embedding: ${e}`);\n this.status = `error fetching embedding: ${e}`;\n }\n \n if (this.embedding) {\n let formatted = JSON.stringify(this.embedding)\n this.get('#pool_embedding').innerText = formatted\n this.get('#astro-plot').showFeature(this.embedding)\n }\n\n this.log('fetched tokens', this.tokens)\n \n if (this.tokens) {\n this.get(\"#tokens\").innerHTML = \"\"\n this.tokens.forEach((token) => {\n let tokenView = \n
\n
{token.id}
\n
this.selectToken(tokenView, token)}\n pointerenter={() => this.hoverToken(tokenView, token, true)}\n pointerleave={() => this.hoverToken(tokenView, token, false)}\n >{token.value}
\n
{token.start}-{token.end}
\n
\n this.get(\"#tokens\").appendChild(tokenView)\n })\n } else {\n this.get(\"#tokens\").innerHTML = \"Error fetching tokens\"\n this.status = `error fetching tokens`;\n }\n }\n \n selectToken(view, token) {\n if (this.selectedTokenView) this.selectedTokenView.classList.remove(\"selected\")\n view.classList.add(\"selected\")\n this.selectedTokenView = view\n \n this.get(\"#embeddings\").innerHTML = \"\"\n let rows = []\n \n let tds = Array.from(token.value)\n .map(ea => ea.charCodeAt(0))\n .map(ea => {ea})\n \n rows.push({...tds})\n \n let table = {...rows}
\n \n this.get(\"#embeddings\").appendChild(table)\n \n }\n \n \n hoverToken(view, token, active) {\n if (active) {\n const start = loc(this.sourceCM.posFromIndex(token.start));\n const end = loc(this.sourceCM.posFromIndex(token.end));\n this.hoverMarker = this.sourceCM.markText(start.asCM(), end.asCM(), {css: \"background-color: #fe3\"});\n } else {\n this.hoverMarker.clear();\n this.hoverMarker = null;\n }\n }\n \n /*MD ## Execution MD*/\n \n async update() {\n this.lastSource = this.source\n this.log(\"source code changed, length: \" + this.source.length + \"\")\n \n try {\n var node = await this.astInspector.treeSitterParse(this.source)\n this.treeSitterRootNode = node.rootNode\n this.astInspector.inspect(this.treeSitterRootNode);\n } catch (e) {\n this.astInspector.inspect({Error: e.message});\n }\n \n this.updateTokens();\n }\n \n async updateTransformer() {\n this.status = \"transformer: sending...\"\n try {\n let response = await fetch(`${this.api}/dataset/${this.projectName}/transformer`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify({\n text: this.transformerSource\n }),\n })\n response = await response.json();\n if (response.error) throw new Error(response.error);\n this.status = \"transformer: using \" + response.transformer;\n } catch (e) {\n this.status = \"transformer: \" + e;\n }\n }\n \n async onRunParse() {\n this.status = \"parser: running...\"\n try {\n let language = \"typescript\";\n let response = await fetch(`${this.api}/dataset/${this.projectName}/parse?language=${language}`, {\n method: 'POST',\n })\n response = await response.json();\n if (response.error) throw new Error(response.error);\n this.status = \"parser: currently \" + response.ASTs + \" ASTs in memory \";\n } catch (e) {\n this.status = \"parser: \" + e;\n }\n }\n \n async onRunQuery() {\n this.status = \"query: running...\"\n try {\n let response = await fetch(`${this.api}/dataset/${this.projectName}/run_query`, {\n method: 'POST',\n })\n response = await response.json();\n if (response.error) throw new Error(response.error);\n this.status = \"query: matched \" + response.matches + \" items in \" + response.files + \" files\";\n } catch (e) {\n this.status = \"query: \" + e;\n }\n }\n \n async onRunMap() {\n this.status = \"map: running...\"\n try {\n let response = await fetch(`${this.api}/dataset/${this.projectName}/run_map`, {\n method: 'POST',\n })\n response = await response.json();\n if (response.error) throw new Error(response.error);\n this.status = \"map: success. Data columns: \" + response.columns;\n } catch (e) {\n this.status = \"map: \" + e;\n }\n }\n \n async onRunReduce() {\n this.status = \"reduce: running...\"\n try {\n let response = await fetch(`${this.api}/dataset/${this.projectName}/run_reduce`, {\n method: 'POST',\n })\n response = await response.json();\n if (response.error) throw new Error(response.error);\n this.status = \"reduce: success\";\n } catch (e) {\n this.status = \"reduce: \" + e;\n }\n }\n \n async onRunUmap() {\n this.status = \"umap: running...\"\n let data;\n try {\n let response = await fetch(`${this.api}/dataset/${this.projectName}/run_umap`, {\n method: 'POST',\n })\n response = await response.json();\n if (response.error) throw new Error(response.error);\n data = response.umap;\n this.status = \"umap: success\";\n } catch (e) {\n this.status = \"umap: \" + e;\n return;\n }\n debugger;\n \n try {\n this.astroPlot.displayData(data)\n } catch (e) {\n this.status = \"plot: \" + e;\n }\n }\n\n async save() {\n if (this.sourceURL) {\n await this.sourceEditor.saveFile();\n }\n if (this.transformerSourceURL) {\n await this.transformerSourceEditor.saveFile();\n }\n this.update();\n }\n\n /*MD ## Lively Integration MD*/\n\n livelyPrepareSave() {\n this.setAttribute('source', this.sourceURL);\n this.setAttribute('transformerSource', this.transformerSourceURL);\n this.setAttribute('projectName', this.projectName);\n \n console.log(\"PREPARE SAVE (AST Explorer)\");\n }\n \n livelyMigrate(other) {\n }\n\n async livelyExample() {\n await this.loadSourceFile(AstroView.defaultSourceURL);\n await this.loadTransformerSourceFile(AstroView.defaultTransformerSourceURL)\n }\n}"} \ No newline at end of file