Skip to content

Commit

Permalink
more improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@greenmtnboy
greenmtnboy committed May 3, 2024
1 parent b6cd78e commit 4f47869
Show file tree
Hide file tree
Showing 9 changed files with 171 additions and 103 deletions.
1 change: 1 addition & 0 deletions backend/io_models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ class UIConcept(BaseModel):
class Model(BaseModel):
name: str
concepts: List[UIConcept]
rendered: str | None = None


class ListModelResponse(BaseModel):
Expand Down
56 changes: 19 additions & 37 deletions backend/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,15 @@
from google.cloud import bigquery
from google.oauth2 import service_account
from preql.constants import DEFAULT_NAMESPACE
from preql.core.enums import DataType, Purpose
from preql.core.enums import Purpose
from preql.core.models import (
ProcessedQuery,
ProcessedQueryPersist,
ProcessedShowStatement,
ShowStatement,
Select,
Persist,
DataType,
)
from preql import Environment, Executor, Dialects
from preql.parser import parse_text
Expand All @@ -46,7 +47,6 @@
from backend.io_models import (
ListModelResponse,
Model,
UIConcept,
GenAIConnectionInSchema,
QueryInSchema,
GenAIQueryInSchema,
Expand All @@ -59,7 +59,7 @@
ConnectionListItem,
ConnectionListOutput,
)
from backend.models.helpers import flatten_lineage
from backend.models.helpers import model_to_response
from sqlalchemy_bigquery import * # this is for pyinstaller

from sqlalchemy_bigquery.base import BigQueryDialect
Expand All @@ -82,7 +82,7 @@

STATEMENT_LIMIT = 100

PARSE_DEPENDENCY_RESOLUTION_ATTEMPTS = 50
PARSE_DEPENDENCY_RESOLUTION_ATTEMPTS = 10

app = FastAPI()

Expand Down Expand Up @@ -144,10 +144,6 @@ class InstanceSettings:

GENAI_CONNECTIONS: Dict[str, NLPEngine] = {}

## BEGIN REQUESTS


## Begin Endpoints
router = APIRouter()


Expand Down Expand Up @@ -194,44 +190,31 @@ def parse_env_from_full_model(input: ModelInSchema) -> Environment:
raise ValueError(
f"unable to parse input models after {attempts} attempts; "
f"successfully parsed {parsed.keys()}; error was {str(exception)},"
"have {[c.address for c in env.concepts.values()]}"
f"have {[c.address for c in env.concepts.values()]}"
)

return env


## Begin Endpoints
@router.get("/models", response_model=ListModelResponse)
async def get_models() -> ListModelResponse:
models = []
for key, value in public_models.items():
value = public_models[key]
final_concepts = []
for skey, sconcept in value.concepts.items():
# don't show private concepts
if sconcept.name.startswith("_"):
continue
final_concepts.append(
UIConcept(
name=(
sconcept.name.split(".")[-1]
if sconcept.namespace == DEFAULT_NAMESPACE
else sconcept.name
),
datatype=sconcept.datatype,
purpose=sconcept.purpose,
description=(
sconcept.metadata.description if sconcept.metadata else None
),
namespace=sconcept.namespace,
key=skey,
lineage=flatten_lineage(sconcept, depth=0),
)
)
final_concepts.sort(key=lambda x: x.namespace + x.key)
models.append(Model(name=key, concepts=final_concepts))
models.append(model_to_response(name=key, env=value))
return ListModelResponse(models=models)


@router.get("/model/{model}", response_model=Model)
async def get_model(model: str) -> Model:
if model in CONNECTIONS:
connection = CONNECTIONS[model]
return model_to_response(model, connection.environment, True)
if model not in public_models:
raise HTTPException(404, f"model {model} not found")
return model_to_response(model, public_models[model], True)


@router.get("/connections")
async def list_connections():
output = []
Expand Down Expand Up @@ -425,7 +408,6 @@ def run_query(query: QueryInSchema):
]
sql = executor.generator.generate_queries(executor.environment, parsed)
except Exception as e:
print(e)
raise HTTPException(status_code=422, detail="Parsing error: " + str(e))
# execution errors should be 500
try:
Expand Down Expand Up @@ -453,7 +435,7 @@ def run_query(query: QueryInSchema):
else col.address.replace(".", "_")
),
purpose=col.purpose,
datatype=col.datatype,
datatype=col.datatype.data_type,
),
)
for col in statement.output_columns
Expand All @@ -471,7 +453,7 @@ def run_query(query: QueryInSchema):
else col.address.replace(".", "_")
),
purpose=col.purpose,
datatype=col.datatype,
datatype=col.datatype.data_type,
),
)
for col in statement.output_columns
Expand Down
37 changes: 35 additions & 2 deletions backend/models/helpers.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from typing import Any, List, Union

from preql.core.enums import DataType
from preql.core.models import (
AggregateWrapper,
Comparison,
Expand All @@ -9,9 +8,13 @@
FilterItem,
Function,
WindowItem,
Environment,
DataType,
)

from backend.io_models import LineageItem
from backend.io_models import LineageItem, Model, UIConcept
from preql.constants import DEFAULT_NAMESPACE
from preql.parsing.render import Renderer


def flatten_array(input: Any, depth: int = 0) -> List[LineageItem]:
Expand Down Expand Up @@ -80,3 +83,33 @@ def flatten_lineage(
chain += [LineageItem(token="<-", depth=depth)]
chain += flatten_lineage(input.lineage, depth + 1)
return chain


def model_to_response(
name: str, env: Environment, render_to_text: bool = False
) -> Model:
final_concepts = []
rendered = Renderer().to_string(env) if render_to_text else None
for skey, sconcept in env.concepts.items():
# don't show private concepts
if sconcept.name.startswith("_"):
continue
final_concepts.append(
UIConcept(
name=(
sconcept.name.split(".")[-1]
if sconcept.namespace == DEFAULT_NAMESPACE
else sconcept.name
),
datatype=sconcept.datatype,
purpose=sconcept.purpose,
description=(
sconcept.metadata.description if sconcept.metadata else None
),
namespace=sconcept.namespace,
key=skey,
lineage=flatten_lineage(sconcept, depth=0),
)
)
final_concepts.sort(key=lambda x: x.namespace + x.key)
return Model(name=name, concepts=final_concepts, rendered=rendered)
1 change: 1 addition & 0 deletions backend/scripts/hit_local.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
print(response.status_code)
print(response.text)


response = requests.post(
"http://0.0.0.0:5678/query",
data=QueryInSchema(
Expand Down
86 changes: 45 additions & 41 deletions frontend/src/components/editor/EditorEditor.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import { defineComponent } from 'vue';
import { mapGetters, mapActions } from 'vuex';
import editorMap from '/src/store/modules/monaco';
import axiosHelpers from '/src/api/helpers.ts';
import instance from '/src/api/instance';
import { Editor, RawEditor } from '/src/models/Editor'
import * as monaco from 'monaco-editor';
Expand Down Expand Up @@ -70,60 +69,51 @@ export default defineComponent({
},
methods: {
...mapActions(['saveEditors', 'saveEditorText', 'connectConnection', 'addMonacoEditor',
'setConnectionInactive', 'addHistory','setEditorError']),
async submitGenAI(selection:String) {
...mapActions(['saveEditors', 'saveEditorText', 'formatEditorText', 'connectConnection', 'addMonacoEditor',
'setConnectionInactive', 'addHistory', 'setEditorError']),
async submitGenAI(selection: String) {
this.info = 'Generating PreQL query from prompt...'
let response = await this.editorData.runGenAIQuery(this.$store,this.activeGenAIConnection.name, selection);
let response = await this.editorData.runGenAIQuery(this.$store, this.activeGenAIConnection.name, selection);
return response
},
async submit() {
// this.loading = true;
this.info = 'Executing query...'
const start = new Date()
// todo: set this in a action
this.editorData.loading = true;
// this.error = null;
let current_query = this.editorData.contents;
if (!this.connection) {
this.setEditorError({ name: this.editorData.name, error: 'No connection selected for this editor.' })
return
// throw new Error('No connection selected for this editor')
}
// invalidation of logic
if (!this.connection.active) {
await this.connectConnection(this.connection)
try {
await this.connectConnection(this.connection)
}
catch (error) {
this.setEditorError({ name: this.editorData.name, error: 'Error with connection.' })
}
}
await this.editorData.runQuery(this.$store);
this.addHistory({
this.addHistory({
connection: this.editorData.connection,
text: this.editorData.contents,
editor: this.editorData.name,
timestamp: start,
duration: this.editorData.duration,
executed: this.editorData.executed,
text: this.editorData.contents,
editor: this.editorData.name,
timestamp: start,
duration: this.editorData.duration,
executed: this.editorData.executed,
error: this.editorData.error
})
this.last_passed_query_text = current_query;
},
async format() {
this.loading = true;
this.info = 'Formatting query...'
this.error = null;
var self = this;
await instance.post('format_query', { model: this.query.model, query: this.editorData.contents, id: this.query.id }).then(function (resp) {
self.query.query = resp.data.new_text;
self.loading = false;
}).catch((error) => {
self.error = axiosHelpers.getErrorMessage(error);
self.loading = false;
})
},
querySaveComplete() {
this.$emit('querySaveComplete')
},
clearPrompt() {
this.prompt = '';
},
createEditor() {
let editorElement = document.getElementById('editor')
if (!editorElement) {
Expand Down Expand Up @@ -187,25 +177,40 @@ export default defineComponent({
}
// run our async call
Promise.all([this.submitGenAI(editor.getModel()?.getValueInRange(range))]).then(() => {
var op = {range: range, text: this.editorData.generated_sql, forceMoveMarkers: true};
var op = { range: range, text: this.editorData.generated_sql, forceMoveMarkers: true };
editor.executeEdits("gen-ai-prompt-shortcut", [op]);
this.saveEditors()
this.editorData.runQuery(this.$store);
}).catch((error) => {
this.setEditorError({name:this.editorData.name, error: axiosHelpers.getErrorMessage(error)});
this.setEditorError({ name: this.editorData.name, error: axiosHelpers.getErrorMessage(error) });
}
)
}
});
let editorData = this.editorData;
editor.addAction({
id: 'format-preql',
label: 'Format PreQL',
keybindings: [monaco.KeyMod.CtrlCmd | monaco.KeyMod.Shift | monaco.KeyCode.KeyI],
run: function () {
console.log('FORMAT SOMETHING')
editorData.formatText(editor.getValue()).then((response) => {
editor.setValue(response)
})
}
});
editor.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, () => {
this.saveEditorText({ contents: editor.getValue(), name: this.editorData.name,
connection:this.editorData.connection }).then( ()=> {
this.saveEditorText({
contents: editor.getValue(), name: this.editorData.name,
connection: this.editorData.connection
}).then(() => {
this.saveEditors()
}).catch((error) => {
this.setEditorError({name:this.editorData.name, error: axiosHelpers.getErrorMessage(error)});
this.setEditorError({ name: this.editorData.name, error: axiosHelpers.getErrorMessage(error) });
})
});
// loader.init().then((monaco) => {
Expand Down Expand Up @@ -238,5 +243,4 @@ export default defineComponent({
}
}
})
</script>

</script>
5 changes: 4 additions & 1 deletion frontend/src/components/editor/HintsComponent.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,10 @@ export default {
name: 'Find and Replace',
keys: ['H']
},
{
name: 'Format',
keys: ['Shift', 'I']
},
// {
// name: 'Format',
// keys: ['B']
Expand All @@ -108,7 +112,6 @@ export default {
},
},
mounted() {
console.log(this.$store.getters.genAIConnections)
},
computed: {
Expand Down
3 changes: 1 addition & 2 deletions frontend/src/components/sidebar/ModelManager.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
</v-expansion-panel-title>
<v-expansion-panel-text class="px-0">
<template v-if="activeModelFromConnection">
<v-list-item @click="setActiveEditor(editor.name)" v-for="editor in editors"
<v-list-item @click="() =>setActiveEditor(editor.editor)" v-for="editor in editors"
class="editor-list">
{{ editor.editor }} ({{ editor.alias }})
</v-list-item>
Expand Down Expand Up @@ -160,7 +160,6 @@ export default {
},
editors() {
console.log(this.activeModelFromConnection)
if (!this.activeModelFromConnection) {
return []
}
Expand Down
Loading

0 comments on commit 4f47869

Please sign in to comment.