Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sandevh/stu 307 generate tests with an llm #1122

Open
wants to merge 14 commits into
base: main
Choose a base branch
from
Prev Previous commit
Next Next commit
name tests
izi-on committed Mar 2, 2024

Verified

This commit was signed with the committer’s verified signature.
commit c7194c4a53bbdcf520628ffe23fcfec01608fedd
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@
# python
/venv
/venv2
/gen_tests

# dependencies
/node_modules
2 changes: 2 additions & 0 deletions captain/routes/test_sequence.py
Original file line number Diff line number Diff line change
@@ -68,6 +68,8 @@ async def generate_test_endpoint(requested_test: GenerateTestRequest):
test_type = requested_test.test_type
prompt = requested_test.prompt
await generate_test(test_name, test_type, prompt, test_container)
if "test" not in test_container:
return
return TestGenerationContainer(test=test_container["test"]).model_dump_json(
by_alias=True
)
14 changes: 8 additions & 6 deletions captain/utils/test_sequencer/generator/generate_test.py
Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@ async def generate_test(
messages=[
{
"role": "system",
"content": "You generate only raw python code for tests using assertions. If the user's request does not imply a test, simply output NULL",
"content": "You generate only raw python code for tests using assertions. If the user's request does not imply a test, simply output 'NULL'",
},
{
"role": "user",
@@ -34,18 +34,20 @@ async def generate_test(
if code_resp is None:
raise Exception("Unable to generate code")
if code_resp == "NULL":
raise Exception("Unable to generate test")
raise Exception("Invalid prompt/request, please specify something to test")
code = "\n".join(code_resp.splitlines()[1:-1])
filename = test_name
with open(filename, "w") as file:
path_to_gen_folder = os.path.join(os.getcwd(), "gen_tests")
path_to_file = os.path.join(path_to_gen_folder, test_name)
if not os.path.exists(path_to_gen_folder):
os.makedirs(path_to_gen_folder)
with open(path_to_file, "w") as file:
file.write(code)
path_to_file = os.path.join(os.getcwd(), filename)
test_container["test"] = Test.construct(
type="test",
id=uuid.uuid4(),
group_id=uuid.uuid4(),
path=path_to_file,
test_name=filename,
test_name=test_name,
run_in_parallel=False,
test_type=TestTypes.python,
status=StatusTypes.pending,
Original file line number Diff line number Diff line change
@@ -339,7 +339,16 @@ export function DataTable() {
const getSpecificContextMenuItems = (row: Row<TestSequenceElement>) => {
switch (row.original.type) {
case "test":
return <></>;
return (
<ContextMenuItem
onClick={() => {
setOpenPyTestFileModal(true);
setTestToDisplay(row.original as Test);
}}
>
Consult Code
</ContextMenuItem>
);
case "conditional":
return (
<>
@@ -472,16 +481,6 @@ export function DataTable() {
>
Remove Test
</ContextMenuItem>
{row.original.type === "test" && (
<ContextMenuItem
onClick={() => {
setOpenPyTestFileModal(true);
setTestToDisplay(row.original as Test);
}}
>
Consult Code
</ContextMenuItem>
)}
</ContextMenuContent>
</ContextMenu>
))
Original file line number Diff line number Diff line change
@@ -8,27 +8,32 @@ import {
} from "../../../models/drag_and_drop";
import { useTestSequencerState } from "@/renderer/hooks/useTestSequencerState";

export const GeneratedTest = ({ row }: { row: Row<Test> }) => {
const { setElems } = useTestSequencerState();
const [, drag] = useDrag(() => ({
type: ItemTypes.TestElementRow,
item: { row: row },
end: (item, monitor) => {
const dropResult = monitor.getDropResult<TestSequenceDropResult>();
if (item && dropResult) {
setElems((elems) => {
const newElems = [...elems];
newElems.splice(dropResult.targetIdx, 0, item.row.original);
return newElems;
});
}
},
}));
export const GeneratedTest = ({ row, ...props }: { row: Row<Test> }) => {
const { elems, setElems } = useTestSequencerState();
const [, drag] = useDrag(
() => ({
type: ItemTypes.TestElementRow,
item: { row: row },
end: (item, monitor) => {
const dropResult = monitor.getDropResult<TestSequenceDropResult>();
if (item && dropResult) {
setElems((elems) => {
const newElems = [...elems];
newElems.splice(dropResult.targetIdx, 0, item.row.original);
return newElems;
});
}
},
}),
[elems],
);
return (
<TableRow
className="relative"
key={row.id}
data-state={row.getIsSelected() && "selected"}
ref={drag}
{...props}
>
{row.getVisibleCells().map((cell) => (
<TableCell key={cell.id}>
Original file line number Diff line number Diff line change
@@ -34,6 +34,14 @@ import { GeneratedTest } from "./GeneratedTest";
// } from "@/renderer/components/ui/command";
// import { cn } from "@/renderer/lib/utils";
import { baseClient } from "@/renderer/lib/base-client";
import { atomWithImmer } from "jotai-immer";
import { useAtom } from "jotai";
import {
ContextMenu,
ContextMenuContent,
ContextMenuItem,
ContextMenuTrigger,
} from "@/renderer/components/ui/context-menu";

export const columns: ColumnDef<Test>[] = [
{
@@ -106,9 +114,10 @@ export const columns: ColumnDef<Test>[] = [
// </Popover>
// );
// };
const generatedTestsAtom = atomWithImmer<Test[]>([]);

export const TestGeneratorPanel = () => {
const [data, setData] = useState<Test[]>([] as Test[]);
const [data, setData] = useAtom(generatedTestsAtom);
const table = useReactTable({
data,
columns,
@@ -129,6 +138,10 @@ export const TestGeneratorPanel = () => {
return [...prevData, { ...data_obj.test }];
});
}
const handleRemoveTest = (testId: string) => {
setData((data) => data.filter((elem) => elem.id !== testId));
};
// const handleConsultCode =
return (
<div className="flex w-full flex-col space-y-4">
<Label htmlFor="testName">Enter test name:</Label>
@@ -177,7 +190,29 @@ export const TestGeneratorPanel = () => {
</TableHeader>
<TableBody>
{table.getRowModel().rows?.length ? (
table.getRowModel().rows.map((row) => <GeneratedTest row={row} />)
table.getRowModel().rows.map((row) => (
<ContextMenu>
<ContextMenuTrigger asChild>
<GeneratedTest
row={row}
key={row.id}
data-state={row.getIsSelected() && "selected"}
/>
</ContextMenuTrigger>
<ContextMenuContent>
<ContextMenuItem
onClick={() => handleRemoveTest(row.original.id)}
>
Remove test
</ContextMenuItem>
<ContextMenuItem
// onClick={() => handleConsultCode(row.original.id)}
>
Consult Code
</ContextMenuItem>
</ContextMenuContent>
</ContextMenu>
))
) : (
<TableRow>
<TableCell colSpan={columns.length} className="text-center">