diff --git a/helicone-node/api/generatedTypes/public.ts b/helicone-node/api/generatedTypes/public.ts index c048f53bcc..1dee447d84 100644 --- a/helicone-node/api/generatedTypes/public.ts +++ b/helicone-node/api/generatedTypes/public.ts @@ -293,15 +293,6 @@ Json: JsonObject; status?: components["schemas"]["Partial_NumberOperators_"]; model?: components["schemas"]["Partial_TextOperators_"]; }; - /** @description Make all properties in T optional */ - Partial_VectorOperators_: { - contains?: string; - }; - /** @description Make all properties in T optional */ - Partial_RequestResponseSearchToOperators_: { - request_body_vector?: components["schemas"]["Partial_VectorOperators_"]; - response_body_vector?: components["schemas"]["Partial_VectorOperators_"]; - }; /** @description From T, pick a set of properties whose keys are in the union K */ "Pick_FilterLeaf.feedback-or-request-or-response-or-properties-or-values-or-request_response_search_": { feedback?: components["schemas"]["Partial_FeedbackTableToOperators_"]; @@ -313,7 +304,7 @@ Json: JsonObject; values?: { [key: string]: components["schemas"]["Partial_TextOperators_"]; }; - request_response_search?: components["schemas"]["Partial_RequestResponseSearchToOperators_"]; + request_response_search: unknown; }; "FilterLeafSubset_feedback-or-request-or-response-or-properties-or-values-or-request_response_search_": components["schemas"]["Pick_FilterLeaf.feedback-or-request-or-response-or-properties-or-values-or-request_response_search_"]; RequestFilterNode: components["schemas"]["FilterLeafSubset_feedback-or-request-or-response-or-properties-or-values-or-request_response_search_"] | components["schemas"]["RequestFilterBranch"] | "all"; diff --git a/valhalla/jawn/src/controllers/private/datasetController.ts b/valhalla/jawn/src/controllers/private/datasetController.ts index 92a5a010e1..5eabd1786e 100644 --- a/valhalla/jawn/src/controllers/private/datasetController.ts +++ b/valhalla/jawn/src/controllers/private/datasetController.ts @@ -13,7 +13,7 @@ import { import { supabaseServer } from "../../lib/routers/withAuth"; import { FilterNode } from "../../lib/shared/filters/filterDefs"; import { getRequests } from "../../lib/stores/request/request"; -import { FineTuningManager } from "../../managers/FineTuningManager"; +import { FineTuningManager } from "../../managers/finetuning/FineTuningManager"; import { JawnAuthenticatedRequest } from "../../types/request"; import { postHogClient } from "../../lib/clients/postHogClient"; diff --git a/valhalla/jawn/src/controllers/private/fineTuneController.ts b/valhalla/jawn/src/controllers/private/fineTuneController.ts index 16980f59a6..60cbfd0953 100644 --- a/valhalla/jawn/src/controllers/private/fineTuneController.ts +++ b/valhalla/jawn/src/controllers/private/fineTuneController.ts @@ -13,15 +13,76 @@ import { } from "tsoa"; import { postHogClient } from "../../lib/clients/postHogClient"; import { supabaseServer } from "../../lib/routers/withAuth"; -import { getRequests } from "../../lib/stores/request/request"; -import { FineTuningManager } from "../../managers/FineTuningManager"; +import { getRequests, fetchBodies } from "../../lib/stores/request/request"; +import { FineTuningManager } from "../../managers/finetuning/FineTuningManager"; import { JawnAuthenticatedRequest } from "../../types/request"; // src/users/usersController.ts import { hasAccessToFineTune } from "./datasetController"; +import { + NewFineTuningManager, + IFineTuningJob, + IFineTuningJobEvent, +} from "../../managers/finetuning/NewFineTuningManager"; +import { Result, err, ok } from "../../lib/shared/result"; +import { DatasetManager } from "../../managers/dataset/DatasetManager"; export interface FineTuneBody { providerKeyId: string; + datasetId?: string; +} + +async function prepareFineTuneRequest( + request: FineTuneBody, + authParams: JawnAuthenticatedRequest["authParams"] +): Promise< + Result<{ fineTuneManager: NewFineTuningManager; datasetId: string }, string> +> { + const datasetManager = new DatasetManager(authParams); + let datasetId = request.datasetId; + if (!request.datasetId) { + const { data: requests, error: requestsError } = await getRequests( + authParams.organizationId, + "all", + 0, + 1000, + {} + ); + if (!requests || requests.length === 0) { + return err("No requests found"); + } + const newDataset = await datasetManager.addDataset({ + requestIds: requests.map((r) => r.request_id), + datasetName: "Automated Dataset", + }); + if (newDataset.error || !newDataset.data) { + return err(newDataset.error || "Failed to create dataset"); + } + datasetId = newDataset.data; + } + + const { data: key, error: keyError } = await supabaseServer.client + .from("decrypted_provider_keys") + .select("decrypted_provider_key") + .eq("id", request.providerKeyId) + .eq("org_id", authParams.organizationId) + .single(); + if (keyError || !key || !key.decrypted_provider_key) { + return { + error: "No Provider Key found", + data: null, + }; + } + return ok(new FineTuningManager(request.providerKeyId)); +} + +export interface NewFineTuneJob { + fineTuneJobId: string; +} + +interface FineTuneJobStats { + job: IFineTuningJob; + events: IFineTuningJobEvent[]; } @Route("/v1/fine-tune") @@ -33,108 +94,32 @@ export class FineTuneMainController extends Controller { @Body() body: FineTuneBody, @Request() request: JawnAuthenticatedRequest - ): Promise< - | { - error: string; - } - | { - success: boolean; - data: { - fineTuneJob: string; - url: string; - }; - } - > { - if (!(await hasAccessToFineTune(request.authParams.organizationId))) { - this.setStatus(403); - return { - error: "You do not have access to fine tune", - }; - } - const filter = "all"; - const { providerKeyId } = body; - const metrics = await getRequests( - request.authParams.organizationId, - filter, - 0, - 1000, - {} - ); - if (metrics.error || !metrics.data || metrics.data.length === 0) { - this.setStatus(500); - return { - error: "No requests found", - }; - } - const { data: key, error: keyError } = await supabaseServer.client - .from("decrypted_provider_keys") - .select("decrypted_provider_key") - .eq("id", providerKeyId) - .eq("org_id", request.authParams.organizationId) - .single(); - if (keyError || !key || !key.decrypted_provider_key) { + ): Promise> { + const { data: fineTuningObjects, error: requestBuildingError } = + await prepareFineTuneRequest(datasetManager, body, request.authParams); + + if (requestBuildingError || !fineTuningObjects) { this.setStatus(500); - return { - error: "No Provider Key found", - }; + return err(requestBuildingError); } - const fineTuningManager = new FineTuningManager(key.decrypted_provider_key); + + const { fineTuneManager, datasetId } = fineTuningObjects; + try { - const fineTuneJob = await fineTuningManager.createFineTuneJob( - metrics.data, - "model", - "suffix" - ); - if (fineTuneJob.error || !fineTuneJob.data) { - this.setStatus(500); - return { - error: "Failed to create fine tune job", - }; - } - const url = `https://platform.openai.com/finetune/${fineTuneJob.data.id}?filter=all`; - Sentry.captureMessage( - `fine-tune job created - ${fineTuneJob.data.id} - ${request.authParams.organizationId}` - ); - postHogClient?.capture({ - distinctId: `${fineTuneJob.data.id}-${request.authParams.organizationId}`, - event: "fine_tune_job", - properties: { - id: fineTuneJob.data.id, - success: true, - org_id: request.authParams.organizationId, - }, + const fineTuneJob = await fineTuneManager.createFineTuneJob(datasetId, { + _type: "OpenAIFineTuneJobOptions", + model: "gpt-3.5", }); - const dataset = await supabaseServer.client - .from("finetune_dataset") - .insert({ - name: `Automated Dataset for ${fineTuneJob.data.id}`, - filters: JSON.stringify([]), - organization_id: request.authParams.organizationId, - }) - .select("*") - .single(); - if (dataset.error || !dataset.data) { + + if (fineTuneJob.error || !fineTuneJob.data) { this.setStatus(500); - return { - error: dataset.error.message, - }; + return err("Failed to create fine tune job"); } - const fineTunedJobId = await supabaseServer.client - .from("finetune_job") - .insert({ - dataset_id: dataset.data.id, - finetune_job_id: fineTuneJob.data.id, - provider_key_id: providerKeyId, - status: "created", - organization_id: request.authParams.organizationId, - }) - .select("*") - .single(); + return { success: true, data: { fineTuneJob: fineTunedJobId.data?.id ?? "", - url: url, }, }; } catch (e) { @@ -159,15 +144,7 @@ export class FineTuneMainController extends Controller { public async fineTuneJobStats( @Path() jobId: string, @Request() request: JawnAuthenticatedRequest - ): Promise< - | { - error: string; - } - | { - job: any; - events: any; - } - > { + ): Promise> { const { data: fineTuneJob, error: fineTuneJobError } = await supabaseServer.client .from("finetune_job") diff --git a/valhalla/jawn/src/lib/shared/filters/filterDefs.ts b/valhalla/jawn/src/lib/shared/filters/filterDefs.ts index 7006c084ab..03c5733251 100644 --- a/valhalla/jawn/src/lib/shared/filters/filterDefs.ts +++ b/valhalla/jawn/src/lib/shared/filters/filterDefs.ts @@ -299,6 +299,13 @@ type ExperimentHypothesisRunToOperator = { export type ExperimentHypothesisRunScoreValue = SingleKey; +export type ExperimentDatasetV2RowToOperators = { + dataset_id: SingleKey; +}; + +export type FilterLeafExperimentDatasetV2Row = + SingleKey; + export type TablesAndViews = { user_metrics: FilterLeafUserMetrics; user_api_keys: FilterLeafUserApiKeys; @@ -311,6 +318,7 @@ export type TablesAndViews = { experiment: FilterLeafExperiment; experiment_hypothesis_run: ExperimentHypothesisRunScoreValue; score_value: FilterLeafScoreValue; + experiment_dataset_v2_row: FilterLeafExperimentDatasetV2Row; request_response_search: FilterLeafRequestResponseSearch; // CLICKHOUSE TABLES diff --git a/valhalla/jawn/src/lib/shared/filters/filters.ts b/valhalla/jawn/src/lib/shared/filters/filters.ts index 3c326b7e56..ebacaa0ce3 100644 --- a/valhalla/jawn/src/lib/shared/filters/filters.ts +++ b/valhalla/jawn/src/lib/shared/filters/filters.ts @@ -264,6 +264,9 @@ const whereKeyMappings: KeyMappings = { experiment_hypothesis_run: easyKeyMappings<"experiment_hypothesis_run">({ result_request_id: "experiment_v2_hypothesis_run.result_request_id", }), + experiment_dataset_v2_row: easyKeyMappings<"experiment_dataset_v2_row">({ + dataset_id: "experiment_dataset_v2_row.dataset_id", + }), // Deprecated values: NOT_IMPLEMENTED, @@ -305,6 +308,7 @@ const havingKeyMappings: KeyMappings = { prompt_v2: NOT_IMPLEMENTED, prompts_versions: NOT_IMPLEMENTED, experiment: NOT_IMPLEMENTED, + experiment_dataset_v2_row: NOT_IMPLEMENTED, // Deprecated values: NOT_IMPLEMENTED, diff --git a/valhalla/jawn/src/lib/stores/request/request.ts b/valhalla/jawn/src/lib/stores/request/request.ts index 267008459f..2f29e495ee 100644 --- a/valhalla/jawn/src/lib/stores/request/request.ts +++ b/valhalla/jawn/src/lib/stores/request/request.ts @@ -99,6 +99,12 @@ function addJoinQueries(joinQuery: string, filter: FilterNode): string { left join score_value ON request.id = score_value.request_id`; } + if (JSON.stringify(filter).includes("experiment_dataset_v2_row")) { + joinQuery += ` + left join prompt_input_record on request.id = prompt_input_record.source_request + left join experiment_dataset_v2_row on prompt_input_record.id = experiment_dataset_v2_row.input_record`; + } + if (JSON.stringify(filter).includes("request_response_search")) { joinQuery += ` left join request_response_search on request.id = request_response_search.request_id`; @@ -106,6 +112,25 @@ function addJoinQueries(joinQuery: string, filter: FilterNode): string { return joinQuery; } +export async function fetchBodies( + requests: HeliconeRequest[] +): Promise { + const getAllBodies = requests.map(async (request) => { + if (!request.signed_body_url) { + throw new Error("No signed body url"); + } + const bodies = await (await fetch(request.signed_body_url)).json(); + + return { + ...request, + request_body: bodies.request_body, + response_body: bodies.response_body, + }; + }); + + return Promise.all(getAllBodies); +} + export async function getRequests( orgId: string, filter: FilterNode, diff --git a/valhalla/jawn/src/managers/dataset/DatasetManager.ts b/valhalla/jawn/src/managers/dataset/DatasetManager.ts index fe04ff8c48..3897af3cb2 100644 --- a/valhalla/jawn/src/managers/dataset/DatasetManager.ts +++ b/valhalla/jawn/src/managers/dataset/DatasetManager.ts @@ -50,6 +50,25 @@ export class DatasetManager extends BaseManager { return result; } + private async ensureInputRecordsExist( + requestIds: string[] + ): Promise> { + const res = await dbExecute( + ` + INSERT INTO prompt_input_record (source_request, inputs, prompt_version) + SELECT id, '{}'::jsonb, null + FROM request + WHERE id = ANY($1) + ON CONFLICT DO NOTHING + `, + [requestIds] + ); + if (res.error) { + return err(res.error); + } + return ok(null); + } + async addDataset(params: NewDatasetParams): Promise> { const dataset = await supabaseServer.client .from("experiment_dataset_v2") @@ -65,6 +84,14 @@ export class DatasetManager extends BaseManager { return err(dataset.error.message); } + const ensureInputRecords = await this.ensureInputRecordsExist( + params.requestIds + ); + + if (ensureInputRecords.error) { + return err(ensureInputRecords.error); + } + const res = await dbExecute( ` INSERT INTO experiment_dataset_v2_row (dataset_id, input_record) diff --git a/valhalla/jawn/src/managers/FineTuningManager.ts b/valhalla/jawn/src/managers/finetuning/FineTuningManager.ts similarity index 95% rename from valhalla/jawn/src/managers/FineTuningManager.ts rename to valhalla/jawn/src/managers/finetuning/FineTuningManager.ts index 7c817f0517..1af34c8aa9 100644 --- a/valhalla/jawn/src/managers/FineTuningManager.ts +++ b/valhalla/jawn/src/managers/finetuning/FineTuningManager.ts @@ -2,12 +2,12 @@ import { FineTuningJob, FineTuningJobEventsPage, } from "openai/resources/fine-tuning/jobs"; -import { OpenAIClient } from "../lib/clients/OpenAIClient"; -import { HeliconeRequest } from "../lib/stores/request/request"; -import { Result, err, ok } from "../lib/shared/result"; +import { OpenAIClient } from "../../lib/clients/OpenAIClient"; +import { HeliconeRequest } from "../../lib/stores/request/request"; +import { Result, err, ok } from "../../lib/shared/result"; import crypto from "crypto"; import fs from "fs"; -import { chatCompletionMessage } from "./types"; +import { chatCompletionMessage } from "../types"; import { ChatCompletionMessageParam } from "openai/resources"; import { ChatCompletionCreateParamsBase } from "openai/resources/chat/completions"; import OpenAI from "openai"; diff --git a/valhalla/jawn/src/managers/finetuning/NewFineTuningManager.ts b/valhalla/jawn/src/managers/finetuning/NewFineTuningManager.ts new file mode 100644 index 0000000000..0695e474a5 --- /dev/null +++ b/valhalla/jawn/src/managers/finetuning/NewFineTuningManager.ts @@ -0,0 +1,142 @@ +import { + FineTuningJob, + FineTuningJobEventsPage, +} from "openai/resources/fine-tuning/jobs"; +import { OpenAIClient } from "../../lib/clients/OpenAIClient"; +import { + HeliconeRequest, + fetchBodies, + getRequests, +} from "../../lib/stores/request/request"; +import { Result, err, ok } from "../../lib/shared/result"; +import crypto from "crypto"; +import fs from "fs"; +import { chatCompletionMessage } from "../types"; +import { ChatCompletionMessageParam } from "openai/resources"; +import { ChatCompletionCreateParamsBase } from "openai/resources/chat/completions"; +import OpenAI from "openai"; +import { JawnAuthenticatedRequest } from "../../types/request"; +import { DatasetManager } from "../dataset/DatasetManager"; +import { BaseManager } from "../BaseManager"; +import { AuthParams, supabaseServer } from "../../lib/db/supabase"; + +export interface OpenPipeFineTuneJobOptions { + _type: "OpenPipeFineTuneJobOptions"; + model: + | "OpenPipe/Hermes-2-Theta-Llama-3-8B-32k" + | "meta-llama/Meta-Llama-3-8B-Instruct" + | "meta-llama/Meta-Llama-3-70B-Instruct" + | "OpenPipe/mistral-ft-optimized-1227" + | "mistralai/Mixtral-8x7B-Instruct-v0.1"; + learningRate?: number; + numEpochs?: number; +} + +export interface OpenAIFineTuneJobOptions { + _type: "OpenAIFineTuneJobOptions"; + model: string; +} + +export type FineTuneJobOptions = + | OpenPipeFineTuneJobOptions + | OpenAIFineTuneJobOptions; + +export interface IFineTuningJob { + id: string; + model: string; + status: string; + created_at: string; + updated_at: string; + baseModel: string; + errorMessage: string; + datasetId?: string; +} + +export interface IFineTuningJobEvent { + id: string; + type: string; + created_at: string; + data: Record; +} + +//TODO rename this back to just FineTuneManager +export abstract class NewFineTuningManager extends BaseManager { + datasetManager: DatasetManager; + + constructor( + authParams: JawnAuthenticatedRequest["authParams"], + private providerKeyId: string + ) { + super(authParams); + this.datasetManager = new DatasetManager(authParams); + } + + async getFineTuneJob(id: string): Promise< + Result< + { + job: IFineTuningJob; + events: IFineTuningJobEvent[]; + }, + string + > + > { + throw new Error("Not implemented"); + } + + private createFineTuneJobOpenAI( + datasetId: string, + options: OpenAIFineTuneJobOptions + ): Promise> { + throw new Error("Not implemented"); + } + + private createFineTuneJobRow(datasetId: string) { + return supabaseServer.client + .from("finetune_job") + .insert({ + dataset_id: datasetId, + finetune_job_id: "", + provider_key_id: this.providerKeyId, + status: "created", + organization_id: this.authParams.organizationId, + }) + .select("*") + .single(); + } + async createFineTuneJob( + datasetId: string, + options: FineTuneJobOptions + ): Promise> { + const fineTunedJobId = await this.createFineTuneJobRow(datasetId); + + if (fineTunedJobId.error) { + return err(fineTunedJobId.error.message); + } + + const { data: dataset, error: datasetError } = await getRequests( + this.authParams.organizationId, + { + experiment_dataset_v2_row: { + dataset_id: { + equals: datasetId, + }, + }, + }, + + 0, + 1000, + { + random: true, + } + ); + + if (datasetError || !dataset) { + return err("Failed to get dataset from request"); + } + + const datasetWithBodies = await fetchBodies(dataset); + // TODO upload data to either OpenAI or OpenPipe + + throw new Error("Not implemented"); + } +} diff --git a/valhalla/jawn/src/tsoa-build/private/routes.ts b/valhalla/jawn/src/tsoa-build/private/routes.ts index 71b63d8920..bbdf35ba3e 100644 --- a/valhalla/jawn/src/tsoa-build/private/routes.ts +++ b/valhalla/jawn/src/tsoa-build/private/routes.ts @@ -13,6 +13,8 @@ import { GenerateHashController } from './../../controllers/private/generateHash // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa import { DatasetController } from './../../controllers/private/datasetController'; // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa +import { ExperimentDatasetController } from './../../controllers/public/experimentDatasetController'; +// WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa import { FineTuneMainController } from './../../controllers/private/fineTuneController'; // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa import { AdminController } from './../../controllers/private/adminController'; @@ -278,14 +280,214 @@ const models: TsoaRoute.Models = { "additionalProperties": false, }, // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "ResultSuccess__datasetId-string__": { + "dataType": "refObject", + "properties": { + "data": {"dataType":"nestedObjectLiteral","nestedProperties":{"datasetId":{"dataType":"string","required":true}},"required":true}, + "error": {"dataType":"enum","enums":[null],"required":true}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "Result__datasetId-string_.string_": { + "dataType": "refAlias", + "type": {"dataType":"union","subSchemas":[{"ref":"ResultSuccess__datasetId-string__"},{"ref":"ResultError_string_"}],"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "DatasetMetadata": { + "dataType": "refObject", + "properties": { + "promptId": {"dataType":"string"}, + "inputRecordsIds": {"dataType":"array","array":{"dataType":"string"}}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "NewDatasetParams": { + "dataType": "refObject", + "properties": { + "datasetName": {"dataType":"string","required":true}, + "requestIds": {"dataType":"array","array":{"dataType":"string"},"required":true}, + "meta": {"ref":"DatasetMetadata"}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "Partial_TimestampOperators_": { + "dataType": "refAlias", + "type": {"dataType":"nestedObjectLiteral","nestedProperties":{"gte":{"dataType":"string"},"lte":{"dataType":"string"},"lt":{"dataType":"string"},"gt":{"dataType":"string"}},"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "Partial_RequestTableToOperators_": { + "dataType": "refAlias", + "type": {"dataType":"nestedObjectLiteral","nestedProperties":{"prompt":{"ref":"Partial_TextOperators_"},"created_at":{"ref":"Partial_TimestampOperators_"},"user_id":{"ref":"Partial_TextOperators_"},"auth_hash":{"ref":"Partial_TextOperators_"},"org_id":{"ref":"Partial_TextOperators_"},"id":{"ref":"Partial_TextOperators_"},"node_id":{"ref":"Partial_TextOperators_"},"model":{"ref":"Partial_TextOperators_"},"modelOverride":{"ref":"Partial_TextOperators_"},"path":{"ref":"Partial_TextOperators_"},"prompt_id":{"ref":"Partial_TextOperators_"}},"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "Partial_NumberOperators_": { + "dataType": "refAlias", + "type": {"dataType":"nestedObjectLiteral","nestedProperties":{"not-equals":{"dataType":"double"},"equals":{"dataType":"double"},"gte":{"dataType":"double"},"lte":{"dataType":"double"},"lt":{"dataType":"double"},"gt":{"dataType":"double"}},"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "Partial_PromptVersionsToOperators_": { + "dataType": "refAlias", + "type": {"dataType":"nestedObjectLiteral","nestedProperties":{"minor_version":{"ref":"Partial_NumberOperators_"},"major_version":{"ref":"Partial_NumberOperators_"},"id":{"ref":"Partial_TextOperators_"},"prompt_v2":{"ref":"Partial_TextOperators_"}},"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "Pick_FilterLeaf.request-or-prompts_versions_": { + "dataType": "refAlias", + "type": {"dataType":"nestedObjectLiteral","nestedProperties":{"request":{"ref":"Partial_RequestTableToOperators_"},"prompts_versions":{"ref":"Partial_PromptVersionsToOperators_"}},"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "FilterLeafSubset_request-or-prompts_versions_": { + "dataType": "refAlias", + "type": {"ref":"Pick_FilterLeaf.request-or-prompts_versions_","validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "DatasetFilterNode": { + "dataType": "refAlias", + "type": {"dataType":"union","subSchemas":[{"ref":"FilterLeafSubset_request-or-prompts_versions_"},{"ref":"DatasetFilterBranch"},{"dataType":"enum","enums":["all"]}],"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "DatasetFilterBranch": { + "dataType": "refAlias", + "type": {"dataType":"nestedObjectLiteral","nestedProperties":{"right":{"ref":"DatasetFilterNode","required":true},"operator":{"dataType":"union","subSchemas":[{"dataType":"enum","enums":["or"]},{"dataType":"enum","enums":["and"]}],"required":true},"left":{"ref":"DatasetFilterNode","required":true}},"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "RandomDatasetParams": { + "dataType": "refObject", + "properties": { + "datasetName": {"dataType":"string","required":true}, + "filter": {"ref":"DatasetFilterNode","required":true}, + "offset": {"dataType":"double"}, + "limit": {"dataType":"double"}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "DatasetResult": { + "dataType": "refObject", + "properties": { + "id": {"dataType":"string","required":true}, + "name": {"dataType":"string","required":true}, + "created_at": {"dataType":"string","required":true}, + "meta": {"ref":"DatasetMetadata"}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "ResultSuccess_DatasetResult-Array_": { + "dataType": "refObject", + "properties": { + "data": {"dataType":"array","array":{"dataType":"refObject","ref":"DatasetResult"},"required":true}, + "error": {"dataType":"enum","enums":[null],"required":true}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "Result_DatasetResult-Array.string_": { + "dataType": "refAlias", + "type": {"dataType":"union","subSchemas":[{"ref":"ResultSuccess_DatasetResult-Array_"},{"ref":"ResultError_string_"}],"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "ResultSuccess___-Array_": { + "dataType": "refObject", + "properties": { + "data": {"dataType":"array","array":{"dataType":"nestedObjectLiteral","nestedProperties":{}},"required":true}, + "error": {"dataType":"enum","enums":[null],"required":true}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "Result___-Array.string_": { + "dataType": "refAlias", + "type": {"dataType":"union","subSchemas":[{"ref":"ResultSuccess___-Array_"},{"ref":"ResultError_string_"}],"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "NewFineTuneJob": { + "dataType": "refObject", + "properties": { + "fineTuneJobId": {"dataType":"string","required":true}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "ResultSuccess_NewFineTuneJob_": { + "dataType": "refObject", + "properties": { + "data": {"ref":"NewFineTuneJob","required":true}, + "error": {"dataType":"enum","enums":[null],"required":true}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "Result_NewFineTuneJob.string_": { + "dataType": "refAlias", + "type": {"dataType":"union","subSchemas":[{"ref":"ResultSuccess_NewFineTuneJob_"},{"ref":"ResultError_string_"}],"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa "FineTuneBody": { "dataType": "refObject", "properties": { "providerKeyId": {"dataType":"string","required":true}, + "datasetId": {"dataType":"string"}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "IFineTuningJob": { + "dataType": "refObject", + "properties": { + "id": {"dataType":"string","required":true}, + "model": {"dataType":"string","required":true}, + "status": {"dataType":"string","required":true}, + "created_at": {"dataType":"string","required":true}, + "updated_at": {"dataType":"string","required":true}, + "baseModel": {"dataType":"string","required":true}, + "errorMessage": {"dataType":"string","required":true}, + "datasetId": {"dataType":"string"}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "Record_string.any_": { + "dataType": "refAlias", + "type": {"dataType":"nestedObjectLiteral","nestedProperties":{},"additionalProperties":{"dataType":"any"},"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "IFineTuningJobEvent": { + "dataType": "refObject", + "properties": { + "id": {"dataType":"string","required":true}, + "type": {"dataType":"string","required":true}, + "created_at": {"dataType":"string","required":true}, + "data": {"ref":"Record_string.any_","required":true}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "FineTuneJobStats": { + "dataType": "refObject", + "properties": { + "job": {"ref":"IFineTuningJob","required":true}, + "events": {"dataType":"array","array":{"dataType":"refObject","ref":"IFineTuningJobEvent"},"required":true}, + }, + "additionalProperties": false, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "ResultSuccess_FineTuneJobStats_": { + "dataType": "refObject", + "properties": { + "data": {"ref":"FineTuneJobStats","required":true}, + "error": {"dataType":"enum","enums":[null],"required":true}, }, "additionalProperties": false, }, // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + "Result_FineTuneJobStats.string_": { + "dataType": "refAlias", + "type": {"dataType":"union","subSchemas":[{"ref":"ResultSuccess_FineTuneJobStats_"},{"ref":"ResultError_string_"}],"validators":{}}, + }, + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa "KafkaSettings": { "dataType": "refObject", "properties": { @@ -654,6 +856,166 @@ export function RegisterRoutes(app: Router) { } }); // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + app.post('/v1/experiment/dataset', + authenticateMiddleware([{"api_key":[]}]), + ...(fetchMiddlewares(ExperimentDatasetController)), + ...(fetchMiddlewares(ExperimentDatasetController.prototype.addDataset)), + + async function ExperimentDatasetController_addDataset(request: ExRequest, response: ExResponse, next: any) { + const args: Record = { + requestBody: {"in":"body","name":"requestBody","required":true,"ref":"NewDatasetParams"}, + request: {"in":"request","name":"request","required":true,"dataType":"object"}, + }; + + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + + let validatedArgs: any[] = []; + try { + validatedArgs = templateService.getValidatedArgs({ args, request, response }); + + const controller = new ExperimentDatasetController(); + + await templateService.apiHandler({ + methodName: 'addDataset', + controller, + response, + next, + validatedArgs, + successStatus: undefined, + }); + } catch (err) { + return next(err); + } + }); + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + app.post('/v1/experiment/dataset/random', + authenticateMiddleware([{"api_key":[]}]), + ...(fetchMiddlewares(ExperimentDatasetController)), + ...(fetchMiddlewares(ExperimentDatasetController.prototype.addRandomDataset)), + + async function ExperimentDatasetController_addRandomDataset(request: ExRequest, response: ExResponse, next: any) { + const args: Record = { + requestBody: {"in":"body","name":"requestBody","required":true,"ref":"RandomDatasetParams"}, + request: {"in":"request","name":"request","required":true,"dataType":"object"}, + }; + + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + + let validatedArgs: any[] = []; + try { + validatedArgs = templateService.getValidatedArgs({ args, request, response }); + + const controller = new ExperimentDatasetController(); + + await templateService.apiHandler({ + methodName: 'addRandomDataset', + controller, + response, + next, + validatedArgs, + successStatus: undefined, + }); + } catch (err) { + return next(err); + } + }); + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + app.post('/v1/experiment/dataset/query', + authenticateMiddleware([{"api_key":[]}]), + ...(fetchMiddlewares(ExperimentDatasetController)), + ...(fetchMiddlewares(ExperimentDatasetController.prototype.getDatasets)), + + async function ExperimentDatasetController_getDatasets(request: ExRequest, response: ExResponse, next: any) { + const args: Record = { + requestBody: {"in":"body","name":"requestBody","required":true,"dataType":"nestedObjectLiteral","nestedProperties":{"promptId":{"dataType":"string"}}}, + request: {"in":"request","name":"request","required":true,"dataType":"object"}, + }; + + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + + let validatedArgs: any[] = []; + try { + validatedArgs = templateService.getValidatedArgs({ args, request, response }); + + const controller = new ExperimentDatasetController(); + + await templateService.apiHandler({ + methodName: 'getDatasets', + controller, + response, + next, + validatedArgs, + successStatus: undefined, + }); + } catch (err) { + return next(err); + } + }); + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + app.post('/v1/experiment/dataset/:datasetId/query', + authenticateMiddleware([{"api_key":[]}]), + ...(fetchMiddlewares(ExperimentDatasetController)), + ...(fetchMiddlewares(ExperimentDatasetController.prototype.getDataset)), + + async function ExperimentDatasetController_getDataset(request: ExRequest, response: ExResponse, next: any) { + const args: Record = { + requestBody: {"in":"body","name":"requestBody","required":true,"dataType":"nestedObjectLiteral","nestedProperties":{}}, + request: {"in":"request","name":"request","required":true,"dataType":"object"}, + }; + + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + + let validatedArgs: any[] = []; + try { + validatedArgs = templateService.getValidatedArgs({ args, request, response }); + + const controller = new ExperimentDatasetController(); + + await templateService.apiHandler({ + methodName: 'getDataset', + controller, + response, + next, + validatedArgs, + successStatus: undefined, + }); + } catch (err) { + return next(err); + } + }); + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + app.post('/v1/experiment/dataset/:datasetId/mutate', + authenticateMiddleware([{"api_key":[]}]), + ...(fetchMiddlewares(ExperimentDatasetController)), + ...(fetchMiddlewares(ExperimentDatasetController.prototype.mutateDataset)), + + async function ExperimentDatasetController_mutateDataset(request: ExRequest, response: ExResponse, next: any) { + const args: Record = { + requestBody: {"in":"body","name":"requestBody","required":true,"dataType":"nestedObjectLiteral","nestedProperties":{"removeRequests":{"dataType":"array","array":{"dataType":"string"},"required":true},"addRequests":{"dataType":"array","array":{"dataType":"string"},"required":true}}}, + request: {"in":"request","name":"request","required":true,"dataType":"object"}, + }; + + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + + let validatedArgs: any[] = []; + try { + validatedArgs = templateService.getValidatedArgs({ args, request, response }); + + const controller = new ExperimentDatasetController(); + + await templateService.apiHandler({ + methodName: 'mutateDataset', + controller, + response, + next, + validatedArgs, + successStatus: undefined, + }); + } catch (err) { + return next(err); + } + }); + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa app.post('/v1/fine-tune', authenticateMiddleware([{"api_key":[]}]), ...(fetchMiddlewares(FineTuneMainController)), diff --git a/valhalla/jawn/src/tsoa-build/private/swagger.json b/valhalla/jawn/src/tsoa-build/private/swagger.json index a109001ceb..a001ae6f5b 100644 --- a/valhalla/jawn/src/tsoa-build/private/swagger.json +++ b/valhalla/jawn/src/tsoa-build/private/swagger.json @@ -754,18 +754,529 @@ "type": "object", "additionalProperties": false }, + "ResultSuccess__datasetId-string__": { + "properties": { + "data": { + "properties": { + "datasetId": { + "type": "string" + } + }, + "required": [ + "datasetId" + ], + "type": "object" + }, + "error": { + "type": "number", + "enum": [ + null + ], + "nullable": true + } + }, + "required": [ + "data", + "error" + ], + "type": "object", + "additionalProperties": false + }, + "Result__datasetId-string_.string_": { + "anyOf": [ + { + "$ref": "#/components/schemas/ResultSuccess__datasetId-string__" + }, + { + "$ref": "#/components/schemas/ResultError_string_" + } + ] + }, + "DatasetMetadata": { + "properties": { + "promptId": { + "type": "string" + }, + "inputRecordsIds": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "additionalProperties": false + }, + "NewDatasetParams": { + "properties": { + "datasetName": { + "type": "string" + }, + "requestIds": { + "items": { + "type": "string" + }, + "type": "array" + }, + "meta": { + "$ref": "#/components/schemas/DatasetMetadata" + } + }, + "required": [ + "datasetName", + "requestIds" + ], + "type": "object", + "additionalProperties": false + }, + "Partial_TimestampOperators_": { + "properties": { + "gte": { + "type": "string" + }, + "lte": { + "type": "string" + }, + "lt": { + "type": "string" + }, + "gt": { + "type": "string" + } + }, + "type": "object", + "description": "Make all properties in T optional" + }, + "Partial_RequestTableToOperators_": { + "properties": { + "prompt": { + "$ref": "#/components/schemas/Partial_TextOperators_" + }, + "created_at": { + "$ref": "#/components/schemas/Partial_TimestampOperators_" + }, + "user_id": { + "$ref": "#/components/schemas/Partial_TextOperators_" + }, + "auth_hash": { + "$ref": "#/components/schemas/Partial_TextOperators_" + }, + "org_id": { + "$ref": "#/components/schemas/Partial_TextOperators_" + }, + "id": { + "$ref": "#/components/schemas/Partial_TextOperators_" + }, + "node_id": { + "$ref": "#/components/schemas/Partial_TextOperators_" + }, + "model": { + "$ref": "#/components/schemas/Partial_TextOperators_" + }, + "modelOverride": { + "$ref": "#/components/schemas/Partial_TextOperators_" + }, + "path": { + "$ref": "#/components/schemas/Partial_TextOperators_" + }, + "prompt_id": { + "$ref": "#/components/schemas/Partial_TextOperators_" + } + }, + "type": "object", + "description": "Make all properties in T optional" + }, + "Partial_NumberOperators_": { + "properties": { + "not-equals": { + "type": "number", + "format": "double" + }, + "equals": { + "type": "number", + "format": "double" + }, + "gte": { + "type": "number", + "format": "double" + }, + "lte": { + "type": "number", + "format": "double" + }, + "lt": { + "type": "number", + "format": "double" + }, + "gt": { + "type": "number", + "format": "double" + } + }, + "type": "object", + "description": "Make all properties in T optional" + }, + "Partial_PromptVersionsToOperators_": { + "properties": { + "minor_version": { + "$ref": "#/components/schemas/Partial_NumberOperators_" + }, + "major_version": { + "$ref": "#/components/schemas/Partial_NumberOperators_" + }, + "id": { + "$ref": "#/components/schemas/Partial_TextOperators_" + }, + "prompt_v2": { + "$ref": "#/components/schemas/Partial_TextOperators_" + } + }, + "type": "object", + "description": "Make all properties in T optional" + }, + "Pick_FilterLeaf.request-or-prompts_versions_": { + "properties": { + "request": { + "$ref": "#/components/schemas/Partial_RequestTableToOperators_" + }, + "prompts_versions": { + "$ref": "#/components/schemas/Partial_PromptVersionsToOperators_" + } + }, + "type": "object", + "description": "From T, pick a set of properties whose keys are in the union K" + }, + "FilterLeafSubset_request-or-prompts_versions_": { + "$ref": "#/components/schemas/Pick_FilterLeaf.request-or-prompts_versions_" + }, + "DatasetFilterNode": { + "anyOf": [ + { + "$ref": "#/components/schemas/FilterLeafSubset_request-or-prompts_versions_" + }, + { + "$ref": "#/components/schemas/DatasetFilterBranch" + }, + { + "type": "string", + "enum": [ + "all" + ] + } + ] + }, + "DatasetFilterBranch": { + "properties": { + "right": { + "$ref": "#/components/schemas/DatasetFilterNode" + }, + "operator": { + "type": "string", + "enum": [ + "or", + "and" + ] + }, + "left": { + "$ref": "#/components/schemas/DatasetFilterNode" + } + }, + "required": [ + "right", + "operator", + "left" + ], + "type": "object" + }, + "RandomDatasetParams": { + "properties": { + "datasetName": { + "type": "string" + }, + "filter": { + "$ref": "#/components/schemas/DatasetFilterNode" + }, + "offset": { + "type": "number", + "format": "double" + }, + "limit": { + "type": "number", + "format": "double" + } + }, + "required": [ + "datasetName", + "filter" + ], + "type": "object", + "additionalProperties": false + }, + "DatasetResult": { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "meta": { + "$ref": "#/components/schemas/DatasetMetadata" + } + }, + "required": [ + "id", + "name", + "created_at" + ], + "type": "object", + "additionalProperties": false + }, + "ResultSuccess_DatasetResult-Array_": { + "properties": { + "data": { + "items": { + "$ref": "#/components/schemas/DatasetResult" + }, + "type": "array" + }, + "error": { + "type": "number", + "enum": [ + null + ], + "nullable": true + } + }, + "required": [ + "data", + "error" + ], + "type": "object", + "additionalProperties": false + }, + "Result_DatasetResult-Array.string_": { + "anyOf": [ + { + "$ref": "#/components/schemas/ResultSuccess_DatasetResult-Array_" + }, + { + "$ref": "#/components/schemas/ResultError_string_" + } + ] + }, + "ResultSuccess___-Array_": { + "properties": { + "data": { + "items": { + "properties": {}, + "type": "object" + }, + "type": "array" + }, + "error": { + "type": "number", + "enum": [ + null + ], + "nullable": true + } + }, + "required": [ + "data", + "error" + ], + "type": "object", + "additionalProperties": false + }, + "Result___-Array.string_": { + "anyOf": [ + { + "$ref": "#/components/schemas/ResultSuccess___-Array_" + }, + { + "$ref": "#/components/schemas/ResultError_string_" + } + ] + }, + "NewFineTuneJob": { + "properties": { + "fineTuneJobId": { + "type": "string" + } + }, + "required": [ + "fineTuneJobId" + ], + "type": "object", + "additionalProperties": false + }, + "ResultSuccess_NewFineTuneJob_": { + "properties": { + "data": { + "$ref": "#/components/schemas/NewFineTuneJob" + }, + "error": { + "type": "number", + "enum": [ + null + ], + "nullable": true + } + }, + "required": [ + "data", + "error" + ], + "type": "object", + "additionalProperties": false + }, + "Result_NewFineTuneJob.string_": { + "anyOf": [ + { + "$ref": "#/components/schemas/ResultSuccess_NewFineTuneJob_" + }, + { + "$ref": "#/components/schemas/ResultError_string_" + } + ] + }, "FineTuneBody": { "properties": { - "providerKeyId": { - "type": "string" + "providerKeyId": { + "type": "string" + }, + "datasetId": { + "type": "string" + } + }, + "required": [ + "providerKeyId" + ], + "type": "object", + "additionalProperties": false + }, + "IFineTuningJob": { + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "status": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + }, + "baseModel": { + "type": "string" + }, + "errorMessage": { + "type": "string" + }, + "datasetId": { + "type": "string" + } + }, + "required": [ + "id", + "model", + "status", + "created_at", + "updated_at", + "baseModel", + "errorMessage" + ], + "type": "object", + "additionalProperties": false + }, + "Record_string.any_": { + "properties": {}, + "additionalProperties": {}, + "type": "object", + "description": "Construct a type with a set of properties K of type T" + }, + "IFineTuningJobEvent": { + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "data": { + "$ref": "#/components/schemas/Record_string.any_" + } + }, + "required": [ + "id", + "type", + "created_at", + "data" + ], + "type": "object", + "additionalProperties": false + }, + "FineTuneJobStats": { + "properties": { + "job": { + "$ref": "#/components/schemas/IFineTuningJob" + }, + "events": { + "items": { + "$ref": "#/components/schemas/IFineTuningJobEvent" + }, + "type": "array" } }, "required": [ - "providerKeyId" + "job", + "events" + ], + "type": "object", + "additionalProperties": false + }, + "ResultSuccess_FineTuneJobStats_": { + "properties": { + "data": { + "$ref": "#/components/schemas/FineTuneJobStats" + }, + "error": { + "type": "number", + "enum": [ + null + ], + "nullable": true + } + }, + "required": [ + "data", + "error" ], "type": "object", "additionalProperties": false }, + "Result_FineTuneJobStats.string_": { + "anyOf": [ + { + "$ref": "#/components/schemas/ResultSuccess_FineTuneJobStats_" + }, + { + "$ref": "#/components/schemas/ResultError_string_" + } + ] + }, "KafkaSettings": { "properties": { "miniBatchSize": { @@ -1267,6 +1778,210 @@ } } }, + "/v1/experiment/dataset": { + "post": { + "operationId": "AddDataset", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Result__datasetId-string_.string_" + } + } + } + } + }, + "tags": [ + "Dataset" + ], + "security": [ + { + "api_key": [] + } + ], + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewDatasetParams" + } + } + } + } + } + }, + "/v1/experiment/dataset/random": { + "post": { + "operationId": "AddRandomDataset", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Result__datasetId-string_.string_" + } + } + } + } + }, + "tags": [ + "Dataset" + ], + "security": [ + { + "api_key": [] + } + ], + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RandomDatasetParams" + } + } + } + } + } + }, + "/v1/experiment/dataset/query": { + "post": { + "operationId": "GetDatasets", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Result_DatasetResult-Array.string_" + } + } + } + } + }, + "tags": [ + "Dataset" + ], + "security": [ + { + "api_key": [] + } + ], + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "properties": { + "promptId": { + "type": "string" + } + }, + "type": "object" + } + } + } + } + } + }, + "/v1/experiment/dataset/{datasetId}/query": { + "post": { + "operationId": "GetDataset", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Result___-Array.string_" + } + } + } + } + }, + "tags": [ + "Dataset" + ], + "security": [ + { + "api_key": [] + } + ], + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "properties": {}, + "type": "object" + } + } + } + } + } + }, + "/v1/experiment/dataset/{datasetId}/mutate": { + "post": { + "operationId": "MutateDataset", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Result___-Array.string_" + } + } + } + } + }, + "tags": [ + "Dataset" + ], + "security": [ + { + "api_key": [] + } + ], + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "properties": { + "removeRequests": { + "items": { + "type": "string" + }, + "type": "array" + }, + "addRequests": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "removeRequests", + "addRequests" + ], + "type": "object" + } + } + } + } + } + }, "/v1/fine-tune": { "post": { "operationId": "FineTune", @@ -1276,46 +1991,7 @@ "content": { "application/json": { "schema": { - "anyOf": [ - { - "properties": { - "error": { - "type": "string" - } - }, - "required": [ - "error" - ], - "type": "object" - }, - { - "properties": { - "data": { - "properties": { - "url": { - "type": "string" - }, - "fineTuneJob": { - "type": "string" - } - }, - "required": [ - "url", - "fineTuneJob" - ], - "type": "object" - }, - "success": { - "type": "boolean" - } - }, - "required": [ - "data", - "success" - ], - "type": "object" - } - ] + "$ref": "#/components/schemas/Result_NewFineTuneJob.string_" } } } @@ -1351,30 +2027,7 @@ "content": { "application/json": { "schema": { - "anyOf": [ - { - "properties": { - "error": { - "type": "string" - } - }, - "required": [ - "error" - ], - "type": "object" - }, - { - "properties": { - "events": {}, - "job": {} - }, - "required": [ - "events", - "job" - ], - "type": "object" - } - ] + "$ref": "#/components/schemas/Result_FineTuneJobStats.string_" } } } diff --git a/web/components/templates/fine-tune/fineTunePage.tsx b/web/components/templates/fine-tune/fineTunePage.tsx index 1d800a7f8d..c03c751f55 100644 --- a/web/components/templates/fine-tune/fineTunePage.tsx +++ b/web/components/templates/fine-tune/fineTunePage.tsx @@ -132,20 +132,72 @@ const FineTuningPage = (props: FineTuningPageProps) => {
-
-
+
+ + + +
+

+ Existing Fine-Tune Jobs +

{isJobsLoading || isDatasetsLoading ? ( diff --git a/web/lib/clients/jawnTypes/private.ts b/web/lib/clients/jawnTypes/private.ts index 16c7cc2d59..72691e32b1 100644 --- a/web/lib/clients/jawnTypes/private.ts +++ b/web/lib/clients/jawnTypes/private.ts @@ -39,6 +39,21 @@ export interface paths { "/v1/dataset/{datasetId}/fine-tune": { post: operations["DatasetFineTune"]; }; + "/v1/experiment/dataset": { + post: operations["AddDataset"]; + }; + "/v1/experiment/dataset/random": { + post: operations["AddRandomDataset"]; + }; + "/v1/experiment/dataset/query": { + post: operations["GetDatasets"]; + }; + "/v1/experiment/dataset/{datasetId}/query": { + post: operations["GetDataset"]; + }; + "/v1/experiment/dataset/{datasetId}/mutate": { + post: operations["MutateDataset"]; + }; "/v1/fine-tune": { post: operations["FineTune"]; }; @@ -268,9 +283,148 @@ export interface components { FineTuneBodyParams: { providerKeyId: string; }; + "ResultSuccess__datasetId-string__": { + data: { + datasetId: string; + }; + /** @enum {number|null} */ + error: null; + }; + "Result__datasetId-string_.string_": components["schemas"]["ResultSuccess__datasetId-string__"] | components["schemas"]["ResultError_string_"]; + DatasetMetadata: { + promptId?: string; + inputRecordsIds?: string[]; + }; + NewDatasetParams: { + datasetName: string; + requestIds: string[]; + meta?: components["schemas"]["DatasetMetadata"]; + }; + /** @description Make all properties in T optional */ + Partial_TimestampOperators_: { + gte?: string; + lte?: string; + lt?: string; + gt?: string; + }; + /** @description Make all properties in T optional */ + Partial_RequestTableToOperators_: { + prompt?: components["schemas"]["Partial_TextOperators_"]; + created_at?: components["schemas"]["Partial_TimestampOperators_"]; + user_id?: components["schemas"]["Partial_TextOperators_"]; + auth_hash?: components["schemas"]["Partial_TextOperators_"]; + org_id?: components["schemas"]["Partial_TextOperators_"]; + id?: components["schemas"]["Partial_TextOperators_"]; + node_id?: components["schemas"]["Partial_TextOperators_"]; + model?: components["schemas"]["Partial_TextOperators_"]; + modelOverride?: components["schemas"]["Partial_TextOperators_"]; + path?: components["schemas"]["Partial_TextOperators_"]; + prompt_id?: components["schemas"]["Partial_TextOperators_"]; + }; + /** @description Make all properties in T optional */ + Partial_NumberOperators_: { + /** Format: double */ + "not-equals"?: number; + /** Format: double */ + equals?: number; + /** Format: double */ + gte?: number; + /** Format: double */ + lte?: number; + /** Format: double */ + lt?: number; + /** Format: double */ + gt?: number; + }; + /** @description Make all properties in T optional */ + Partial_PromptVersionsToOperators_: { + minor_version?: components["schemas"]["Partial_NumberOperators_"]; + major_version?: components["schemas"]["Partial_NumberOperators_"]; + id?: components["schemas"]["Partial_TextOperators_"]; + prompt_v2?: components["schemas"]["Partial_TextOperators_"]; + }; + /** @description From T, pick a set of properties whose keys are in the union K */ + "Pick_FilterLeaf.request-or-prompts_versions_": { + request?: components["schemas"]["Partial_RequestTableToOperators_"]; + prompts_versions?: components["schemas"]["Partial_PromptVersionsToOperators_"]; + }; + "FilterLeafSubset_request-or-prompts_versions_": components["schemas"]["Pick_FilterLeaf.request-or-prompts_versions_"]; + DatasetFilterNode: components["schemas"]["FilterLeafSubset_request-or-prompts_versions_"] | components["schemas"]["DatasetFilterBranch"] | "all"; + DatasetFilterBranch: { + right: components["schemas"]["DatasetFilterNode"]; + /** @enum {string} */ + operator: "or" | "and"; + left: components["schemas"]["DatasetFilterNode"]; + }; + RandomDatasetParams: { + datasetName: string; + filter: components["schemas"]["DatasetFilterNode"]; + /** Format: double */ + offset?: number; + /** Format: double */ + limit?: number; + }; + DatasetResult: { + id: string; + name: string; + created_at: string; + meta?: components["schemas"]["DatasetMetadata"]; + }; + "ResultSuccess_DatasetResult-Array_": { + data: components["schemas"]["DatasetResult"][]; + /** @enum {number|null} */ + error: null; + }; + "Result_DatasetResult-Array.string_": components["schemas"]["ResultSuccess_DatasetResult-Array_"] | components["schemas"]["ResultError_string_"]; + "ResultSuccess___-Array_": { + data: Record[]; + /** @enum {number|null} */ + error: null; + }; + "Result___-Array.string_": components["schemas"]["ResultSuccess___-Array_"] | components["schemas"]["ResultError_string_"]; + NewFineTuneJob: { + fineTuneJobId: string; + }; + ResultSuccess_NewFineTuneJob_: { + data: components["schemas"]["NewFineTuneJob"]; + /** @enum {number|null} */ + error: null; + }; + "Result_NewFineTuneJob.string_": components["schemas"]["ResultSuccess_NewFineTuneJob_"] | components["schemas"]["ResultError_string_"]; FineTuneBody: { providerKeyId: string; + datasetId?: string; }; + IFineTuningJob: { + id: string; + model: string; + status: string; + created_at: string; + updated_at: string; + baseModel: string; + errorMessage: string; + datasetId?: string; + }; + /** @description Construct a type with a set of properties K of type T */ + "Record_string.any_": { + [key: string]: unknown; + }; + IFineTuningJobEvent: { + id: string; + type: string; + created_at: string; + data: components["schemas"]["Record_string.any_"]; + }; + FineTuneJobStats: { + job: components["schemas"]["IFineTuningJob"]; + events: components["schemas"]["IFineTuningJobEvent"][]; + }; + ResultSuccess_FineTuneJobStats_: { + data: components["schemas"]["FineTuneJobStats"]; + /** @enum {number|null} */ + error: null; + }; + "Result_FineTuneJobStats.string_": components["schemas"]["ResultSuccess_FineTuneJobStats_"] | components["schemas"]["ResultError_string_"]; KafkaSettings: { /** Format: double */ miniBatchSize: number; @@ -489,6 +643,86 @@ export interface operations { }; }; }; + AddDataset: { + requestBody: { + content: { + "application/json": components["schemas"]["NewDatasetParams"]; + }; + }; + responses: { + /** @description Ok */ + 200: { + content: { + "application/json": components["schemas"]["Result__datasetId-string_.string_"]; + }; + }; + }; + }; + AddRandomDataset: { + requestBody: { + content: { + "application/json": components["schemas"]["RandomDatasetParams"]; + }; + }; + responses: { + /** @description Ok */ + 200: { + content: { + "application/json": components["schemas"]["Result__datasetId-string_.string_"]; + }; + }; + }; + }; + GetDatasets: { + requestBody: { + content: { + "application/json": { + promptId?: string; + }; + }; + }; + responses: { + /** @description Ok */ + 200: { + content: { + "application/json": components["schemas"]["Result_DatasetResult-Array.string_"]; + }; + }; + }; + }; + GetDataset: { + requestBody: { + content: { + "application/json": Record; + }; + }; + responses: { + /** @description Ok */ + 200: { + content: { + "application/json": components["schemas"]["Result___-Array.string_"]; + }; + }; + }; + }; + MutateDataset: { + requestBody: { + content: { + "application/json": { + removeRequests: string[]; + addRequests: string[]; + }; + }; + }; + responses: { + /** @description Ok */ + 200: { + content: { + "application/json": components["schemas"]["Result___-Array.string_"]; + }; + }; + }; + }; FineTune: { requestBody: { content: { @@ -499,15 +733,7 @@ export interface operations { /** @description Ok */ 200: { content: { - "application/json": { - error: string; - } | { - data: { - url: string; - fineTuneJob: string; - }; - success: boolean; - }; + "application/json": components["schemas"]["Result_NewFineTuneJob.string_"]; }; }; }; @@ -522,12 +748,7 @@ export interface operations { /** @description Ok */ 200: { content: { - "application/json": { - error: string; - } | { - events: unknown; - job: unknown; - }; + "application/json": components["schemas"]["Result_FineTuneJobStats.string_"]; }; }; }; diff --git a/web/lib/clients/jawnTypes/public.ts b/web/lib/clients/jawnTypes/public.ts index c048f53bcc..e7de904627 100644 --- a/web/lib/clients/jawnTypes/public.ts +++ b/web/lib/clients/jawnTypes/public.ts @@ -1,7 +1,3 @@ -type JsonValue = string | number | boolean | null | JsonArray | JsonObject; -interface JsonArray extends Array {} -interface JsonObject { [key: string]: JsonValue; } - /** * This file was auto-generated by openapi-typescript. * Do not make direct changes to the file. @@ -102,7 +98,9 @@ export interface components { startTimeUnixSeconds: number; }; }; -Json: JsonObject; + Json: (string | number | boolean | { + [key: string]: components["schemas"]["Json"]; + } | components["schemas"]["Json"][]) | null; /** @enum {string} */ Provider: "OPENAI" | "ANTHROPIC" | "TOGETHERAI" | "GROQ" | "GOOGLE" | "CUSTOM"; /** @enum {string} */