Skip to content

Commit

Permalink
add new doc store upsert and refresh API
Browse files Browse the repository at this point in the history
  • Loading branch information
@HenryHengZJ
HenryHengZJ committed Nov 23, 2024
1 parent ebf9a98 commit 74ecd8b
Show file tree
Hide file tree
Showing 15 changed files with 1,417 additions and 796 deletions.
252 changes: 130 additions & 122 deletions packages/api-documentation/src/yml/swagger.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,10 @@ paths:
type: string
format: binary
description: Files to be uploaded
base64:
type: boolean
default: false
description: Return contents of the files in base64 format
required:
- files
required: true
Expand Down Expand Up @@ -618,171 +622,109 @@ paths:
'500':
description: Internal server error

/document-store/loader/preview:
/document-store/upsert/{id}:
post:
tags:
- document-store
security:
- bearerAuth: []
summary: Preview document chunks
description: Preview document chunks from loader
operationId: previewChunking
summary: Upsert new document to document store
description: Upsert new document to document store
operationId: upsertDocument
parameters:
- in: path
name: id
required: true
schema:
type: string
format: uuid
description: Document store ID
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/DocumentStoreLoaderForPreview'
required: true
responses:
'200':
description: Successfully preview chunks
content:
application/json:
schema:
type: object
properties:
chunks:
type: array
items:
$ref: '#/components/schemas/Document'
totalChunks:
type: integer
example: 10
previewChunkCount:
type: integer
example: 5
'400':
description: Invalid request body
'500':
description: Internal server error

/document-store/loader/process:
post:
tags:
- document-store
security:
- bearerAuth: []
summary: Process loading & chunking operation
description: Process loading & chunking operation of document from loader
operationId: processChunking
requestBody:
content:
application/json:
$ref: '#/components/schemas/DocumentStoreLoaderForUpsert'
multipart/form-data:
schema:
type: object
required:
- storeId
- id
properties:
storeId:
type: string
description: Document store ID
example: '603a7b51-ae7c-4b0a-8865-e454ed2f6766'
id:
files:
type: array
items:
type: string
format: binary
description: Files to be uploaded
loader:
type: string
description: Document loader ID. If your URL is /document-stores/{storeId}/{id}, then id is the last part of the URL
example: 'c427e569-b81a-469a-b14c-fa73dd5bae49'
required: true
responses:
'200':
description: Successfully process chunking operation
content:
application/json:
schema:
$ref: '#/components/schemas/DocumentStoreFileChunkPagedResponse'

'400':
description: Invalid request body
'500':
description: Internal server error

/document-store/vectorstore/save:
post:
tags:
- document-store
security:
- bearerAuth: []
summary: Save upsert configuration of document store
description: Save upsert configuration of document store
operationId: saveVectorStoreConfig
requestBody:
content:
application/json:
schema:
type: object
required:
- storeId
properties:
storeId:
nullable: true
example: '{"name":"plainText","config":{"text":"why the sky is blue"}}'
description: Loader configurations
splitter:
type: string
description: Document store ID
example: '603a7b51-ae7c-4b0a-8865-e454ed2f6766'
embeddingName:
nullable: true
example: '{"name":"recursiveCharacterTextSplitter","config":{"chunkSize":2000}}'
description: Splitter configurations
embedding:
type: string
description: Name of the embedding
example: 'openAIEmbeddings'
embeddingConfig:
type: object
description: Configuration of the embedding
example: { 'model': 'text-embedding-ada-002', 'credential': '1eba5808-c55b-4817-a285-b0c92846a7ad' }
vectorStoreName:
nullable: true
example: '{"name":"openAIEmbeddings","config":{"modelName":"text-embedding-ada-002"}}'
description: Embedding configurations
vectorStore:
type: string
description: Name of the vector store
example: 'faiss'
vectorStoreConfig:
type: object
description: Configuration of the embedding
example: { 'basePath': './faiss' }
recordManagerName:
nullable: true
example: '{"name":"faiss"}'
description: Vector Store configurations
recordManager:
type: string
description: Name of the record manager
example: 'SQLiteRecordManager'
recordManagerConfig:
type: object
description: Configuration of the embedding
example: { 'databaseFilePath': './recordManager.db' }
nullable: true
example: '{"name":"postgresRecordManager"}'
description: Record Manager configurations
required:
- files
required: true
responses:
'200':
description: Successfully save upsert configuration of document store
description: Successfully execute upsert operation
content:
application/json:
schema:
$ref: '#/components/schemas/DocumentStore'
$ref: '#/components/schemas/VectorUpsertResponse'

'400':
description: Invalid request body
'500':
description: Internal server error

/document-store/vectorstore/insert:
/document-store/refresh/{id}:
post:
tags:
- document-store
security:
- bearerAuth: []
summary: Upsert chunks from document store
description: Upsert chunks from document store using the saved configuration
operationId: insertIntoVectorStore
summary: Re-process and upsert all documents in document store
description: Re-process and upsert all existing documents in document store
operationId: refreshDocument
parameters:
- in: path
name: id
required: true
schema:
type: string
format: uuid
description: Document store ID
requestBody:
content:
application/json:
schema:
type: object
required:
- storeId
properties:
storeId:
type: string
description: Document store ID
example: '603a7b51-ae7c-4b0a-8865-e454ed2f6766'
$ref: '#/components/schemas/DocumentStoreLoaderForRefresh'
required: true
responses:
'200':
description: Successfully save upsert configuration of document store
description: Successfully execute refresh operation
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/VectorUpsertResponse'

'400':
Expand Down Expand Up @@ -2220,6 +2162,72 @@ components:
description:
type: string

DocumentStoreLoaderForUpsert:
type: object
properties:
docId:
type: string
format: uuid
description: Document ID within the store. If provided, existing configuration from the document will be used for the new document
loader:
type: object
properties:
name:
type: string
example: plainText
description: Name of the loader (camelCase)
config:
type: object
description: Configuration for the loader
splitter:
type: object
properties:
name:
type: string
example: recursiveCharacterTextSplitter
description: Name of the text splitter (camelCase)
config:
type: object
description: Configuration for the text splitter
embedding:
type: object
properties:
name:
type: string
example: openAIEmbeddings
description: Name of the embedding generator (camelCase)
config:
type: object
description: Configuration for the embedding generator
vectorStore:
type: object
properties:
name:
type: string
example: faiss
description: Name of the vector store (camelCase)
config:
type: object
description: Configuration for the vector store
recordManager:
type: object
properties:
name:
type: string
example: postgresRecordManager
description: Name of the record manager (camelCase)
config:
type: object
description: Configuration for the record manager

DocumentStoreLoaderForRefresh:
type: object
properties:
items:
type: array
items:
$ref: '#/components/schemas/DocumentStoreLoaderForUpsert'

ChatMessageFeedback:
type: object
properties:
Expand Down
Loading

0 comments on commit 74ecd8b

Please sign in to comment.