From fea96e0b2c11d92d7311c848624f4cf4da4b9439 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Tue, 10 Dec 2024 20:33:21 +0100 Subject: [PATCH 01/10] OpenAI embeddings processor --- go.mod | 3 + go.sum | 12 ++ .../impl/openaiembedding/config_paramgen.go | 71 +++++++ .../impl/openaiembedding/openai_embedding.go | 186 ++++++++++++++++++ pkg/plugin/processor/builtin/registry.go | 2 + 5 files changed, 274 insertions(+) create mode 100644 pkg/plugin/processor/builtin/impl/openaiembedding/config_paramgen.go create mode 100644 pkg/plugin/processor/builtin/impl/openaiembedding/openai_embedding.go diff --git a/go.mod b/go.mod index 2a6e7bba1..166fa5604 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.23.2 require ( buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.35.2-20240617172850-a48fcebcf8f1.1 + github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai v0.7.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 github.com/Masterminds/semver/v3 v3.3.1 github.com/Masterminds/sprig/v3 v3.3.0 github.com/NYTimes/gziphandler v1.1.1 @@ -78,6 +80,7 @@ require ( github.com/Antonboom/errname v1.0.0 // indirect github.com/Antonboom/nilnil v1.0.0 // indirect github.com/Antonboom/testifylint v1.5.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect github.com/Crocmagnon/fatcontext v0.5.3 // indirect diff --git a/go.sum b/go.sum index f2c681393..98502a45b 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,14 @@ github.com/Antonboom/nilnil v1.0.0/go.mod h1:fDJ1FSFoLN6yoG65ANb1WihItf6qt9PJVTn github.com/Antonboom/testifylint v1.5.2 h1:4s3Xhuv5AvdIgbd8wOOEeo0uZG7PbDKQyKY5lGoQazk= github.com/Antonboom/testifylint v1.5.2/go.mod h1:vxy8VJ0bc6NavlYqjZfmp6EfqXMtBgQ4+mhCojwC1P8= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai v0.7.1 h1:6njivKrpo02SQ3CsaGKIFh0c5ZhQyzjVhBmLIl84h4Q= +github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai v0.7.1/go.mod h1:W+7E7pJtvdzscy/I4tqL5C0/weLsa32wyTbHbPdkkv0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= @@ -72,6 +80,8 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs= github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= @@ -396,6 +406,8 @@ github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= diff --git a/pkg/plugin/processor/builtin/impl/openaiembedding/config_paramgen.go b/pkg/plugin/processor/builtin/impl/openaiembedding/config_paramgen.go new file mode 100644 index 000000000..0a80158e9 --- /dev/null +++ b/pkg/plugin/processor/builtin/impl/openaiembedding/config_paramgen.go @@ -0,0 +1,71 @@ +// Code generated by paramgen. DO NOT EDIT. +// Source: github.com/ConduitIO/conduit-commons/tree/main/paramgen + +package openaiembedding + +import ( + "regexp" + + "github.com/conduitio/conduit-commons/config" +) + +const ( + procConfigApiKey = "apiKey" + procConfigEmbeddingEncodingFormat = "embeddingEncodingFormat" + procConfigEndpoint = "endpoint" + procConfigInputField = "inputField" + procConfigModel = "model" + procConfigOutputField = "outputField" +) + +func (procConfig) Parameters() map[string]config.Parameter { + return map[string]config.Parameter{ + procConfigApiKey: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{ + config.ValidationRequired{}, + }, + }, + procConfigEmbeddingEncodingFormat: { + Default: "float", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{ + config.ValidationInclusion{List: []string{"float", "base64"}}, + }, + }, + procConfigEndpoint: { + Default: "https://api.openai.com/v1", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + procConfigInputField: { + Default: ".Payload.After", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{ + config.ValidationRegex{Regex: regexp.MustCompile("^\\.(Payload|Key).*")}, + }, + }, + procConfigModel: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{ + config.ValidationRequired{}, + config.ValidationInclusion{List: []string{"text-embedding-3-small", "text-embedding-3-large"}}, + }, + }, + procConfigOutputField: { + Default: ".Payload.After.vectors", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{ + config.ValidationRegex{Regex: regexp.MustCompile("^\\.(Payload|Key).*")}, + }, + }, + } +} diff --git a/pkg/plugin/processor/builtin/impl/openaiembedding/openai_embedding.go b/pkg/plugin/processor/builtin/impl/openaiembedding/openai_embedding.go new file mode 100644 index 000000000..e5f7de6a1 --- /dev/null +++ b/pkg/plugin/processor/builtin/impl/openaiembedding/openai_embedding.go @@ -0,0 +1,186 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate paramgen -output=config_paramgen.go procConfig + +package openaiembedding + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/conduitio/conduit-commons/config" + "github.com/conduitio/conduit-commons/opencdc" + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit/pkg/foundation/cerrors" + "github.com/conduitio/conduit/pkg/foundation/log" +) + +type procConfig struct { + APIKey string `json:"apiKey" validate:"required"` + Endpoint string `json:"endpoint" default:"https://api.openai.com/v1"` + EmbeddingEncodingFormat string `json:"embeddingEncodingFormat" default:"float" validate:"inclusion=float|base64"` + Model string `json:"model" validate:"required,inclusion=text-embedding-3-small|text-embedding-3-large"` + InputField string `json:"inputField" validate:"regex=^\\.(Payload|Key).*" default:".Payload.After"` + OutputField string `json:"outputField" validate:"regex=^\\.(Payload|Key).*" default:".Payload.After.vectors"` +} + +type processor struct { + sdk.UnimplementedProcessor + + cfg procConfig + + inputFieldRefResolver sdk.ReferenceResolver + outputFieldRefResolver sdk.ReferenceResolver + + client *azopenai.Client + logger log.CtxLogger +} + +func NewProcessor(log log.CtxLogger) sdk.Processor { + return &processor{ + logger: log.WithComponent("openai_embedding"), + } +} + +func (p *processor) Specification() (sdk.Specification, error) { + return sdk.Specification{ + Name: "openai.embedding", + Summary: "Generate OpenAI embeddings.", + Description: "", + Version: "v0.1.0", + Author: "Meroxa, Inc.", + Parameters: procConfig{}.Parameters(), + }, nil +} + +func (p *processor) Configure(ctx context.Context, c config.Config) error { + cfg := procConfig{} + err := sdk.ParseConfig(ctx, c, &cfg, procConfig{}.Parameters()) + if err != nil { + return cerrors.Errorf("failed to parse configuration: %w", err) + } + + inputResolver, err := sdk.NewReferenceResolver(cfg.InputField) + if err != nil { + return cerrors.Errorf(`failed to create a field resolver for %v parameter: %w`, cfg.InputField, err) + } + p.inputFieldRefResolver = inputResolver + + outputResolver, err := sdk.NewReferenceResolver(cfg.OutputField) + if err != nil { + return cerrors.Errorf(`failed to create a field resolver for %v parameter: %w`, cfg.OutputField, err) + } + p.outputFieldRefResolver = outputResolver + + p.cfg = cfg + return nil +} + +func (p *processor) Open(ctx context.Context) error { + keyCredential := azcore.NewKeyCredential(p.cfg.APIKey) + + // NOTE: this constructor creates a client that connects to the public OpenAI endpoint. + // To connect to an Azure OpenAI endpoint, use azopenai.NewClient() or azopenai.NewClientWithyKeyCredential. + client, err := azopenai.NewClientForOpenAI("https://api.openai.com/v1", keyCredential, nil) + if err != nil { + return cerrors.Errorf("failed to create OpenAI client: %w", err) + } + + p.client = client + return nil +} + +func (p *processor) Process(ctx context.Context, records []opencdc.Record) []sdk.ProcessedRecord { + out := make([]sdk.ProcessedRecord, 0, len(records)) + + // Prepare request (embedding inputs) + var embeddingInputs []string + for _, record := range records { + inRef, err := p.inputFieldRefResolver.Resolve(&record) + if err != nil { + out = append(out, sdk.ErrorRecord{Error: fmt.Errorf("failed to resolve reference %v: %w", p.cfg.InputField, err)}) + continue + } + + embeddingInputs = append(embeddingInputs, p.getEmbeddingInput(inRef.Get())) + } + + // Execute request (get embeddings) + embeddings, err := p.client.GetEmbeddings( + ctx, + azopenai.EmbeddingsOptions{ + Input: embeddingInputs, + Dimensions: nil, + EncodingFormat: (*azopenai.EmbeddingEncodingFormat)(&p.cfg.EmbeddingEncodingFormat), + DeploymentName: &p.cfg.Model, + InputType: nil, + User: nil, + }, + nil, + ) + // If the request failed, declare processing for all records as failed + if err != nil { + for range len(records) { + out = append(out, sdk.ErrorRecord{Error: err}) + } + + return out + } + + p.logger.Trace(ctx). + Any("embedding_input", embeddingInputs). + Any("embedding_output", embeddings). + Msg("got embeddings") + + for i, record := range records { + outRef, err := p.outputFieldRefResolver.Resolve(&record) + if err != nil { + out = append(out, sdk.ErrorRecord{Error: cerrors.Errorf("failed to resolve reference %v: %w", p.cfg.OutputField, err)}) + continue + } + + embeddingsMap := opencdc.StructuredData{ + // todo if the encoding format is base64, this needs to change + "embeddings": embeddings.Data[i].Embedding, + } + err = outRef.Set(embeddingsMap) + if err != nil { + out = append(out, sdk.ErrorRecord{Error: cerrors.Errorf("failed to set embeddings to %v: %w", p.cfg.OutputField, err)}) + continue + } + + // todo add metadata related to the embeddings + out = append(out, sdk.SingleRecord(record)) + } + + return out +} + +func (p *processor) Teardown(ctx context.Context) error { + return nil +} + +func (p *processor) getEmbeddingInput(val any) string { + switch v := val.(type) { + case opencdc.RawData: + return string(v) + case opencdc.StructuredData: + return string(v.Bytes()) + default: + return fmt.Sprintf("%v", v) + } +} diff --git a/pkg/plugin/processor/builtin/registry.go b/pkg/plugin/processor/builtin/registry.go index b6e6a18f5..914596885 100644 --- a/pkg/plugin/processor/builtin/registry.go +++ b/pkg/plugin/processor/builtin/registry.go @@ -31,6 +31,7 @@ import ( "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/custom" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/field" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/json" + "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/openaiembedding" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/unwrap" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/webhook" "github.com/conduitio/conduit/pkg/plugin/processor/procutils" @@ -55,6 +56,7 @@ var DefaultBuiltinProcessors = map[string]ProcessorPluginConstructor{ "unwrap.kafkaconnect": unwrap.NewKafkaConnectProcessor, "unwrap.opencdc": unwrap.NewOpenCDCProcessor, "webhook.http": webhook.NewHTTPProcessor, + "openai.embedding": openaiembedding.NewProcessor, } type schemaRegistryProcessor interface { From 6133700d9133be242499b3e4d580feb6f2738585 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Wed, 11 Dec 2024 23:44:15 +0100 Subject: [PATCH 02/10] prompt processor --- .../embedding.go} | 81 +++----- .../impl/openai/embedding_config_paramgen.go | 53 +++++ .../processor/builtin/impl/openai/prompt.go | 193 ++++++++++++++++++ .../prompt_config_paramgen.go} | 42 ++-- pkg/plugin/processor/builtin/registry.go | 5 +- 5 files changed, 294 insertions(+), 80 deletions(-) rename pkg/plugin/processor/builtin/impl/{openaiembedding/openai_embedding.go => openai/embedding.go} (56%) create mode 100644 pkg/plugin/processor/builtin/impl/openai/embedding_config_paramgen.go create mode 100644 pkg/plugin/processor/builtin/impl/openai/prompt.go rename pkg/plugin/processor/builtin/impl/{openaiembedding/config_paramgen.go => openai/prompt_config_paramgen.go} (55%) diff --git a/pkg/plugin/processor/builtin/impl/openaiembedding/openai_embedding.go b/pkg/plugin/processor/builtin/impl/openai/embedding.go similarity index 56% rename from pkg/plugin/processor/builtin/impl/openaiembedding/openai_embedding.go rename to pkg/plugin/processor/builtin/impl/openai/embedding.go index e5f7de6a1..05354e77b 100644 --- a/pkg/plugin/processor/builtin/impl/openaiembedding/openai_embedding.go +++ b/pkg/plugin/processor/builtin/impl/openai/embedding.go @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate paramgen -output=config_paramgen.go procConfig +//go:generate paramgen -output=embedding_config_paramgen.go embeddingProcConfig -package openaiembedding +package openai import ( "context" @@ -23,53 +23,53 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/conduitio/conduit-commons/config" + "github.com/conduitio/conduit-commons/lang" "github.com/conduitio/conduit-commons/opencdc" sdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" ) -type procConfig struct { - APIKey string `json:"apiKey" validate:"required"` - Endpoint string `json:"endpoint" default:"https://api.openai.com/v1"` - EmbeddingEncodingFormat string `json:"embeddingEncodingFormat" default:"float" validate:"inclusion=float|base64"` - Model string `json:"model" validate:"required,inclusion=text-embedding-3-small|text-embedding-3-large"` - InputField string `json:"inputField" validate:"regex=^\\.(Payload|Key).*" default:".Payload.After"` - OutputField string `json:"outputField" validate:"regex=^\\.(Payload|Key).*" default:".Payload.After.vectors"` +const EmbeddingMetadataBase64 = "openai.embedding.base64" + +type embeddingProcConfig struct { + APIKey string `json:"apiKey" validate:"required"` + Endpoint string `json:"endpoint" default:"https://api.openai.com/v1"` + Model string `json:"model" validate:"required,inclusion=gpt-4|gpt-4-turbo|gpt-3.5-turbo|text-davinci-003|text-davinci-002|text-curie-001|text-babbage-001|text-ada-001"` + InputField string `json:"inputField" validate:"regex=^\\.(Payload|Key).*" default:".Payload.After"` } -type processor struct { +type embeddingProcessor struct { sdk.UnimplementedProcessor - cfg procConfig + cfg embeddingProcConfig - inputFieldRefResolver sdk.ReferenceResolver - outputFieldRefResolver sdk.ReferenceResolver + inputFieldRefResolver sdk.ReferenceResolver client *azopenai.Client logger log.CtxLogger } -func NewProcessor(log log.CtxLogger) sdk.Processor { - return &processor{ +func NewEmbeddingProcessor(log log.CtxLogger) sdk.Processor { + return &embeddingProcessor{ logger: log.WithComponent("openai_embedding"), } } -func (p *processor) Specification() (sdk.Specification, error) { +func (p *embeddingProcessor) Specification() (sdk.Specification, error) { return sdk.Specification{ Name: "openai.embedding", Summary: "Generate OpenAI embeddings.", Description: "", Version: "v0.1.0", Author: "Meroxa, Inc.", - Parameters: procConfig{}.Parameters(), + Parameters: embeddingProcConfig{}.Parameters(), }, nil } -func (p *processor) Configure(ctx context.Context, c config.Config) error { - cfg := procConfig{} - err := sdk.ParseConfig(ctx, c, &cfg, procConfig{}.Parameters()) +func (p *embeddingProcessor) Configure(ctx context.Context, c config.Config) error { + cfg := embeddingProcConfig{} + err := sdk.ParseConfig(ctx, c, &cfg, embeddingProcConfig{}.Parameters()) if err != nil { return cerrors.Errorf("failed to parse configuration: %w", err) } @@ -80,17 +80,11 @@ func (p *processor) Configure(ctx context.Context, c config.Config) error { } p.inputFieldRefResolver = inputResolver - outputResolver, err := sdk.NewReferenceResolver(cfg.OutputField) - if err != nil { - return cerrors.Errorf(`failed to create a field resolver for %v parameter: %w`, cfg.OutputField, err) - } - p.outputFieldRefResolver = outputResolver - p.cfg = cfg return nil } -func (p *processor) Open(ctx context.Context) error { +func (p *embeddingProcessor) Open(ctx context.Context) error { keyCredential := azcore.NewKeyCredential(p.cfg.APIKey) // NOTE: this constructor creates a client that connects to the public OpenAI endpoint. @@ -104,10 +98,10 @@ func (p *processor) Open(ctx context.Context) error { return nil } -func (p *processor) Process(ctx context.Context, records []opencdc.Record) []sdk.ProcessedRecord { +func (p *embeddingProcessor) Process(ctx context.Context, records []opencdc.Record) []sdk.ProcessedRecord { out := make([]sdk.ProcessedRecord, 0, len(records)) - // Prepare request (embedding inputs) + // Prepare request (batch all embedding inputs) var embeddingInputs []string for _, record := range records { inRef, err := p.inputFieldRefResolver.Resolve(&record) @@ -124,14 +118,12 @@ func (p *processor) Process(ctx context.Context, records []opencdc.Record) []sdk ctx, azopenai.EmbeddingsOptions{ Input: embeddingInputs, - Dimensions: nil, - EncodingFormat: (*azopenai.EmbeddingEncodingFormat)(&p.cfg.EmbeddingEncodingFormat), + EncodingFormat: lang.Ptr(azopenai.EmbeddingEncodingFormatBase64), DeploymentName: &p.cfg.Model, - InputType: nil, - User: nil, }, nil, ) + // If the request failed, declare processing for all records as failed if err != nil { for range len(records) { @@ -147,34 +139,19 @@ func (p *processor) Process(ctx context.Context, records []opencdc.Record) []sdk Msg("got embeddings") for i, record := range records { - outRef, err := p.outputFieldRefResolver.Resolve(&record) - if err != nil { - out = append(out, sdk.ErrorRecord{Error: cerrors.Errorf("failed to resolve reference %v: %w", p.cfg.OutputField, err)}) - continue - } - - embeddingsMap := opencdc.StructuredData{ - // todo if the encoding format is base64, this needs to change - "embeddings": embeddings.Data[i].Embedding, - } - err = outRef.Set(embeddingsMap) - if err != nil { - out = append(out, sdk.ErrorRecord{Error: cerrors.Errorf("failed to set embeddings to %v: %w", p.cfg.OutputField, err)}) - continue - } - - // todo add metadata related to the embeddings + record.Metadata[EmbeddingMetadataBase64] = embeddings.Data[i].EmbeddingBase64 + // todo add more metadata related to the embeddings out = append(out, sdk.SingleRecord(record)) } return out } -func (p *processor) Teardown(ctx context.Context) error { +func (p *embeddingProcessor) Teardown(context.Context) error { return nil } -func (p *processor) getEmbeddingInput(val any) string { +func (p *embeddingProcessor) getEmbeddingInput(val any) string { switch v := val.(type) { case opencdc.RawData: return string(v) diff --git a/pkg/plugin/processor/builtin/impl/openai/embedding_config_paramgen.go b/pkg/plugin/processor/builtin/impl/openai/embedding_config_paramgen.go new file mode 100644 index 000000000..25f153eb8 --- /dev/null +++ b/pkg/plugin/processor/builtin/impl/openai/embedding_config_paramgen.go @@ -0,0 +1,53 @@ +// Code generated by paramgen. DO NOT EDIT. +// Source: github.com/ConduitIO/conduit-commons/tree/main/paramgen + +package openai + +import ( + "regexp" + + "github.com/conduitio/conduit-commons/config" +) + +const ( + embeddingProcConfigApiKey = "apiKey" + embeddingProcConfigEndpoint = "endpoint" + embeddingProcConfigInputField = "inputField" + embeddingProcConfigModel = "model" +) + +func (embeddingProcConfig) Parameters() map[string]config.Parameter { + return map[string]config.Parameter{ + embeddingProcConfigApiKey: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{ + config.ValidationRequired{}, + }, + }, + embeddingProcConfigEndpoint: { + Default: "https://api.openai.com/v1", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + embeddingProcConfigInputField: { + Default: ".Payload.After", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{ + config.ValidationRegex{Regex: regexp.MustCompile("^\\.(Payload|Key).*")}, + }, + }, + embeddingProcConfigModel: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{ + config.ValidationRequired{}, + config.ValidationInclusion{List: []string{"gpt-4", "gpt-4-turbo", "gpt-3.5-turbo", "text-davinci-003", "text-davinci-002", "text-curie-001", "text-babbage-001", "text-ada-001"}}, + }, + }, + } +} diff --git a/pkg/plugin/processor/builtin/impl/openai/prompt.go b/pkg/plugin/processor/builtin/impl/openai/prompt.go new file mode 100644 index 000000000..a4f12e00a --- /dev/null +++ b/pkg/plugin/processor/builtin/impl/openai/prompt.go @@ -0,0 +1,193 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate paramgen -output=prompt_config_paramgen.go promptProcConfig + +package openai + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/conduitio/conduit-commons/config" + "github.com/conduitio/conduit-commons/lang" + "github.com/conduitio/conduit-commons/opencdc" + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit/pkg/foundation/cerrors" + "github.com/conduitio/conduit/pkg/foundation/log" +) + +const MetadataPromptContext = "openai.prompt.context" + +type promptProcConfig struct { + APIKey string `json:"apiKey" validate:"required"` + Endpoint string `json:"endpoint" default:"https://api.openai.com/v1"` + Prompt string `json:"prompt" validate:"required"` + InputField string `json:"inputField" validate:"regex=^\\.(Payload|Key).*" default:".Payload.After"` + OutputField string `json:"outputField" validate:"regex=^\\.(Payload|Key).*" default:".Payload.After"` +} + +type promptProcessor struct { + sdk.UnimplementedProcessor + + cfg promptProcConfig + + inputFieldRefResolver sdk.ReferenceResolver + outputFieldRefResolver sdk.ReferenceResolver + + client *azopenai.Client + logger log.CtxLogger +} + +func NewPromptProcessor(log log.CtxLogger) sdk.Processor { + return &promptProcessor{ + logger: log.WithComponent("openai.prompt"), + } +} + +func (p *promptProcessor) Specification() (sdk.Specification, error) { + return sdk.Specification{ + Name: "openai.prompt", + Summary: "Generate OpenAI prompts.", + Description: "", + Version: "v0.1.0", + Author: "Meroxa, Inc.", + Parameters: promptProcConfig{}.Parameters(), + }, nil +} + +func (p *promptProcessor) Configure(ctx context.Context, c config.Config) error { + cfg := promptProcConfig{} + err := sdk.ParseConfig(ctx, c, &cfg, promptProcConfig{}.Parameters()) + if err != nil { + return cerrors.Errorf("failed to parse configuration: %w", err) + } + + inputResolver, err := sdk.NewReferenceResolver(cfg.InputField) + if err != nil { + return cerrors.Errorf(`failed to create a field resolver for %v parameter: %w`, cfg.InputField, err) + } + p.inputFieldRefResolver = inputResolver + + outputResolver, err := sdk.NewReferenceResolver(cfg.OutputField) + if err != nil { + return cerrors.Errorf(`failed to create a field resolver for %v parameter: %w`, cfg.OutputField, err) + } + p.outputFieldRefResolver = outputResolver + + p.cfg = cfg + return nil +} + +func (p *promptProcessor) Open(ctx context.Context) error { + keyCredential := azcore.NewKeyCredential(p.cfg.APIKey) + + // NOTE: this constructor creates a client that connects to the public OpenAI endpoint. + // To connect to an Azure OpenAI endpoint, use azopenai.NewClient() or azopenai.NewClientWithyKeyCredential. + client, err := azopenai.NewClientForOpenAI("https://api.openai.com/v1", keyCredential, nil) + if err != nil { + return cerrors.Errorf("failed to create OpenAI client: %w", err) + } + + p.client = client + return nil +} + +func (p *promptProcessor) Process(ctx context.Context, records []opencdc.Record) []sdk.ProcessedRecord { + out := make([]sdk.ProcessedRecord, 0, len(records)) + + for _, rec := range records { + out = append(out, p.processRecord(ctx, rec)) + } + + return out +} + +func (p *promptProcessor) Teardown(ctx context.Context) error { + return nil +} + +func (p *promptProcessor) processRecord(ctx context.Context, rec opencdc.Record) sdk.ProcessedRecord { + inRef, err := p.inputFieldRefResolver.Resolve(&rec) + if err != nil { + return sdk.ErrorRecord{ + Error: fmt.Errorf("failed to resolve reference %v: %w", p.cfg.InputField, err), + } + } + + var messages []azopenai.ChatRequestMessageClassification + promptCtx, ok := rec.Metadata[MetadataPromptContext] + if ok { + messages = append( + messages, + &azopenai.ChatRequestSystemMessage{ + Content: azopenai.NewChatRequestSystemMessageContent( + "Here's the context about the request that's going to be made: " + promptCtx, + ), + }, + ) + } + messages = append( + messages, + &azopenai.ChatRequestUserMessage{ + Content: azopenai.NewChatRequestUserMessageContent(p.cfg.Prompt + ": " + p.getPromptInput(inRef.Get())), + }, + ) + + resp, err := p.client.GetChatCompletions( + ctx, + azopenai.ChatCompletionsOptions{ + // This is a conversation in progress. + // NOTE: all messages count against token usage for this API. + Messages: messages, + DeploymentName: lang.Ptr("gpt-4o"), + }, + nil, + ) + if err != nil { + return sdk.ErrorRecord{ + Error: fmt.Errorf("failed to get chat completions: %w", err), + } + } + + if resp.Choices[0].Message != nil && resp.Choices[0].Message.Content != nil { + outRef, err := p.outputFieldRefResolver.Resolve(&rec) + if err != nil { + return sdk.ErrorRecord{ + Error: fmt.Errorf("failed to resolve reference %v: %w", p.cfg.OutputField, err), + } + } + err = outRef.Set(*resp.Choices[0].Message.Content) + if err != nil { + return sdk.ErrorRecord{ + Error: fmt.Errorf("failed to set chat completion content for %v: %w", p.cfg.OutputField, err), + } + } + } + + return sdk.SingleRecord(rec) +} + +func (p *promptProcessor) getPromptInput(val any) string { + switch v := val.(type) { + case opencdc.RawData: + return string(v) + case opencdc.StructuredData: + return string(v.Bytes()) + default: + return fmt.Sprintf("%v", v) + } +} diff --git a/pkg/plugin/processor/builtin/impl/openaiembedding/config_paramgen.go b/pkg/plugin/processor/builtin/impl/openai/prompt_config_paramgen.go similarity index 55% rename from pkg/plugin/processor/builtin/impl/openaiembedding/config_paramgen.go rename to pkg/plugin/processor/builtin/impl/openai/prompt_config_paramgen.go index 0a80158e9..a46c6c2f5 100644 --- a/pkg/plugin/processor/builtin/impl/openaiembedding/config_paramgen.go +++ b/pkg/plugin/processor/builtin/impl/openai/prompt_config_paramgen.go @@ -1,7 +1,7 @@ // Code generated by paramgen. DO NOT EDIT. // Source: github.com/ConduitIO/conduit-commons/tree/main/paramgen -package openaiembedding +package openai import ( "regexp" @@ -10,17 +10,16 @@ import ( ) const ( - procConfigApiKey = "apiKey" - procConfigEmbeddingEncodingFormat = "embeddingEncodingFormat" - procConfigEndpoint = "endpoint" - procConfigInputField = "inputField" - procConfigModel = "model" - procConfigOutputField = "outputField" + promptProcConfigApiKey = "apiKey" + promptProcConfigEndpoint = "endpoint" + promptProcConfigInputField = "inputField" + promptProcConfigOutputField = "outputField" + promptProcConfigPrompt = "prompt" ) -func (procConfig) Parameters() map[string]config.Parameter { +func (promptProcConfig) Parameters() map[string]config.Parameter { return map[string]config.Parameter{ - procConfigApiKey: { + promptProcConfigApiKey: { Default: "", Description: "", Type: config.ParameterTypeString, @@ -28,21 +27,13 @@ func (procConfig) Parameters() map[string]config.Parameter { config.ValidationRequired{}, }, }, - procConfigEmbeddingEncodingFormat: { - Default: "float", - Description: "", - Type: config.ParameterTypeString, - Validations: []config.Validation{ - config.ValidationInclusion{List: []string{"float", "base64"}}, - }, - }, - procConfigEndpoint: { + promptProcConfigEndpoint: { Default: "https://api.openai.com/v1", Description: "", Type: config.ParameterTypeString, Validations: []config.Validation{}, }, - procConfigInputField: { + promptProcConfigInputField: { Default: ".Payload.After", Description: "", Type: config.ParameterTypeString, @@ -50,21 +41,20 @@ func (procConfig) Parameters() map[string]config.Parameter { config.ValidationRegex{Regex: regexp.MustCompile("^\\.(Payload|Key).*")}, }, }, - procConfigModel: { - Default: "", + promptProcConfigOutputField: { + Default: ".Payload.After", Description: "", Type: config.ParameterTypeString, Validations: []config.Validation{ - config.ValidationRequired{}, - config.ValidationInclusion{List: []string{"text-embedding-3-small", "text-embedding-3-large"}}, + config.ValidationRegex{Regex: regexp.MustCompile("^\\.(Payload|Key).*")}, }, }, - procConfigOutputField: { - Default: ".Payload.After.vectors", + promptProcConfigPrompt: { + Default: "", Description: "", Type: config.ParameterTypeString, Validations: []config.Validation{ - config.ValidationRegex{Regex: regexp.MustCompile("^\\.(Payload|Key).*")}, + config.ValidationRequired{}, }, }, } diff --git a/pkg/plugin/processor/builtin/registry.go b/pkg/plugin/processor/builtin/registry.go index 914596885..120f9e88a 100644 --- a/pkg/plugin/processor/builtin/registry.go +++ b/pkg/plugin/processor/builtin/registry.go @@ -31,7 +31,7 @@ import ( "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/custom" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/field" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/json" - "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/openaiembedding" + "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/openai" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/unwrap" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/webhook" "github.com/conduitio/conduit/pkg/plugin/processor/procutils" @@ -56,7 +56,8 @@ var DefaultBuiltinProcessors = map[string]ProcessorPluginConstructor{ "unwrap.kafkaconnect": unwrap.NewKafkaConnectProcessor, "unwrap.opencdc": unwrap.NewOpenCDCProcessor, "webhook.http": webhook.NewHTTPProcessor, - "openai.embedding": openaiembedding.NewProcessor, + "openai.embedding": openai.NewEmbeddingProcessor, + "openai.prompt": openai.NewPromptProcessor, } type schemaRegistryProcessor interface { From 6d74f54d61ac873b62be68f1b10a527945909927 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Wed, 11 Dec 2024 23:45:21 +0100 Subject: [PATCH 03/10] linter --- pkg/plugin/processor/builtin/impl/openai/embedding.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/plugin/processor/builtin/impl/openai/embedding.go b/pkg/plugin/processor/builtin/impl/openai/embedding.go index 05354e77b..e02c66098 100644 --- a/pkg/plugin/processor/builtin/impl/openai/embedding.go +++ b/pkg/plugin/processor/builtin/impl/openai/embedding.go @@ -123,7 +123,6 @@ func (p *embeddingProcessor) Process(ctx context.Context, records []opencdc.Reco }, nil, ) - // If the request failed, declare processing for all records as failed if err != nil { for range len(records) { From 2bfb4652ae97bd9802569524847c492b2edf78f1 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Thu, 12 Dec 2024 21:12:06 +0100 Subject: [PATCH 04/10] get context processor --- go.mod | 17 ++ go.sum | 127 ++++++++++++ .../builtin/impl/{ => ai}/openai/embedding.go | 0 .../openai/embedding_config_paramgen.go | 0 .../builtin/impl/{ => ai}/openai/prompt.go | 0 .../{ => ai}/openai/prompt_config_paramgen.go | 0 .../builtin/impl/ai/weaviate/get_context.go | 180 ++++++++++++++++++ .../weaviate/get_context_config_paramgen.go | 23 +++ pkg/plugin/processor/builtin/registry.go | 4 +- 9 files changed, 350 insertions(+), 1 deletion(-) rename pkg/plugin/processor/builtin/impl/{ => ai}/openai/embedding.go (100%) rename pkg/plugin/processor/builtin/impl/{ => ai}/openai/embedding_config_paramgen.go (100%) rename pkg/plugin/processor/builtin/impl/{ => ai}/openai/prompt.go (100%) rename pkg/plugin/processor/builtin/impl/{ => ai}/openai/prompt_config_paramgen.go (100%) create mode 100644 pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go create mode 100644 pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go diff --git a/go.mod b/go.mod index 166fa5604..4fd4dc471 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( github.com/tetratelabs/wazero v1.8.2 github.com/twmb/franz-go/pkg/sr v1.2.0 github.com/twmb/go-cache v1.2.1 + github.com/weaviate/weaviate-go-client/v4 v4.16.1 go.uber.org/mock v0.5.0 golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f golang.org/x/tools v0.28.0 @@ -98,6 +99,7 @@ require ( github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect github.com/apache/thrift v0.14.2 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/ashanbrown/forbidigo v1.6.0 // indirect github.com/ashanbrown/makezero v1.1.1 // indirect github.com/aws/aws-sdk-go-v2 v1.32.4 // indirect @@ -174,6 +176,15 @@ require ( github.com/go-critic/go-critic v0.11.5 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/analysis v0.21.2 // indirect + github.com/go-openapi/errors v0.22.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/loads v0.21.1 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-openapi/validate v0.21.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-toolsmith/astcast v1.1.0 // indirect @@ -227,6 +238,7 @@ require ( github.com/jgautheron/goconst v1.7.1 // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect github.com/jjti/go-spancheck v0.6.2 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/julz/importas v0.1.0 // indirect github.com/karamaru-alpha/copyloopvar v1.1.0 // indirect @@ -245,6 +257,7 @@ require ( github.com/leonklingele/grouper v1.1.2 // indirect github.com/macabu/inamedparam v0.1.3 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/maratori/testableexamples v1.0.0 // indirect github.com/maratori/testpackage v1.1.1 // indirect github.com/matoous/godox v0.0.0-20240105082147-c5b5e0e7c0c0 // indirect @@ -277,6 +290,7 @@ require ( github.com/nunnatsa/ginkgolinter v0.18.3 // indirect github.com/nxadm/tail v1.4.11 // indirect github.com/oklog/run v1.1.0 // indirect + github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onsi/ginkgo/v2 v2.21.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -343,6 +357,7 @@ require ( github.com/uudashr/gocognit v1.1.3 // indirect github.com/uudashr/iface v1.2.1 // indirect github.com/vbatts/tar-split v0.11.6 // indirect + github.com/weaviate/weaviate v1.27.0 // indirect github.com/xen0n/gosmopolitan v1.2.2 // indirect github.com/xitongsys/parquet-go v1.6.2 // indirect github.com/xitongsys/parquet-go-source v0.0.0-20211228015320-b4f792c43cd0 // indirect @@ -356,6 +371,7 @@ require ( go.lsp.dev/pkg v0.0.0-20210717090340-384b27a52fb2 // indirect go.lsp.dev/protocol v0.12.0 // indirect go.lsp.dev/uri v0.3.0 // indirect + go.mongodb.org/mongo-driver v1.14.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 // indirect go.opentelemetry.io/otel v1.32.0 // indirect @@ -370,6 +386,7 @@ require ( golang.org/x/exp/typeparams v0.0.0-20241108190413-2d47ceb2692f // indirect golang.org/x/mod v0.22.0 // indirect golang.org/x/net v0.32.0 // indirect + golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect diff --git a/go.sum b/go.sum index 98502a45b..c4c043381 100644 --- a/go.sum +++ b/go.sum @@ -108,6 +108,8 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA= github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/go-check-sumtype v0.2.0 h1:Bo+e4DFf3rs7ME9w/0SU/g6nmzJaphduP8Cjiz0gbwY= @@ -127,6 +129,9 @@ github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QN github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.14.2 h1:hY4rAyg7Eqbb27GB6gkhUKrRAuc8xRjlNtJq+LseKeY= github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/ashanbrown/forbidigo v1.6.0 h1:D3aewfM37Yb3pxHujIPSpTf6oQk9sc9WZi8gerOIVIY= github.com/ashanbrown/forbidigo v1.6.0/go.mod h1:Y8j9jy9ZYAEHXdu723cUlraTqbzjKF1MUyfOKL+AjcU= github.com/ashanbrown/makezero v1.1.1 h1:iCQ87C0V0vSyO+M9E/FZYbu65auqH0lnsOkf5FcB28s= @@ -280,6 +285,7 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo= @@ -364,11 +370,39 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/analysis v0.21.2 h1:hXFrOYFHUAMQdu6zwAiKKJHJQ8kqZs1ux/ru1P1wLJU= +github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= +github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= +github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/loads v0.21.1 h1:Wb3nVZpdEzDTcly8S4HMkey6fjARRzb7iEaySimlDW0= +github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= +github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/validate v0.21.0 h1:+Wqk39yKOhfpLqNLEC0/eViCkzM5FVXVqrvt526+wcI= +github.com/go-openapi/validate v0.21.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= @@ -394,6 +428,30 @@ github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIx github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-xmlfmt/xmlfmt v1.1.2 h1:Nea7b4icn8s57fTx1M5AI4qQT5HEM3rVUO8MuE6g80U= github.com/go-xmlfmt/xmlfmt v1.1.2/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= @@ -434,6 +492,7 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -488,6 +547,7 @@ github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0Z github.com/google/pprof v0.0.0-20241101162523-b92577c0c142 h1:sAGdeJj0bnMgUNVeUpp6AYlVdCt3/GdI3pGRqsNSQLs= github.com/google/pprof v0.0.0-20241101162523-b92577c0c142/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -537,6 +597,7 @@ github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0= @@ -569,6 +630,8 @@ github.com/jjti/go-spancheck v0.6.2/go.mod h1:+X7lvIrR5ZdUTkxFYqzJ0abr8Sb5LOo80u github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -580,6 +643,8 @@ github.com/julz/importas v0.1.0 h1:F78HnrsjY3cR7j0etXy5+TU1Zuy7Xt08X/1aJnH5xXY= github.com/julz/importas v0.1.0/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSXO0DV0= github.com/karamaru-alpha/copyloopvar v1.1.0 h1:x7gNyKcC2vRBO1H2Mks5u1VxQtYvFiym7fCjIP8RPos= github.com/karamaru-alpha/copyloopvar v1.1.0/go.mod h1:u7CIfztblY0jZLOQZgH3oYsJzpC2A7S6u/lfgSXHy0k= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.8.0 h1:ZX/URYa7ilESY19ik/vBmCn6zdGQLxACwjAcWbHlYlg= github.com/kisielk/errcheck v1.8.0/go.mod h1:1kLL+jV4e+CFfueBmI1dSK2ADDyQnlrnrY/FqKluHJQ= @@ -588,10 +653,13 @@ github.com/kkHAIKE/contextcheck v1.1.5 h1:CdnJh63tcDe53vG+RebdpdXJTc9atMgGqdx8LX github.com/kkHAIKE/contextcheck v1.1.5/go.mod h1:O930cpht4xb1YQpK+1+AgoM3mFsvxr7uyFptcnWTYUA= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -625,11 +693,17 @@ github.com/macabu/inamedparam v0.1.3 h1:2tk/phHkMlEL/1GNe/Yf6kkR/hkcUdAEY3L0hjYV github.com/macabu/inamedparam v0.1.3/go.mod h1:93FLICAIk/quk7eaPPQvbzihUdn/QkGDwIZEoLtpH6I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maratori/testableexamples v1.0.0 h1:dU5alXRrD8WKSjOUnmJZuzdxWOEQ57+7s93SLMxb2vI= github.com/maratori/testableexamples v1.0.0/go.mod h1:4rhjL1n20TUTT4vdh3RDqSizKLyXp7K2u6HgraZCGzE= github.com/maratori/testpackage v1.1.1 h1:S58XVV5AD7HADMmD0fNnziNHqKvSdDuEKdPD1rNTU04= github.com/maratori/testpackage v1.1.1/go.mod h1:s4gRK/ym6AMrqpOa/kEbQTV4Q4jb7WeLZzVhVVVOQMc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matoous/godox v0.0.0-20240105082147-c5b5e0e7c0c0 h1:Ny7cm4KSWceJLYyI1sm+aFIVDWSGXLcOJ0O0UaS5wdU= github.com/matoous/godox v0.0.0-20240105082147-c5b5e0e7c0c0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= @@ -655,6 +729,8 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -682,6 +758,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/moricho/tparallel v0.3.2 h1:odr8aZVFA3NZrNybggMkYO3rgPRcqjeQUlBBFVxKHTI= github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -695,6 +772,7 @@ github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJm github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/neilotoole/slogt v1.1.0 h1:c7qE92sq+V0yvCuaxph+RQ2jOKL61c4hqS1Bv9W7FZE= github.com/neilotoole/slogt v1.1.0/go.mod h1:RCrGXkPc/hYybNulqQrMHRtvlQ7F6NktNVLuLwk6V+w= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg= github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= @@ -705,6 +783,8 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= @@ -728,6 +808,7 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= @@ -739,6 +820,8 @@ github.com/piotrkowalczuk/promgrpc/v4 v4.1.4 h1:c0uCOlfeDBb0Mr1os7TY4g9RfPtaK4EE github.com/piotrkowalczuk/promgrpc/v4 v4.1.4/go.mod h1:MGaZJU+KEuYk4UftJOYDhcOEy2cN0CmuaPw+ILFlweY= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= @@ -780,6 +863,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= @@ -818,6 +903,9 @@ github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+Wwfd0XE= @@ -835,8 +923,10 @@ github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= @@ -850,6 +940,7 @@ github.com/stealthrocket/wazergo v0.19.1/go.mod h1:riI0hxw4ndZA5e6z7PesHg2BtTftc github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= @@ -859,6 +950,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= @@ -879,6 +971,7 @@ github.com/tetafro/godot v1.4.18 h1:ouX3XGiziKDypbpXqShBfnNLTSjR8r3/HVzrtJ+bHlI= github.com/tetafro/godot v1.4.18/go.mod h1:2oVxTBSftRTh4+MVfUaUXR6bn2GDXCaMcOG4Dk3rfio= github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4= github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= github.com/timonwong/loggercheck v0.10.1 h1:uVZYClxQFpw55eh+PIoqM7uAOHMrhVcDoWDery9R8Lg= @@ -907,6 +1000,13 @@ github.com/uudashr/iface v1.2.1 h1:vHHyzAUmWZ64Olq6NZT3vg/z1Ws56kyPdBOd5kTXDF8= github.com/uudashr/iface v1.2.1/go.mod h1:4QvspiRd3JLPAEXBQ9AiZpLbJlrWWgRChOKDJEuQTdg= github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23envGs= github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI= +github.com/weaviate/weaviate v1.27.0 h1:ovFnKER+HRpT5PPuR1ysbKgit0NSpHbBLcsjWR1UyWI= +github.com/weaviate/weaviate v1.27.0/go.mod h1:ppTWDzt/atYk1KhyYzxVD8XckmaCaOYnnmelD5M4LK4= +github.com/weaviate/weaviate-go-client/v4 v4.16.1 h1:jkDYuRCYly6zG2ngqTpv6z8azzbqiMUXcmaJHJmAV0Q= +github.com/weaviate/weaviate-go-client/v4 v4.16.1/go.mod h1:XmoRpzNpWrTW5/TE07dUtxy5kMZbG3uAG/3b69nuwFk= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xen0n/gosmopolitan v1.2.2 h1:/p2KTnMzwRexIW8GlKawsTWOxn7UHA+jCMF/V8HHtvU= github.com/xen0n/gosmopolitan v1.2.2/go.mod h1:7XX7Mj61uLYrj0qmeN0zi7XDon9JRAEhYQqAPLVNTeg= github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= @@ -922,6 +1022,7 @@ github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5Jsjqto github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4= github.com/ykadowak/zerologlint v0.1.5 h1:Gy/fMz1dFQN9JZTPjv1hxEk+sRWm05row04Yoolgdiw= github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -945,6 +1046,10 @@ go.lsp.dev/protocol v0.12.0 h1:tNprUI9klQW5FAFVM4Sa+AbPFuVQByWhP1ttNUAjIWg= go.lsp.dev/protocol v0.12.0/go.mod h1:Qb11/HgZQ72qQbeyPfJbu3hZBH23s1sr4st8czGeDMQ= go.lsp.dev/uri v0.3.0 h1:KcZJmh6nFIBeJzTugn5JTU6OOyG0lDOo3R9KwTxTYbo= go.lsp.dev/uri v0.3.0/go.mod h1:P5sbO1IQR+qySTWOCnhnK7phBx+W3zbLqSMDJNTw88I= +go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= +go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= +go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= +go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -982,10 +1087,13 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= @@ -1062,6 +1170,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= @@ -1077,10 +1186,13 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1093,11 +1205,16 @@ golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1118,6 +1235,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1159,6 +1277,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -1180,9 +1299,13 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190321232350-e250d351ecad/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -1291,6 +1414,7 @@ google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8i google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -1310,6 +1434,9 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= diff --git a/pkg/plugin/processor/builtin/impl/openai/embedding.go b/pkg/plugin/processor/builtin/impl/ai/openai/embedding.go similarity index 100% rename from pkg/plugin/processor/builtin/impl/openai/embedding.go rename to pkg/plugin/processor/builtin/impl/ai/openai/embedding.go diff --git a/pkg/plugin/processor/builtin/impl/openai/embedding_config_paramgen.go b/pkg/plugin/processor/builtin/impl/ai/openai/embedding_config_paramgen.go similarity index 100% rename from pkg/plugin/processor/builtin/impl/openai/embedding_config_paramgen.go rename to pkg/plugin/processor/builtin/impl/ai/openai/embedding_config_paramgen.go diff --git a/pkg/plugin/processor/builtin/impl/openai/prompt.go b/pkg/plugin/processor/builtin/impl/ai/openai/prompt.go similarity index 100% rename from pkg/plugin/processor/builtin/impl/openai/prompt.go rename to pkg/plugin/processor/builtin/impl/ai/openai/prompt.go diff --git a/pkg/plugin/processor/builtin/impl/openai/prompt_config_paramgen.go b/pkg/plugin/processor/builtin/impl/ai/openai/prompt_config_paramgen.go similarity index 100% rename from pkg/plugin/processor/builtin/impl/openai/prompt_config_paramgen.go rename to pkg/plugin/processor/builtin/impl/ai/openai/prompt_config_paramgen.go diff --git a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go new file mode 100644 index 000000000..850745859 --- /dev/null +++ b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go @@ -0,0 +1,180 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate paramgen -output=get_context_config_paramgen.go getContextConfig + +package weaviate + +import ( + "context" + "encoding/base64" + "encoding/binary" + "fmt" + "math" + "strings" + + "github.com/conduitio/conduit-commons/config" + "github.com/conduitio/conduit-commons/opencdc" + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit/pkg/foundation/cerrors" + "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/weaviate/weaviate-go-client/v4/weaviate" + "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" +) + +type getContextConfig struct { + ClassName string `json:"className"` +} + +type getContextProcessor struct { + sdk.UnimplementedProcessor + cfg getContextConfig + client *weaviate.Client +} + +func NewGetContextProcessor(logger log.CtxLogger) sdk.Processor { + return &getContextProcessor{} +} + +func (p *getContextProcessor) Specification() (sdk.Specification, error) { + return sdk.Specification{ + Name: "weaviate.getContext", + Summary: "Get context from Weaviate", + Description: "Get context from Weaviate", + Version: "v0.1.0", + Author: "Meroxa, Inc.", + Parameters: getContextConfig{}.Parameters(), + }, nil +} + +func (p *getContextProcessor) Configure(ctx context.Context, c config.Config) error { + cfg := getContextConfig{} + err := sdk.ParseConfig(ctx, c, &cfg, getContextConfig{}.Parameters()) + if err != nil { + return cerrors.Errorf("failed to parse configuration: %w", err) + } + + p.cfg = cfg + return nil +} + +func (p *getContextProcessor) Open(ctx context.Context) error { + // Configure Weaviate client + cfg := weaviate.Config{ + Host: "localhost:8080", // Replace with your Weaviate instance URL + Scheme: "http", + } + client, err := weaviate.NewClient(cfg) + if err != nil { + return cerrors.Errorf("Failed to create Weaviate client: %v", err) + } + + p.client = client + return nil +} + +func (p *getContextProcessor) Process(ctx context.Context, records []opencdc.Record) []sdk.ProcessedRecord { + out := make([]sdk.ProcessedRecord, 0, len(records)) + + for _, record := range records { + out = append(out, p.setContext(ctx, record)) + } + + return out +} + +func (p *getContextProcessor) Teardown(ctx context.Context) error { + return nil +} + +func (p *getContextProcessor) setContext(ctx context.Context, rec opencdc.Record) sdk.ProcessedRecord { + embeddings, err := p.decodeEmbeddings(rec.Metadata["openai.embedding.base64"]) + if err != nil { + return sdk.ErrorRecord{Error: cerrors.Errorf("failed to decode embeddings: %w", err)} + } + + textContext, err := p.getContext(ctx, embeddings) + if err != nil { + return sdk.ErrorRecord{Error: cerrors.Errorf("failed to perform vector search: %w", err)} + } + + rec.Metadata["weaviate.context"] = textContext + + return sdk.SingleRecord(rec) +} + +func (p *getContextProcessor) decodeEmbeddings(base64Str string) ([]float32, error) { + return []float32{0.1}, nil + + // Decode the base64 string + decodedBytes, err := base64.StdEncoding.DecodeString(base64Str) + if err != nil { + return nil, err + } + + // Convert byte slice to float32 slice + embeddings := make([]float32, len(decodedBytes)/4) + + for i := 0; i < len(embeddings); i++ { + // Convert 4 bytes to a single float32 + bits := binary.LittleEndian.Uint32(decodedBytes[i*4 : (i+1)*4]) + embeddings[i] = math.Float32frombits(bits) + } + + return embeddings, nil +} + +func (p *getContextProcessor) getContext(ctx context.Context, targetEmbedding []float32) (string, error) { + // Prepare the GraphQL query for vector search + fields := []graphql.Field{ + {Name: "content"}, + {Name: "_additional { distance }"}, + } + + result, err := p.client.GraphQL(). + Get(). + WithClassName(p.cfg.ClassName). + WithFields(fields...). + WithNearVector( + p.client.GraphQL().NearVectorArgBuilder().WithVector(targetEmbedding), + ). + WithLimit(5). + Do(ctx) + + if err != nil { + return "", fmt.Errorf("vector search failed: %v", err) + } + + // Process and print the results + if result.Errors != nil { + errs := make([]string, 0) + for _, resultErr := range result.Errors { + errs = append(errs, resultErr.Message) + } + return "", cerrors.Errorf("vector search failed: %v", strings.Join(errs, ", ")) + } + + data := result.Data["Get"].(map[string]interface{})[p.cfg.ClassName].([]interface{}) + + sb := strings.Builder{} + fmt.Println("Top 5 Closest Texts:") + for _, item := range data { + itemMap := item.(map[string]interface{}) + text := itemMap["text"].(string) + + sb.WriteString(text) + } + + return sb.String(), nil +} diff --git a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go new file mode 100644 index 000000000..d6a5df18c --- /dev/null +++ b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go @@ -0,0 +1,23 @@ +// Code generated by paramgen. DO NOT EDIT. +// Source: github.com/ConduitIO/conduit-commons/tree/main/paramgen + +package weaviate + +import ( + "github.com/conduitio/conduit-commons/config" +) + +const ( + getContextConfigClassName = "className" +) + +func (getContextConfig) Parameters() map[string]config.Parameter { + return map[string]config.Parameter{ + getContextConfigClassName: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + } +} diff --git a/pkg/plugin/processor/builtin/registry.go b/pkg/plugin/processor/builtin/registry.go index 120f9e88a..3f428afb9 100644 --- a/pkg/plugin/processor/builtin/registry.go +++ b/pkg/plugin/processor/builtin/registry.go @@ -26,12 +26,13 @@ import ( "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/plugin" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl" + "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/ai/openai" + "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/ai/weaviate" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/avro" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/base64" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/custom" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/field" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/json" - "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/openai" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/unwrap" "github.com/conduitio/conduit/pkg/plugin/processor/builtin/impl/webhook" "github.com/conduitio/conduit/pkg/plugin/processor/procutils" @@ -58,6 +59,7 @@ var DefaultBuiltinProcessors = map[string]ProcessorPluginConstructor{ "webhook.http": webhook.NewHTTPProcessor, "openai.embedding": openai.NewEmbeddingProcessor, "openai.prompt": openai.NewPromptProcessor, + "weaviate.getContext": weaviate.NewGetContextProcessor, } type schemaRegistryProcessor interface { From cd96a2172d16f3a21c5e11816cfe0934425da125 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Thu, 12 Dec 2024 22:28:19 +0100 Subject: [PATCH 05/10] get context --- .../builtin/impl/ai/weaviate/get_context.go | 13 +++++++------ .../ai/weaviate/get_context_config_paramgen.go | 15 +++++++++++++-- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go index 850745859..446b7e444 100644 --- a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go +++ b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go @@ -34,7 +34,8 @@ import ( ) type getContextConfig struct { - ClassName string `json:"className"` + ClassName string `json:"className" validate:"required"` + ContextField string `json:"contextField" validate:"required"` } type getContextProcessor struct { @@ -43,7 +44,7 @@ type getContextProcessor struct { client *weaviate.Client } -func NewGetContextProcessor(logger log.CtxLogger) sdk.Processor { +func NewGetContextProcessor(_ log.CtxLogger) sdk.Processor { return &getContextProcessor{} } @@ -115,8 +116,6 @@ func (p *getContextProcessor) setContext(ctx context.Context, rec opencdc.Record } func (p *getContextProcessor) decodeEmbeddings(base64Str string) ([]float32, error) { - return []float32{0.1}, nil - // Decode the base64 string decodedBytes, err := base64.StdEncoding.DecodeString(base64Str) if err != nil { @@ -138,7 +137,7 @@ func (p *getContextProcessor) decodeEmbeddings(base64Str string) ([]float32, err func (p *getContextProcessor) getContext(ctx context.Context, targetEmbedding []float32) (string, error) { // Prepare the GraphQL query for vector search fields := []graphql.Field{ - {Name: "content"}, + {Name: p.cfg.ContextField}, {Name: "_additional { distance }"}, } @@ -171,9 +170,11 @@ func (p *getContextProcessor) getContext(ctx context.Context, targetEmbedding [] fmt.Println("Top 5 Closest Texts:") for _, item := range data { itemMap := item.(map[string]interface{}) - text := itemMap["text"].(string) + text := itemMap[p.cfg.ContextField].(string) sb.WriteString(text) + sb.WriteRune('\n') + sb.WriteRune('\n') } return sb.String(), nil diff --git a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go index d6a5df18c..4c74c3f5b 100644 --- a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go +++ b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go @@ -8,7 +8,8 @@ import ( ) const ( - getContextConfigClassName = "className" + getContextConfigClassName = "className" + getContextConfigContextField = "contextField" ) func (getContextConfig) Parameters() map[string]config.Parameter { @@ -17,7 +18,17 @@ func (getContextConfig) Parameters() map[string]config.Parameter { Default: "", Description: "", Type: config.ParameterTypeString, - Validations: []config.Validation{}, + Validations: []config.Validation{ + config.ValidationRequired{}, + }, + }, + getContextConfigContextField: { + Default: "", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{ + config.ValidationRequired{}, + }, }, } } From 33681dc22903d7fab62a896de15eed507b67a68b Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Thu, 12 Dec 2024 22:28:52 +0100 Subject: [PATCH 06/10] lint --- pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go index 446b7e444..761e6c977 100644 --- a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go +++ b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go @@ -150,7 +150,6 @@ func (p *getContextProcessor) getContext(ctx context.Context, targetEmbedding [] ). WithLimit(5). Do(ctx) - if err != nil { return "", fmt.Errorf("vector search failed: %v", err) } From 28db62d2359d34eefb31df9234d395f58f714f88 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Thu, 12 Dec 2024 22:34:28 +0100 Subject: [PATCH 07/10] logs --- .../processor/builtin/impl/ai/weaviate/get_context.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go index 761e6c977..a18de1638 100644 --- a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go +++ b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go @@ -42,10 +42,13 @@ type getContextProcessor struct { sdk.UnimplementedProcessor cfg getContextConfig client *weaviate.Client + logger log.CtxLogger } -func NewGetContextProcessor(_ log.CtxLogger) sdk.Processor { - return &getContextProcessor{} +func NewGetContextProcessor(logger log.CtxLogger) sdk.Processor { + return &getContextProcessor{ + logger: logger.WithComponent("weaviate.getContextProcessor"), + } } func (p *getContextProcessor) Specification() (sdk.Specification, error) { @@ -166,6 +169,7 @@ func (p *getContextProcessor) getContext(ctx context.Context, targetEmbedding [] data := result.Data["Get"].(map[string]interface{})[p.cfg.ClassName].([]interface{}) sb := strings.Builder{} + p.logger.Info(ctx).Msg("found closest texts") fmt.Println("Top 5 Closest Texts:") for _, item := range data { itemMap := item.(map[string]interface{}) @@ -176,5 +180,8 @@ func (p *getContextProcessor) getContext(ctx context.Context, targetEmbedding [] sb.WriteRune('\n') } + p.logger.Info(ctx). + Str("context", sb.String()). + Msg("context built") return sb.String(), nil } From 956c9f6c237b44abe4703c82a7ab97db757071f3 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Fri, 13 Dec 2024 11:01:38 +0100 Subject: [PATCH 08/10] wrong list of embedding models --- pkg/plugin/processor/builtin/impl/ai/openai/embedding.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/processor/builtin/impl/ai/openai/embedding.go b/pkg/plugin/processor/builtin/impl/ai/openai/embedding.go index e02c66098..dbb3972db 100644 --- a/pkg/plugin/processor/builtin/impl/ai/openai/embedding.go +++ b/pkg/plugin/processor/builtin/impl/ai/openai/embedding.go @@ -35,7 +35,7 @@ const EmbeddingMetadataBase64 = "openai.embedding.base64" type embeddingProcConfig struct { APIKey string `json:"apiKey" validate:"required"` Endpoint string `json:"endpoint" default:"https://api.openai.com/v1"` - Model string `json:"model" validate:"required,inclusion=gpt-4|gpt-4-turbo|gpt-3.5-turbo|text-davinci-003|text-davinci-002|text-curie-001|text-babbage-001|text-ada-001"` + Model string `json:"model" validate:"required,inclusion=text-embedding-3-large|text-embedding-3-small|text-embedding-ada-002"` InputField string `json:"inputField" validate:"regex=^\\.(Payload|Key).*" default:".Payload.After"` } From 0c8a7032854cd5ad20f5794c43788e2511544449 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Fri, 13 Dec 2024 15:12:42 +0100 Subject: [PATCH 09/10] AI pipeline --- go.mod | 1 + go.sum | 2 + pkg/plugin/connector/builtin/registry.go | 14 ++++--- .../builtin/impl/ai/openai/embedding.go | 3 +- .../ai/openai/embedding_config_paramgen.go | 2 +- .../builtin/impl/ai/weaviate/get_context.go | 6 ++- .../weaviate/get_context_config_paramgen.go | 14 +++++++ pkg/plugin/processor/builtin/registry.go | 41 +++++++++++-------- 8 files changed, 57 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 4fd4dc471..fb22c38f2 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/Masterminds/sprig/v3 v3.3.0 github.com/NYTimes/gziphandler v1.1.1 github.com/bufbuild/buf v1.47.2 + github.com/conduitio-labs/conduit-connector-weaviate v0.1.2-0.20241125183412-ae12206633d7 github.com/conduitio/conduit-commons v0.5.0 github.com/conduitio/conduit-connector-file v0.9.0 github.com/conduitio/conduit-connector-generator v0.9.1 diff --git a/go.sum b/go.sum index c4c043381..8f572784c 100644 --- a/go.sum +++ b/go.sum @@ -237,6 +237,8 @@ github.com/ckaznocha/intrange v0.2.1/go.mod h1:7NEhVyf8fzZO5Ds7CRaqPEm52Ut83hsTi github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= +github.com/conduitio-labs/conduit-connector-weaviate v0.1.2-0.20241125183412-ae12206633d7 h1:DnugxVjYVSCX2pvLlQAlTK2AEopieV0LjUKtnAedStE= +github.com/conduitio-labs/conduit-connector-weaviate v0.1.2-0.20241125183412-ae12206633d7/go.mod h1:rIJkVcS9cfFcy5rrimYu3wzaQJypkmXDNMmWqJKbGeg= github.com/conduitio/conduit-commons v0.5.0 h1:28UIuOIo+6WvBZ4EU54KfPhSf44I1/Y65zQ9dC0Ps1E= github.com/conduitio/conduit-commons v0.5.0/go.mod h1:xyT6XpGvj79gdtsn3qaD2KxadhsAYS+mmBOdln08Wio= github.com/conduitio/conduit-connector-file v0.9.0 h1:bp6mU5hMbWS5QK9hh3Vt+AigXSPFfc62lzHZKJZkscg= diff --git a/pkg/plugin/connector/builtin/registry.go b/pkg/plugin/connector/builtin/registry.go index efd36f709..c5ce8c260 100644 --- a/pkg/plugin/connector/builtin/registry.go +++ b/pkg/plugin/connector/builtin/registry.go @@ -18,6 +18,7 @@ import ( "context" "runtime/debug" + weaviate "github.com/conduitio-labs/conduit-connector-weaviate" file "github.com/conduitio/conduit-connector-file" generator "github.com/conduitio/conduit-connector-generator" kafka "github.com/conduitio/conduit-connector-kafka" @@ -38,12 +39,13 @@ import ( // The key of the map is the import path of the module // containing the connector implementation. var DefaultBuiltinConnectors = map[string]sdk.Connector{ - "github.com/conduitio/conduit-connector-file": file.Connector, - "github.com/conduitio/conduit-connector-generator": generator.Connector, - "github.com/conduitio/conduit-connector-kafka": kafka.Connector, - "github.com/conduitio/conduit-connector-log": connLog.Connector, - "github.com/conduitio/conduit-connector-postgres": postgres.Connector, - "github.com/conduitio/conduit-connector-s3": s3.Connector, + "github.com/conduitio/conduit-connector-file": file.Connector, + "github.com/conduitio/conduit-connector-generator": generator.Connector, + "github.com/conduitio/conduit-connector-kafka": kafka.Connector, + "github.com/conduitio/conduit-connector-log": connLog.Connector, + "github.com/conduitio/conduit-connector-postgres": postgres.Connector, + "github.com/conduitio/conduit-connector-s3": s3.Connector, + "github.com/conduitio-labs/conduit-connector-weaviate": weaviate.Connector, } type blueprint struct { diff --git a/pkg/plugin/processor/builtin/impl/ai/openai/embedding.go b/pkg/plugin/processor/builtin/impl/ai/openai/embedding.go index dbb3972db..22820d81f 100644 --- a/pkg/plugin/processor/builtin/impl/ai/openai/embedding.go +++ b/pkg/plugin/processor/builtin/impl/ai/openai/embedding.go @@ -52,7 +52,7 @@ type embeddingProcessor struct { func NewEmbeddingProcessor(log log.CtxLogger) sdk.Processor { return &embeddingProcessor{ - logger: log.WithComponent("openai_embedding"), + logger: log.WithComponent("openai.embedding"), } } @@ -138,6 +138,7 @@ func (p *embeddingProcessor) Process(ctx context.Context, records []opencdc.Reco Msg("got embeddings") for i, record := range records { + // todo this is huge record.Metadata[EmbeddingMetadataBase64] = embeddings.Data[i].EmbeddingBase64 // todo add more metadata related to the embeddings out = append(out, sdk.SingleRecord(record)) diff --git a/pkg/plugin/processor/builtin/impl/ai/openai/embedding_config_paramgen.go b/pkg/plugin/processor/builtin/impl/ai/openai/embedding_config_paramgen.go index 25f153eb8..678dab366 100644 --- a/pkg/plugin/processor/builtin/impl/ai/openai/embedding_config_paramgen.go +++ b/pkg/plugin/processor/builtin/impl/ai/openai/embedding_config_paramgen.go @@ -46,7 +46,7 @@ func (embeddingProcConfig) Parameters() map[string]config.Parameter { Type: config.ParameterTypeString, Validations: []config.Validation{ config.ValidationRequired{}, - config.ValidationInclusion{List: []string{"gpt-4", "gpt-4-turbo", "gpt-3.5-turbo", "text-davinci-003", "text-davinci-002", "text-curie-001", "text-babbage-001", "text-ada-001"}}, + config.ValidationInclusion{List: []string{"text-embedding-3-large", "text-embedding-3-small", "text-embedding-ada-002"}}, }, }, } diff --git a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go index a18de1638..df6f5a055 100644 --- a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go +++ b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context.go @@ -34,6 +34,8 @@ import ( ) type getContextConfig struct { + Endpoint string `json:"endpoint" default:"localhost:18080"` + Scheme string `json:"scheme" default:"http"` ClassName string `json:"className" validate:"required"` ContextField string `json:"contextField" validate:"required"` } @@ -76,8 +78,8 @@ func (p *getContextProcessor) Configure(ctx context.Context, c config.Config) er func (p *getContextProcessor) Open(ctx context.Context) error { // Configure Weaviate client cfg := weaviate.Config{ - Host: "localhost:8080", // Replace with your Weaviate instance URL - Scheme: "http", + Host: p.cfg.Endpoint, // Replace with your Weaviate instance URL + Scheme: p.cfg.Scheme, } client, err := weaviate.NewClient(cfg) if err != nil { diff --git a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go index 4c74c3f5b..533be74ac 100644 --- a/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go +++ b/pkg/plugin/processor/builtin/impl/ai/weaviate/get_context_config_paramgen.go @@ -10,6 +10,8 @@ import ( const ( getContextConfigClassName = "className" getContextConfigContextField = "contextField" + getContextConfigEndpoint = "endpoint" + getContextConfigScheme = "scheme" ) func (getContextConfig) Parameters() map[string]config.Parameter { @@ -30,5 +32,17 @@ func (getContextConfig) Parameters() map[string]config.Parameter { config.ValidationRequired{}, }, }, + getContextConfigEndpoint: { + Default: "localhost:18080", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + getContextConfigScheme: { + Default: "http", + Description: "", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, } } diff --git a/pkg/plugin/processor/builtin/registry.go b/pkg/plugin/processor/builtin/registry.go index 3f428afb9..85cb6754b 100644 --- a/pkg/plugin/processor/builtin/registry.go +++ b/pkg/plugin/processor/builtin/registry.go @@ -40,25 +40,34 @@ import ( ) var DefaultBuiltinProcessors = map[string]ProcessorPluginConstructor{ - "avro.decode": avro.NewDecodeProcessor, - "avro.encode": avro.NewEncodeProcessor, - "base64.decode": base64.NewDecodeProcessor, - "base64.encode": base64.NewEncodeProcessor, - "custom.javascript": custom.NewJavascriptProcessor, - "error": impl.NewErrorProcessor, - "filter": impl.NewFilterProcessor, - "field.convert": field.NewConvertProcessor, - "field.exclude": field.NewExcludeProcessor, - "field.rename": field.NewRenameProcessor, - "field.set": field.NewSetProcessor, - "json.decode": json.NewDecodeProcessor, - "json.encode": json.NewEncodeProcessor, + "avro.decode": avro.NewDecodeProcessor, + "avro.encode": avro.NewEncodeProcessor, + + "base64.decode": base64.NewDecodeProcessor, + "base64.encode": base64.NewEncodeProcessor, + + "custom.javascript": custom.NewJavascriptProcessor, + "error": impl.NewErrorProcessor, + + "filter": impl.NewFilterProcessor, + + "field.convert": field.NewConvertProcessor, + "field.exclude": field.NewExcludeProcessor, + "field.rename": field.NewRenameProcessor, + "field.set": field.NewSetProcessor, + + "json.decode": json.NewDecodeProcessor, + "json.encode": json.NewEncodeProcessor, + "unwrap.debezium": unwrap.NewDebeziumProcessor, "unwrap.kafkaconnect": unwrap.NewKafkaConnectProcessor, "unwrap.opencdc": unwrap.NewOpenCDCProcessor, - "webhook.http": webhook.NewHTTPProcessor, - "openai.embedding": openai.NewEmbeddingProcessor, - "openai.prompt": openai.NewPromptProcessor, + + "webhook.http": webhook.NewHTTPProcessor, + + "openai.embedding": openai.NewEmbeddingProcessor, + "openai.prompt": openai.NewPromptProcessor, + "weaviate.getContext": weaviate.NewGetContextProcessor, } From 0d5038567ac65d0e92d18e77b9e3c8bd6064116a Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Fri, 13 Dec 2024 15:32:12 +0100 Subject: [PATCH 10/10] hack --- pkg/plugin/processor/builtin/impl/ai/openai/prompt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/processor/builtin/impl/ai/openai/prompt.go b/pkg/plugin/processor/builtin/impl/ai/openai/prompt.go index a4f12e00a..fc6a82f7c 100644 --- a/pkg/plugin/processor/builtin/impl/ai/openai/prompt.go +++ b/pkg/plugin/processor/builtin/impl/ai/openai/prompt.go @@ -30,7 +30,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/log" ) -const MetadataPromptContext = "openai.prompt.context" +const MetadataPromptContext = "weaviate.context" type promptProcConfig struct { APIKey string `json:"apiKey" validate:"required"`