Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: ignore empty chunk from chunk pipeline #114

Merged
merged 1 commit into from
Oct 14, 2024
Merged

fix: ignore empty chunk from chunk pipeline #114

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions pkg/service/mgmt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func (s *Service) GetNamespaceTier(ctx context.Context, ns *resource.Namespace)
if ok && statusError.Code() == codes.Unimplemented {
// Handle the case where the method is not implemented on the server
log.Warn("GetUserSubscriptionAdmin is not implemented. Assuming enterprise tier")
return TierEntriprise, nil
return TierEnterprise, nil
} else {
// Handle other errors
return "", fmt.Errorf("failed to get user subscription: %w", err)
Expand All @@ -90,7 +90,7 @@ func (s *Service) GetNamespaceTier(ctx context.Context, ns *resource.Namespace)
if ok && statusError.Code() == codes.Unimplemented {
// handle the case where the method is not implemented on the server
log.Warn("GetUserSubscriptionAdmin is not implemented. Assuming enterprise tier")
return TierEntriprise, nil
return TierEnterprise, nil
} else {
// handle other errors
return "", fmt.Errorf("failed to get organization subscription: %w", err)
Expand All @@ -102,7 +102,7 @@ func (s *Service) GetNamespaceTier(ctx context.Context, ns *resource.Namespace)
} else if sub.GetSubscription().Plan == mgmtPB.OrganizationSubscription_PLAN_TEAM {
return TierTeam, nil
} else if sub.GetSubscription().Plan == mgmtPB.OrganizationSubscription_PLAN_ENTERPRISE {
return TierEntriprise, nil
return TierEnterprise, nil
}
return "", fmt.Errorf("unknown organization subscription plan: %v", sub.GetSubscription().Plan)
default:
Expand All @@ -116,7 +116,7 @@ const (
TierFree Tier = "free"
TierPro Tier = "pro"
TierTeam Tier = "team"
TierEntriprise Tier = "enterprise"
TierEnterprise Tier = "enterprise"
)

func (t Tier) String() string {
Expand All @@ -132,7 +132,7 @@ func (t Tier) GetPrivateCatalogLimit() int {
case TierTeam:
// unlimited
return math.MaxInt
case TierEntriprise:
case TierEnterprise:
// unlimited
return math.MaxInt

Expand All @@ -154,7 +154,7 @@ func (t Tier) GetFileStorageTotalQuota() (int, string) {
case TierTeam:
// 2GB
return 2 * gb, "2GB"
case TierEntriprise:
case TierEnterprise:
// unlimited
return math.MaxInt, "unlimited"
}
Expand Down
19 changes: 15 additions & 4 deletions pkg/service/pipeline.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,8 @@ type Chunk = struct {
Tokens int
}

// SplitMarkdownPipe using splitting pipeline to split markdown and consume caller's credits
// SplitMarkdownPipe triggers the markdown splitting pipeline, processes the markdown text, and deducts credits from the caller's account.
// It sets up the necessary metadata, triggers the pipeline, and processes the response to return the non-empty chunks.
func (s *Service) SplitMarkdownPipe(ctx context.Context, caller uuid.UUID, requester uuid.UUID, markdown string) ([]Chunk, error) {
var md metadata.MD
if requester != uuid.Nil {
Expand Down Expand Up @@ -157,7 +158,8 @@ func (s *Service) SplitMarkdownPipe(ctx context.Context, caller uuid.UUID, reque
if err != nil {
return nil, err
}
// remove the empty chunk
// remove the empty chunk.
// note: this is a workaround for the pipeline bug that sometimes returns empty chunks.
var filteredResult []Chunk
for _, chunk := range result {
if chunk.Text != "" {
Expand Down Expand Up @@ -198,7 +200,8 @@ func GetChunksFromResponse(resp *pipelinePb.TriggerNamespacePipelineReleaseRespo
return chunks, nil
}

// SplitTextPipe using splitting pipeline to split text and consume caller's credits
// SplitTextPipe splits the input text into chunks using the splitting pipeline and consumes the caller's credits.
// It sets up the necessary metadata, triggers the pipeline, and processes the response to return the non-empty chunks.
func (s *Service) SplitTextPipe(ctx context.Context, caller uuid.UUID, requester uuid.UUID, text string) ([]Chunk, error) {
var md metadata.MD
if requester != uuid.Nil {
Expand Down Expand Up @@ -237,7 +240,15 @@ func (s *Service) SplitTextPipe(ctx context.Context, caller uuid.UUID, requester
if err != nil {
return nil, fmt.Errorf("failed to get chunks from response: %w", err)
}
return result, nil
// remove the empty chunk.
// note: this is a workaround for the pipeline bug that sometimes returns empty chunks.
var filteredResult []Chunk
for _, chunk := range result {
if chunk.Text != "" {
filteredResult = append(filteredResult, chunk)
}
}
return filteredResult, nil
}

// EmbeddingTextPipe uses the embedding pipeline to convert text into vectors and consume caller's credits.
Expand Down
Loading