POC for indexing images using LLM vision

server/ai/conversation.go

@@ -22,6 +22,7 @@ const (
 )
 
 type File struct {
+	ID       string
 	MimeType string
 	Size     int64
 	Reader   io.Reader

server/plugin.go

@@ -195,6 +195,12 @@ func (p *Plugin) MessageHasBeenPosted(c *plugin.Context, post *model.Post) {
 			p.pluginAPI.Log.Error(err.Error())
 		}
 	}
+
+	if len(post.FileIds) > 0 {
+		if err := p.handleImageIndexing(post); err != nil {
+			p.pluginAPI.Log.Error(err.Error())
+		}
+	}
 }
 
 const (
@@ -297,3 +303,38 @@ func (p *Plugin) handleDMs(bot *Bot, channel *model.Channel, postingUser *model.
 
 	return nil
 }
+
+func (p *Plugin) handleImageIndexing(post *model.Post) error {
+	bot := p.GetBotByUsernameOrFirst("copilot")
+	if bot == nil {
+		return fmt.Errorf("no bot found")
+	}
+
+	if !bot.cfg.EnableVision {
+		return fmt.Errorf("vision is disabled")
+	}
+
+	llm := p.getLLM(bot.cfg)
+
+	aiFiles := p.PostFilesToAIFiles(bot, post)
+	for _, file := range aiFiles {
+		result, err := llm.ChatCompletionNoStream(ai.BotConversation{
+			Posts: []ai.Post{{
+				Role:    ai.PostRoleUser,
+				Message: "Write a descripton and a list of keywords that describe this image. Use plaintext only. Write only the description and the keywords. The keywords should be separted with commas.",
+				Files:   []ai.File{file},
+			}},
+		})
+		if err != nil {
+			p.pluginAPI.Log.Warn("failed to generate image description", "error", err, "file_id", file.ID)
+			continue
+		}
+
+		if err := p.pluginAPI.File.SetSearchableContent(file.ID, result); err != nil {
+			p.pluginAPI.Log.Warn("failed to set searchable content", "error", err, "file_id", file.ID)
+			continue
+		}
+	}
+
+	return nil
+}

server/post_processing.go

@@ -381,7 +381,7 @@ type WorkerResult struct {
 	return nil
 }*/
 
-func (p *Plugin) PostToAIPost(bot *Bot, post *model.Post) ai.Post {
+func (p *Plugin) PostFilesToAIFiles(bot *Bot, post *model.Post) []ai.File {
 	var files []ai.File
 	if bot.cfg.EnableVision {
 		files = make([]ai.File, 0, len(post.FileIds))
@@ -397,17 +397,22 @@ func (p *Plugin) PostToAIPost(bot *Bot, post *model.Post) ai.Post {
 				continue
 			}
 			files = append(files, ai.File{
+				ID:       fileID,
 				Reader:   file,
 				MimeType: fileInfo.MimeType,
 				Size:     fileInfo.Size,
 			})
 		}
 	}
 
+	return files
+}
+
+func (p *Plugin) PostToAIPost(bot *Bot, post *model.Post) ai.Post {
 	return ai.Post{
 		Role:    ai.GetPostRole(bot.mmBot.UserId, post),
 		Message: ai.FormatPostBody(post),
-		Files:   files,
+		Files:   p.PostFilesToAIFiles(bot, post),
 	}
 }