diff --git a/cmd/assets/index.html b/cmd/assets/index.html deleted file mode 100644 index 0963ec5..0000000 --- a/cmd/assets/index.html +++ /dev/null @@ -1,160 +0,0 @@ - - - - - - Ứng dụng Dịch Sách - - - -
-
-

Ứng dụng Dịch Sách

-
- - - - 16px -
-
-
-
- - - - \ No newline at end of file diff --git a/cmd/mark.go b/cmd/mark.go index a1037b8..a440dd8 100644 --- a/cmd/mark.go +++ b/cmd/mark.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "os/signal" + "regexp" "runtime" "strings" "syscall" @@ -18,17 +19,17 @@ import ( "github.com/nguyenvanduocit/epubtrans/pkg/util" ) -var blacklist = []string{ - "math", - "figure", - "pre", - "code", - "head", - "script", - "style", - "template", - "svg", - "noscript", +var blacklist = map[string]bool{ + "math": true, + "figure": true, + "pre": true, + "code": true, + "head": true, + "script": true, + "style": true, + "template": true, + "svg": true, + "noscript": true, } var Mark = &cobra.Command{ @@ -69,7 +70,14 @@ func runMark(cmd *cobra.Command, args []string) error { return err } - workers, _ := cmd.Flags().GetInt("workers") + workers, err := cmd.Flags().GetInt("workers") + if err != nil { + return fmt.Errorf("getting workers flag: %w", err) + } + + if workers <= 0 { + return fmt.Errorf("workers must be greater than 0") + } return processor.ProcessEpub(ctx, unzipPath, processor.Config{ Workers: workers, @@ -79,6 +87,10 @@ func runMark(cmd *cobra.Command, args []string) error { } func markContentInFile(ctx context.Context, filePath string) error { + if filePath == "" { + return fmt.Errorf("filePath cannot be empty") + } + f, err := os.Open(filePath) if err != nil { return fmt.Errorf("opening file %s: %w", filePath, err) @@ -105,6 +117,8 @@ func markContentInFile(ctx context.Context, filePath string) error { return nil } +const minContentLength = 2 + func processNode(n *html.Node) { if n.Type == html.ElementNode { // Skip if already marked @@ -115,16 +129,14 @@ func processNode(n *html.Node) { } // Skip if blacklisted - for _, tag := range blacklist { - if n.Data == tag { - return - } + if blacklist[n.Data] { + return } if !isContainer(n) { content := extractTextContent(n) - if util.IsEmptyOrWhitespace(content) || len(content) <= 1 || util.IsNumeric(content) { - fmt.Println("Skipping non-empty or non-numeric content: ", content) + if util.IsEmptyOrWhitespace(content) || len(content) <= minContentLength || util.IsNumeric(content) || isSpecialContent(content) { + fmt.Printf("Skipping content in <%s> tag: %q\n", n.Data, content) return } else { // Mark this node @@ -145,6 +157,12 @@ func processNode(n *html.Node) { } } +var re = regexp.MustCompile(`^[*=\-_.,:;!?#\s]+$`) + +func isSpecialContent(content string) bool { + return re.MatchString(content) +} + func isContainer(n *html.Node) bool { if n.Type != html.ElementNode { return false diff --git a/cmd/pack.go b/cmd/pack.go index d63c112..e614b87 100644 --- a/cmd/pack.go +++ b/cmd/pack.go @@ -14,6 +14,12 @@ import ( "github.com/spf13/cobra" ) +const ( + defaultBufferSize = 32 * 1024 // 32KB + channelBufferSize = 100 + defaultSuffix = "-bilangual.epub" +) + var Pack = &cobra.Command{ Use: "pack [unpackedEpubPath]", Short: "Create an EPUB file from an unpacked directory", @@ -43,10 +49,18 @@ func runPack(cmd *cobra.Command, args []string) error { func packFiles(srcDir string, outputPath string) error { if outputPath == "" { - outputPath = getUniqueFilename(srcDir + "-bilangual.epub") + outputPath = getUniqueFilename(srcDir + defaultSuffix) } else { outputPath = getUniqueFilename(outputPath) } + + // Validate source directory + if info, err := os.Stat(srcDir); err != nil || !info.IsDir() { + return fmt.Errorf("invalid source directory: %w", err) + } + + progress := &packingProgress{} + fmt.Printf("Creating zip file: %s\n", outputPath) newZipFile, err := os.Create(outputPath) @@ -58,11 +72,8 @@ func packFiles(srcDir string, outputPath string) error { zipWriter := zip.NewWriter(newZipFile) defer zipWriter.Close() - fileCount := int64(0) - totalSize := int64(0) - // Create a buffered channel for file info - fileInfoChan := make(chan fileInfo, 100) + fileInfoChan := make(chan fileInfo, channelBufferSize) // Start a single goroutine to write to the zip file var wg sync.WaitGroup @@ -71,14 +82,16 @@ func packFiles(srcDir string, outputPath string) error { go func() { defer wg.Done() for fi := range fileInfoChan { + if err := validateFile(fi.path, fi.info); err != nil { + writeErr = err + return + } - err := addFileToZip(zipWriter, fi.path, fi.relPath, fi.info) - if err != nil { + if err := addFileToZip(zipWriter, fi, progress); err != nil { writeErr = err return } - atomic.AddInt64(&fileCount, 1) - atomic.AddInt64(&totalSize, fi.info.Size()) + fmt.Printf("Added file: %s (%.2f KB)\n", fi.relPath, float64(fi.info.Size())/1024) } }() @@ -114,8 +127,8 @@ func packFiles(srcDir string, outputPath string) error { } fmt.Printf("\nZip creation complete:\n") - fmt.Printf("Total files: %d\n", fileCount) - fmt.Printf("Total size: %.2f MB\n", float64(totalSize)/(1024*1024)) + fmt.Printf("Total files: %d\n", progress.fileCount) + fmt.Printf("Total size: %.2f MB\n", float64(progress.totalSize)/(1024*1024)) fmt.Printf("Output file: %s\n", outputPath) return nil @@ -127,31 +140,42 @@ type fileInfo struct { info os.FileInfo } -func addFileToZip(zipWriter *zip.Writer, filePath, relPath string, info os.FileInfo) error { - zipFileHeader, err := zip.FileInfoHeader(info) +type packingProgress struct { + fileCount int64 + totalSize int64 + mu sync.Mutex +} + +func (p *packingProgress) update(size int64) { + atomic.AddInt64(&p.fileCount, 1) + atomic.AddInt64(&p.totalSize, size) +} + +func addFileToZip(zipWriter *zip.Writer, fi fileInfo, progress *packingProgress) error { + zipFileHeader, err := zip.FileInfoHeader(fi.info) if err != nil { return fmt.Errorf("failed to create file header: %w", err) } - zipFileHeader.Name = relPath - zipFileHeader.Method = chooseCompressionMethod(filePath) + zipFileHeader.Name = fi.relPath + zipFileHeader.Method = chooseCompressionMethod(fi.path) writer, err := zipWriter.CreateHeader(zipFileHeader) if err != nil { return fmt.Errorf("failed to create zip entry: %w", err) } - file, err := os.Open(filePath) + file, err := os.Open(fi.path) if err != nil { return fmt.Errorf("failed to open file: %w", err) } defer file.Close() - buf := make([]byte, 32*1024) // 32KB buffer - _, err = io.CopyBuffer(writer, file, buf) - if err != nil { + buf := make([]byte, defaultBufferSize) + if _, err := io.CopyBuffer(writer, file, buf); err != nil { return fmt.Errorf("failed to write file to zip: %w", err) } + progress.update(fi.info.Size()) return nil } @@ -205,3 +229,13 @@ func getUniqueFilename(filename string) string { counter++ } } + +func validateFile(path string, info os.FileInfo) error { + if info.Size() == 0 { + return fmt.Errorf("empty file detected: %s", path) + } + if strings.Contains(path, "..") { + return fmt.Errorf("potential directory traversal detected: %s", path) + } + return nil +} diff --git a/cmd/serve.go b/cmd/serve.go index 0227863..f612d2b 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -132,13 +132,13 @@ type TranslateAIRequest struct { } // Add this function to call the AI translation service (you'll need to implement this) -func translateWithAI(content string, instructions string) (string, error) { +func translateWithAI(content string, instructions string, bookTitle string) (string, error) { ctx := context.Background() // Create an Anthropic translator anthropicTranslator, err := translator.GetAnthropicTranslator(&translator.Config{ APIKey: os.Getenv("ANTHROPIC_KEY"), - Model: anthropic.ModelClaude3Dot5Sonnet20240620, // You might want to make this configurable + Model: string(anthropic.ModelClaude3Dot5SonnetLatest), // You might want to make this configurable Temperature: 0.7, MaxTokens: 8192, }) @@ -147,7 +147,7 @@ func translateWithAI(content string, instructions string) (string, error) { } // Translate the content - translatedContent, err := anthropicTranslator.Translate(ctx, instructions, content, "english", "vietnamese") + translatedContent, err := anthropicTranslator.Translate(ctx, instructions, content, "english", "vietnamese", bookTitle) if err != nil { return "", fmt.Errorf("translation error: %v", err) } @@ -163,6 +163,23 @@ func runServe(cmd *cobra.Command, args []string) error { return fmt.Errorf("the specified directory does not exist: %s", unpackedEpubPath) } + // Parse the package to get book information + container, err := loader.ParseContainer(unpackedEpubPath) + if err != nil { + return err + } + + opfPath := filepath.Join(unpackedEpubPath, container.Rootfile.FullPath) + pkg, err := loader.ParsePackage(opfPath) + if err != nil { + return fmt.Errorf("error parsing package: %v", err) + } + + // Get the book title + bookTitle := pkg.Metadata.Title + + slog.Info("Book title: " + bookTitle) + app := fiber.New(fiber.Config{ DisableStartupMessage: true, }) @@ -202,11 +219,6 @@ func runServe(cmd *cobra.Command, args []string) error { return c.Send(body) }) - container, err := loader.ParseContainer(unpackedEpubPath) - if err != nil { - return err - } - contentDirPath := path.Dir(path.Join(unpackedEpubPath, container.Rootfile.FullPath)) app.Get("/toc.html", func(c *fiber.Ctx) error { @@ -413,12 +425,11 @@ func runServe(cmd *cobra.Command, args []string) error { instructment := req.Instructions - if len(currentTranslatedContent) == 0{ - instructment = `Previous translation:\n\n` + currentTranslatedContent + `\n\n` + instructment + if len(currentTranslatedContent) > 0 { + instructment = fmt.Sprintf("Previous translation:\n\n%s\n\n%s", currentTranslatedContent, instructment) } - - translatedContent, err := translateWithAI(originalContent, instructment) + translatedContent, err := translateWithAI(originalContent, instructment, bookTitle) if err != nil { return c.Status(500).JSON(fiber.Map{"error": "Translation failed"}) } diff --git a/cmd/translate.go b/cmd/translate.go index 257bc21..0eac168 100644 --- a/cmd/translate.go +++ b/cmd/translate.go @@ -8,6 +8,7 @@ import ( "math/rand" "os" "os/signal" + "path" "strings" "sync" "syscall" @@ -15,6 +16,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/liushuangls/go-anthropic/v2" + "github.com/nguyenvanduocit/epubtrans/pkg/loader" "github.com/nguyenvanduocit/epubtrans/pkg/processor" "github.com/nguyenvanduocit/epubtrans/pkg/translator" "github.com/nguyenvanduocit/epubtrans/pkg/util" @@ -47,7 +49,7 @@ Make sure to provide the path to the unpacked EPUB directory and the desired lan func init() { Translate.Flags().StringVar(&sourceLanguage, "source", "English", "source language") Translate.Flags().StringVar(&targetLanguage, "target", "Vietnamese", "target language") - Translate.Flags().String("model", anthropic.ModelClaude3Dot5Sonnet20240620, "Anthropic model to use") + Translate.Flags().String("model", string(anthropic.ModelClaude3Dot5SonnetLatest), "Anthropic model to use") } type elementToTranslate struct { @@ -56,6 +58,12 @@ type elementToTranslate struct { doc *goquery.Document totalElements int index int + content string +} + +type translationBatch struct { + elements []elementToTranslate + filePath string } var fileLocks = make(map[string]*sync.Mutex) @@ -93,6 +101,12 @@ func runTranslate(cmd *cobra.Command, args []string) error { return err } + // Extract book name from EPUB metadata + bookName, err := extractBookName(unzipPath) + if err != nil { + return fmt.Errorf("error extracting book name: %v", err) + } + limiter := rate.NewLimiter(rate.Every(time.Minute/50), 10) anthropicTranslator, err := translator.GetAnthropicTranslator(&translator.Config{ @@ -109,7 +123,7 @@ func runTranslate(cmd *cobra.Command, args []string) error { elementChan := make(chan elementToTranslate, 10) doneChan := make(chan struct{}) - go processElements(ctx, elementChan, doneChan, anthropicTranslator, limiter) + go processElements(ctx, elementChan, doneChan, anthropicTranslator, limiter, bookName) // 1 worker and 1 job at a time, mean 1 file at a time err = processor.ProcessEpub(ctx, unzipPath, processor.Config{ @@ -126,28 +140,160 @@ func runTranslate(cmd *cobra.Command, args []string) error { return err } -func processElements(ctx context.Context, elementChan <-chan elementToTranslate, doneChan chan<- struct{}, anthropicTranslator translator.Translator, limiter *rate.Limiter) { +func extractBookName(unzipPath string) (string, error) { + container, err := loader.ParseContainer(unzipPath) + if err != nil { + return "", fmt.Errorf("failed to parse container: %w", err) + } + + packagePath := path.Join(unzipPath, container.Rootfile.FullPath) + pkg, err := loader.ParsePackage(packagePath) + if err != nil { + return "", fmt.Errorf("failed to parse package: %w", err) + } + + return pkg.Metadata.Title, nil +} + +func processElements(ctx context.Context, elementChan <-chan elementToTranslate, doneChan chan<- struct{}, anthropicTranslator translator.Translator, limiter *rate.Limiter, bookName string) { defer close(doneChan) + fmt.Println("Starting element processing...") + + // Create a batch map to group elements by file + batchMap := make(map[string]*translationBatch) + maxBatchLength := 4000 // Maximum characters per batch + for element := range elementChan { - select { - case <-ctx.Done(): - return - default: - fmt.Printf("Processing %s:%s\n", element.filePath, element.contentEl.AttrOr(util.ContentIdKey, "")) - if translated := translateElement(ctx, element, anthropicTranslator, limiter); translated { - fileLock := getFileLock(element.filePath) - fileLock.Lock() - if err := writeContentToFile(element.filePath, element.doc); err != nil { - fmt.Printf("Error writing to file: %v\n", err) - } - fileLock.Unlock() + fmt.Printf("Processing element from file: %s (%d/%d)\n", + path.Base(element.filePath), element.index+1, element.totalElements) + + // Get HTML content + htmlContent, err := element.contentEl.Html() + if err != nil || len(htmlContent) <= 1 { + continue + } + element.content = htmlContent + + // Add to batch if it won't exceed max length, otherwise process current batch + batch := batchMap[element.filePath] + if batch == nil { + batch = &translationBatch{ + elements: []elementToTranslate{}, + filePath: element.filePath, + } + batchMap[element.filePath] = batch + } + + currentBatchLength := getBatchLength(batch) + if currentBatchLength+len(htmlContent) > maxBatchLength && len(batch.elements) > 0 { + // Process current batch before adding new element + fmt.Printf("Processing batch for file %s (length: %d, elements: %d)\n", + path.Base(element.filePath), currentBatchLength, len(batch.elements)) + processBatch(ctx, batch, anthropicTranslator, limiter, bookName) + + // Start new batch with current element + batchMap[element.filePath] = &translationBatch{ + elements: []elementToTranslate{element}, + filePath: element.filePath, } + } else { + batch.elements = append(batch.elements, element) + } + } + + // Process remaining batches + fmt.Printf("Processing remaining batches (count: %d)\n", len(batchMap)) + for filePath, batch := range batchMap { + if len(batch.elements) > 0 { + fmt.Printf("Processing final batch for file %s (elements: %d)\n", + path.Base(filePath), len(batch.elements)) + processBatch(ctx, batch, anthropicTranslator, limiter, bookName) } } } +func getBatchLength(batch *translationBatch) int { + var length int + for _, element := range batch.elements { + length += len(element.content) + } + return length +} + +func processBatch(ctx context.Context, batch *translationBatch, anthropicTranslator translator.Translator, limiter *rate.Limiter, bookName string) { + if len(batch.elements) == 0 { + return + } + + fmt.Printf("Translating batch from file %s (segments: %d)\n", + path.Base(batch.filePath), len(batch.elements)) + + // Combine contents with more distinct markers and instructions + var combinedContent strings.Builder + combinedContent.WriteString("Translate the following HTML segments. Each segment is marked with BEGIN_SEGMENT_X and END_SEGMENT_X markers. Preserve these markers exactly in your response and maintain all HTML tags.\n\n") + + for i, element := range batch.elements { + combinedContent.WriteString(fmt.Sprintf("\n%s\n\n\n", i, element.content, i)) + } + + // Translate combined content + translatedContent, err := retryTranslate(ctx, anthropicTranslator, limiter, combinedContent.String(), sourceLanguage, targetLanguage, bookName) + if err != nil { + fmt.Printf("Batch translation error: %v\n", err) + return + } + + // Split translated content and process individual elements + translations := splitTranslations(translatedContent) + if len(translations) != len(batch.elements) { + fmt.Printf("Translation segments mismatch for %s: got %d, expected %d\n", + path.Base(batch.filePath), len(translations), len(batch.elements)) + return + } + + fmt.Printf("Successfully translated batch from %s, writing to file...\n", + path.Base(batch.filePath)) + + fileLock := getFileLock(batch.filePath) + fileLock.Lock() + defer fileLock.Unlock() + + for i, element := range batch.elements { + if isTranslationValid(element.content, translations[i]) { + if err := manipulateHTML(element.contentEl, targetLanguage, translations[i]); err != nil { + fmt.Printf("HTML manipulation error: %v\n", err) + continue + } + } + } + + if err := writeContentToFile(batch.filePath, batch.elements[0].doc); err != nil { + fmt.Printf("Error writing to file: %v\n", err) + } +} + +func splitTranslations(translatedContent string) []string { + var translations []string + segments := strings.Split(translatedContent, " 1 { + // Extract segment number and content + segmentContent := parts[0] + if idx := strings.Index(segmentContent, ">"); idx != -1 { + content := strings.TrimSpace(segmentContent[idx+1:]) + translations = append(translations, content) + } + } + } + + return translations +} + func processFile(ctx context.Context, filePath string, elementChan chan<- elementToTranslate) error { + fmt.Printf("\nProcessing file: %s\n", path.Base(filePath)) + doc, err := openAndReadFile(filePath) if err != nil { return err @@ -159,9 +305,13 @@ func processFile(ctx context.Context, filePath string, elementChan chan<- elemen needToBeTranslateEls := doc.Find(selector) if needToBeTranslateEls.Length() == 0 { + fmt.Printf("No elements to translate in %s\n", path.Base(filePath)) return nil } + fmt.Printf("Found %d elements to translate in %s\n", + needToBeTranslateEls.Length(), path.Base(filePath)) + needToBeTranslateEls.Each(func(i int, contentEl *goquery.Selection) { select { case <-ctx.Done(): @@ -198,39 +348,7 @@ func ensureUTF8Charset(doc *goquery.Document) { } -func translateElement(ctx context.Context, element elementToTranslate, anthropicTranslator translator.Translator, limiter *rate.Limiter) bool { - if ctx.Err() != nil { - return false - } - - contentID := element.contentEl.AttrOr(util.ContentIdKey, "") - htmlToTranslate, err := element.contentEl.Html() - if err != nil || len(htmlToTranslate) <= 1 { - return false - } - - translatedContent, err := retryTranslate(ctx, anthropicTranslator, limiter, htmlToTranslate, sourceLanguage, targetLanguage) - if err != nil { - fmt.Printf("\t\tTranslation error: %v\n", err) - return false - } - - if !isTranslationValid(htmlToTranslate, translatedContent) { - fmt.Printf("\t\tInvalid translation for: %s\n", contentID) - fmt.Printf("\t\tOriginal: %s\n", htmlToTranslate) - fmt.Printf("\t\tTranslated: %s\n", translatedContent) - return false - } - - if err = manipulateHTML(element.contentEl, targetLanguage, translatedContent); err != nil { - fmt.Printf("HTML manipulation error: %v\n", err) - return false - } - - return true -} - -func retryTranslate(ctx context.Context, t translator.Translator, limiter *rate.Limiter, content, sourceLang, targetLang string) (string, error) { +func retryTranslate(ctx context.Context, t translator.Translator, limiter *rate.Limiter, content, sourceLang, targetLang, bookName string) (string, error) { maxRetries := 3 baseDelay := time.Second @@ -244,7 +362,7 @@ func retryTranslate(ctx context.Context, t translator.Translator, limiter *rate. return "", fmt.Errorf("rate limiter error: %w", err) } - translatedContent, err := t.Translate(ctx, "", content, sourceLang, targetLang) + translatedContent, err := t.Translate(ctx, "", content, sourceLang, targetLang, bookName) if err == nil { return translatedContent, nil } @@ -273,73 +391,42 @@ func isTranslationValid(original, translated string) bool { return true } - if countWords(translated) > countWords(original)*5 { + // Check if translation is suspiciously long or short + originalWords := countWords(original) + translatedWords := countWords(translated) + if translatedWords > originalWords*5 || translatedWords < originalWords/5 { return false } - if isHtmlDifferent(original, translated) { + // Ensure all HTML tags are preserved + originalTags := extractHTMLTags(original) + translatedTags := extractHTMLTags(translated) + + if len(originalTags) != len(translatedTags) { return false } + + for i := range originalTags { + if originalTags[i] != translatedTags[i] { + return false + } + } return true } -func isHtmlDifferent(html1, html2 string) bool { - doc1, err := goquery.NewDocumentFromReader(strings.NewReader(html1)) - if err != nil { - return false - } - - doc2, err := goquery.NewDocumentFromReader(strings.NewReader(html2)) +func extractHTMLTags(html string) []string { + var tags []string + doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) if err != nil { - return false - } - - // loop to compare - return compareNodes(doc1.Contents(), doc2.Contents()) -} - -func compareNodes(nodes1, nodes2 *goquery.Selection) bool { - if nodes1.Length() != nodes2.Length() { - return true - } - - for i := range nodes1.Nodes { - node1 := nodes1.Eq(i) - node2 := nodes2.Eq(i) - - if node1.Nodes[0].Type != node2.Nodes[0].Type { - return true - } - - if node1.Nodes[0].Type == 1 { - if node1.Nodes[0].Data != node2.Nodes[0].Data { - return true - } - - if node1.Nodes[0].Attr != nil && node2.Nodes[0].Attr != nil { - if len(node1.Nodes[0].Attr) != len(node2.Nodes[0].Attr) { - return true - } - - for j := range node1.Nodes[0].Attr { - if node1.Nodes[0].Attr[j].Key != node2.Nodes[0].Attr[j].Key || node1.Nodes[0].Attr[j].Val != node2.Nodes[0].Attr[j].Val { - return true - } - } - } - - if compareNodes(node1.Contents(), node2.Contents()) { - return true - } - } + return tags } - - return false -} - -func countWords(s string) int { - return len(strings.Fields(s)) + + doc.Find("*").Each(func(i int, s *goquery.Selection) { + tags = append(tags, goquery.NodeName(s)) + }) + + return tags } func manipulateHTML(doc *goquery.Selection, targetLang, translatedContent string) error { @@ -375,3 +462,8 @@ func writeContentToFile(filePath string, doc *goquery.Document) error { _, err = file.WriteString(html) return err } + +func countWords(text string) int { + words := strings.Fields(text) + return len(words) +} \ No newline at end of file diff --git a/cmd/unpack.go b/cmd/unpack.go index 370cf61..1d22f7b 100644 --- a/cmd/unpack.go +++ b/cmd/unpack.go @@ -26,13 +26,15 @@ var Unpack = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { zipPath := args[0] unzipPath, err := util.GetUnzipDestination(zipPath) - - cmd.Println("Unzipping to:", unzipPath) if err != nil { - return err + return fmt.Errorf("failed to determine unzip destination: %w", err) } - if err := unzipBook(zipPath, unzipPath); err != nil { - return err + cmd.Println("Unzipping to:", unzipPath) + if err := unzipBook(zipPath, unzipPath, func(format string, a ...interface{}) error { + cmd.Printf(format, a...) + return nil + }); err != nil { + return fmt.Errorf("failed to unzip book: %w", err) } cmd.Println("Unpacking completed successfully.") @@ -40,51 +42,49 @@ var Unpack = &cobra.Command{ }, } -func unzipBook(source, destination string) error { +func unzipBook(source, destination string, progress func(format string, a ...interface{}) error) error { r, err := zip.OpenReader(source) if err != nil { - return err + return fmt.Errorf("failed to open zip file: %w", err) } defer r.Close() - // Ensure the destination directory exists if err := os.MkdirAll(destination, os.ModePerm); err != nil { - return err + return fmt.Errorf("failed to create destination directory: %w", err) } for _, f := range r.File { - fmt.Println("Unzipping file:", f.Name) - fpath := filepath.Join(destination, f.Name) - - if f.FileInfo().IsDir() { - os.MkdirAll(fpath, os.ModePerm) - continue - } - - if err := os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { - return err + if err := extractFile(f, destination, progress); err != nil { + return fmt.Errorf("failed to extract file %s: %w", f.Name, err) } + } + return nil +} - outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return err - } +func extractFile(f *zip.File, destination string, progress func(format string, a ...interface{}) error) error { + progress("Unzipping file: %s\n", f.Name) + fpath := filepath.Join(destination, f.Name) - rc, err := f.Open() - if err != nil { - outFile.Close() - return err - } + if f.FileInfo().IsDir() { + return os.MkdirAll(fpath, os.ModePerm) + } - _, err = io.Copy(outFile, rc) + if err := os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { + return err + } - // Close the file without masking the previous error - outFile.Close() - rc.Close() + outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + defer outFile.Close() - if err != nil { - return err - } + rc, err := f.Open() + if err != nil { + return err } - return nil + defer rc.Close() + + _, err = io.Copy(outFile, rc) + return err } diff --git a/cmd/upgrade.go b/cmd/upgrade.go index bae817a..70c933f 100644 --- a/cmd/upgrade.go +++ b/cmd/upgrade.go @@ -14,6 +14,7 @@ import ( "path/filepath" "runtime" "strings" + "time" "github.com/Masterminds/semver/v3" "github.com/spf13/cobra" @@ -36,6 +37,12 @@ type GithubRelease struct { } `json:"assets"` } +type VersionInfo struct { + Current *semver.Version + Latest *semver.Version + Release *GithubRelease +} + func parseVersion(fullVersion string) (*semver.Version, error) { parts := strings.SplitN(fullVersion, "-", 2) if len(parts) < 1 { @@ -46,42 +53,35 @@ func parseVersion(fullVersion string) (*semver.Version, error) { } func runSelfUpgrade(cmd *cobra.Command, args []string) error { - currentVersion, err := parseVersion(Root.Version) - if err != nil { - return fmt.Errorf("invalid current version: %w", err) - } - cmd.Println("Checking for updates...") - latestRelease, err := getLatestRelease() + + info, err := checkVersion(cmd) if err != nil { - return fmt.Errorf("failed to check for updates: %w", err) - } - - latestVersion, err := semver.NewVersion(strings.TrimPrefix(latestRelease.TagName, "v")) - if err != nil { - return fmt.Errorf("invalid latest version: %w", err) + return err } - if !latestVersion.GreaterThan(currentVersion) { + if !info.Latest.GreaterThan(info.Current) { cmd.Println("You are already running the latest version.") return nil } - cmd.Printf("Current version: %s\n", currentVersion) - cmd.Printf("New version available: %s\n", latestVersion) - cmd.Print("Do you want to update? (y/n): ") - var response string - fmt.Scanln(&response) - if strings.ToLower(response) != "y" { + cmd.Printf("Current version: %s\n", info.Current) + cmd.Printf("New version available: %s\n", info.Latest) + + proceed, err := getUserConfirmation(cmd) + if err != nil { + return err + } + if !proceed { cmd.Println("Upgrade cancelled.") return nil } - if err := downloadAndInstall(cmd, latestRelease); err != nil { + if err := downloadAndInstall(cmd, info.Release); err != nil { return fmt.Errorf("failed to update: %w", err) } - cmd.Println("Upgrade completed successfully. Please restart the application.") + cmd.Println("\nUpgrade completed successfully. Please restart the application.") return nil } @@ -337,16 +337,62 @@ func copyFile(src, dst string) error { } type ProgressBar struct { - total int64 + total int64 + current int64 + lastPrint time.Time } func NewProgressBar(total int64) *ProgressBar { - return &ProgressBar{total: total} + return &ProgressBar{ + total: total, + lastPrint: time.Now(), + } } func (pb *ProgressBar) Update(current int64) { - // Simple progress bar implementation - // This can be improved with a more sophisticated progress bar library - percent := float64(current) / float64(pb.total) * 100 - fmt.Printf("\rProgress: %.2f%%", percent) + pb.current = current + // Update progress at most every 100ms to avoid flooding terminal + if time.Since(pb.lastPrint) >= 100*time.Millisecond { + percent := float64(pb.current) / float64(pb.total) * 100 + fmt.Printf("\rProgress: %.1f%% [%s]", percent, pb.getProgressBar()) + pb.lastPrint = time.Now() + } +} + +func (pb *ProgressBar) getProgressBar() string { + width := 30 + progress := int(float64(pb.current) / float64(pb.total) * float64(width)) + return strings.Repeat("=", progress) + strings.Repeat(" ", width-progress) +} + +func checkVersion(cmd *cobra.Command) (*VersionInfo, error) { + info := &VersionInfo{} + var err error + + info.Current, err = parseVersion(Root.Version) + if err != nil { + return nil, fmt.Errorf("invalid current version: %w", err) + } + + info.Release, err = getLatestRelease() + if err != nil { + return nil, fmt.Errorf("failed to check for updates: %w", err) + } + + info.Latest, err = semver.NewVersion(strings.TrimPrefix(info.Release.TagName, "v")) + if err != nil { + return nil, fmt.Errorf("invalid latest version: %w", err) + } + + return info, nil +} + +func getUserConfirmation(cmd *cobra.Command) (bool, error) { + reader := bufio.NewReader(os.Stdin) + cmd.Print("Do you want to update? (y/n): ") + response, err := reader.ReadString('\n') + if err != nil { + return false, fmt.Errorf("failed to read user input: %w", err) + } + return strings.ToLower(strings.TrimSpace(response)) == "y", nil } diff --git a/go.mod b/go.mod index f2d71b6..20a254a 100644 --- a/go.mod +++ b/go.mod @@ -1,39 +1,41 @@ module github.com/nguyenvanduocit/epubtrans -go 1.22.4 +go 1.23 + +toolchain go1.23.2 require ( - github.com/PuerkitoBio/goquery v1.9.2 - github.com/dgraph-io/ristretto v0.1.1 + github.com/Masterminds/semver/v3 v3.3.0 + github.com/PuerkitoBio/goquery v1.10.0 + github.com/dgraph-io/ristretto v0.2.0 github.com/gofiber/fiber/v2 v2.52.5 - github.com/liushuangls/go-anthropic/v2 v2.6.0 + github.com/liushuangls/go-anthropic/v2 v2.9.0 + github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.8.1 - golang.org/x/net v0.27.0 - golang.org/x/sync v0.1.0 - golang.org/x/time v0.5.0 + golang.org/x/net v0.30.0 + golang.org/x/sync v0.8.0 + golang.org/x/time v0.7.0 ) require ( - github.com/Masterminds/semver/v3 v3.2.1 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect - github.com/google/uuid v1.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/golang/glog v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/rivo/uniseg v0.2.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.9.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.51.0 // indirect + github.com/valyala/fasthttp v1.57.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/sys v0.26.0 // indirect ) diff --git a/go.sum b/go.sum index a87e73c..6179c2e 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,21 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= +github.com/PuerkitoBio/goquery v1.10.0 h1:6fiXdLuUvYs2OJSvNRqlNPoBm6YABE226xrbavY5Wv4= +github.com/PuerkitoBio/goquery v1.10.0/go.mod h1:TjZZl68Q3eGHNBA8CWaxAN7rOU1EbDz3CWuolcO5Yu4= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -15,24 +23,37 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= +github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= +github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/liushuangls/go-anthropic/v2 v2.3.1 h1:CtARXi91YFhhRKdf5/zh+NogmWgGaUZmHywQ61/sNGA= github.com/liushuangls/go-anthropic/v2 v2.3.1/go.mod h1:8BKv/fkeTaL5R9R9bGkaknYBueyw2WxY20o7bImbOek= github.com/liushuangls/go-anthropic/v2 v2.6.0 h1:hkgLQPD04wL4lFrV5ZoGlIyy4f6P+brIuRlzn2S8K9s= github.com/liushuangls/go-anthropic/v2 v2.6.0/go.mod h1:8BKv/fkeTaL5R9R9bGkaknYBueyw2WxY20o7bImbOek= +github.com/liushuangls/go-anthropic/v2 v2.9.0 h1:uGtXaypQf4D79hZdmajPciBcHvz5Z7tdU77DLJ4siI4= +github.com/liushuangls/go-anthropic/v2 v2.9.0/go.mod h1:8BKv/fkeTaL5R9R9bGkaknYBueyw2WxY20o7bImbOek= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -40,6 +61,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -47,6 +70,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -60,8 +85,11 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= +github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg= +github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -74,10 +102,14 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -90,6 +122,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -101,6 +135,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/justfile b/justfile new file mode 100644 index 0000000..3b5c1cb --- /dev/null +++ b/justfile @@ -0,0 +1,3 @@ +install: + rm -rf $(which epubtrans) + go install ./... diff --git a/pkg/loader/loader.go b/pkg/loader/loader.go index df5c2bd..4c2d94d 100644 --- a/pkg/loader/loader.go +++ b/pkg/loader/loader.go @@ -44,14 +44,18 @@ type Manifest struct { Items []Item `xml:"item" json:"items"` } -// GetItemByID returns the item with the given ID +// GetItemByID returns the item with the given ID. +// Returns nil if no item is found or if id is empty. func (m Manifest) GetItemByID(id string) *Item { - for _, item := range m.Items { - if item.ID == id { - return &item - } - } - return nil + if id == "" { + return nil + } + for i := range m.Items { + if m.Items[i].ID == id { + return &m.Items[i] // Return reference to slice element + } + } + return nil } type Item struct { @@ -71,6 +75,10 @@ type ItemRef struct { } func ParseContainer(filePath string) (*Container, error) { + if filePath == "" { + return nil, errors.New("filePath cannot be empty") + } + file, err := os.Open(path.Join(filePath, containerFilePath)) if err != nil { return nil, errors.WithMessage(err, "failed to open container file") @@ -86,6 +94,10 @@ func ParseContainer(filePath string) (*Container, error) { } func ParsePackage(filePath string) (*Package, error) { + if filePath == "" { + return nil, errors.New("filePath cannot be empty") + } + file, err := os.Open(filePath) if err != nil { return nil, errors.WithMessage(err, "failed to open package file") diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 5f3800b..453fb67 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -107,7 +107,7 @@ func worker(ctx context.Context, jobs <-chan string, results chan<- error, proce } } -var excludeRegex = regexp.MustCompile(`(?i)(preface|introduction|foreword|prologue|toc|table\s*of\s*contents|title|cover|copyright|colophon|dedication|acknowledgements?|about\s*the\s*author|bibliography|glossary|index|appendix|notes?|footnotes?|endnotes?|references|epub-meta|metadata|nav|ncx|opf|front\s*matter|back\s*matter|halftitle|frontispiece|epigraph|list\s*of\s*(figures|tables|illustrations)|copyright\s*page|series\s*page|reviews|praise|also\s*by\s*the\s*author|author\s*bio|publication\s*info|imprint|credits|permissions|disclaimer|errata|synopsis|summary)`) +var excludeRegex = regexp.MustCompile(`(?i)(preface|introduction|foreword|prologue|toc|table\s*of\s*contents|title|cover|copyright|colophon|dedication|acknowledgements?|about\s*the\s*author|bibliography|glossary|index|appendix|notes?|footnotes?|endnotes?|references|epub-meta|metadata|nav|ncx|opf|front\s*matter|back\s*matter|halftitle|frontispiece|epigraph|list\s*of\s*(figures|tables|illustrations)|copyright\s*page|series\s*page|reviews|praise|also\s*by\s*the\s*author|author\s*bio|publication\s*info|imprint|credits|permissions|disclaimer|errata|synopsis|summary|f\d+)`) // ShouldExcludeFile determines if a file should be excluded based on its name func ShouldExcludeFile(fileName string) bool { diff --git a/pkg/translator/anthropic.go b/pkg/translator/anthropic.go index 28de4c6..4c67a56 100644 --- a/pkg/translator/anthropic.go +++ b/pkg/translator/anthropic.go @@ -48,9 +48,9 @@ func GetAnthropicTranslator(cfg *Config) (*Anthropic, error) { if cfg == nil { cfg = &Config{ APIKey: os.Getenv("ANTHROPIC_KEY"), - Model: anthropic.ModelClaude3Dot5Sonnet20240620, + Model: string(anthropic.ModelClaude3Dot5SonnetLatest), Temperature: 0.3, - MaxTokens: 1000, + MaxTokens: 8192, } } @@ -141,51 +141,82 @@ type Anthropic struct { mu sync.Mutex } -func createTranslationSystem(source, target string, guidelines string) string { +func createTranslationSystem(source, target, guidelines, bookName string) string { if guidelines == "" { - guidelines = `Translation guidelines: -- Preserve HTML structure -- Writing style: Clear, concise, professional, technical, Use %[2]s flexibly, fluently and softly. -- Use active voice and maintain logical flow. -- Translation approach: - • Translate for meaning, not word-for-word - • Adapt idioms and cultural references to %[2]s equivalents - • Restructure sentences if needed for clarity in %[2]s -- Target audience: Programmers and technical professionals -- Terminology: - • Keep %[1]s technical terms (e.g., commit, branch, push code, code, engineer, PM, PO, etc.) - • Ensure consistent use of technical terms throughout -- Match the source's level of formality and technical depth -- Aim for a translation that reads like native %[2]s technical writing -- Do not add explanations or answer questions in the content -- We own the copyright to the material` + guidelines = `You are a skilled translator who excels at making complex psychology concepts simple and accessible for everyday readers. Your task is to translate the psychology book "%[3]s" from %[1]s to %[2]s, focusing on creating an engaging and easy-to-understand version for general readers. + +Translation guidelines: + +1. Core principles: + - Explain like you're talking to a friend + - Use everyday language and examples + - Break down complex ideas into simple terms + - Focus on practical applications + - Make it relatable to daily life + - Do not give extra explanation for the title, section titles, or headings + +2. Writing style: + - Warm and conversational tone + - Short, clear sentences + - Simple words over technical terms + +3. Making concepts accessible: + - Replace technical terms with everyday words + - Use real-life examples and situations + - Connect ideas to common experiences + - Add helpful metaphors and comparisons + +4. Cultural relevance: + - Use local examples and situations + - Reference familiar cultural elements + - Include relatable daily scenarios + - Adapt examples to local context + - Use local expressions when appropriate + +5. Target audience: + - People with no psychology background + - Readers seeking self-help and personal growth + - Anyone interested in understanding themselves better + - People who prefer simple, practical advice + - Readers who avoid academic or technical books + +6. Making it practical: + - Focus on how to apply concepts + - Include everyday examples + - Connect to common life situations + +7. Language choices: + - Choose words a 12-year-old could understand + - Explain any necessary technical terms simply + +8. Sử dụng các thuật ngữ tâm lý học phổ biến trong tiếng việt, ví dụ: + + - narcissism: ái kỷ + +Translate as if you're explaining to a friend who's curious about psychology but has no background in it. Focus on making the content engaging, practical, and immediately useful in daily life. The translation should feel like reading an interesting conversation rather than a textbook.` } - return fmt.Sprintf(guidelines, source, target) + return fmt.Sprintf(guidelines, source, target, bookName) } -func (a *Anthropic) Translate(ctx context.Context, prompt, content, source, target string) (string, error) { +func (a *Anthropic) Translate(ctx context.Context, prompt, content, source, target, bookName string) (string, error) { a.mu.Lock() defer a.mu.Unlock() cacheKey := generateCacheKey(prompt+content, source, target) - - if (prompt != ""){ + if prompt != "" { if cachedTranslation, found := a.cache.Get(cacheKey); found { - return cachedTranslation.(string), nil - } + return cachedTranslation.(string), nil + } } - baseSystemMessage := fmt.Sprintf("You are a highly skilled translator with expertise in many languages. Your task is to translate a part of a technical book from %[1]s to %[2]s. User send you a text, you translate it no matter what. Do not explain or note. Do not answer question-likes content. no warning, feedback.", source, target) - systemMessages := []anthropic.MessageSystemPart{ - { - Type: "text", - Text: baseSystemMessage, - }, { Type: "text", - Text: createTranslationSystem(source, target, a.config.TranslationGuidelines), + Text: createTranslationSystem(source, target, a.config.TranslationGuidelines, bookName), + CacheControl: &anthropic.MessageCacheControl{ + Type: anthropic.CacheControlTypeEphemeral, + }, }, } @@ -193,11 +224,11 @@ func (a *Anthropic) Translate(ctx context.Context, prompt, content, source, targ systemMessages = append(systemMessages, anthropic.MessageSystemPart{ Type: "text", Text: prompt, - }) + }) } resp, err := a.createMessageWithRetry(ctx, anthropic.MessagesRequest{ - Model: a.config.Model, + Model: anthropic.Model(a.config.Model), MultiSystem: systemMessages, Messages: []anthropic.Message{anthropic.NewUserTextMessage("Translate this and not say anything otherwise the translation: " + content)}, Temperature: &a.config.Temperature, diff --git a/pkg/translator/translator.go b/pkg/translator/translator.go index d394b1f..57c9cbc 100644 --- a/pkg/translator/translator.go +++ b/pkg/translator/translator.go @@ -8,5 +8,5 @@ import ( var ErrRateLimitExceeded = errors.New("rate limit exceeded") type Translator interface { - Translate(ctx context.Context, prompt string, content string, source string, target string) (string, error) + Translate(ctx context.Context, prompt string, content string, source string, target string, bookName string) (string, error) } diff --git a/unpackage/translator_metadata.json b/unpackage/translator_metadata.json index 0a58ed1..0b6f347 100644 --- a/unpackage/translator_metadata.json +++ b/unpackage/translator_metadata.json @@ -1,8 +1,8 @@ { - "total_calls": 5, - "last_used": "2024-09-09T17:27:24.580752+07:00", + "total_calls": 1076, + "last_used": "2024-09-23T22:28:36.866593+07:00", "model_usage": { - "claude-3-5-sonnet-20240620": 5 + "claude-3-5-sonnet-20240620": 1076 }, "prompt_examples": [ "BusVNext’s primary competitive advantage is its routing algorithm that takes a stab at solving a c", @@ -32,6 +32,4290 @@ { "input_tokens": 488, "output_tokens": 131 + }, + { + "input_tokens": 382, + "output_tokens": 271 + }, + { + "input_tokens": 402, + "output_tokens": 284 + }, + { + "input_tokens": 381, + "output_tokens": 250 + }, + { + "input_tokens": 329, + "output_tokens": 144 + }, + { + "input_tokens": 281, + "output_tokens": 10 + }, + { + "input_tokens": 384, + "output_tokens": 269 + }, + { + "input_tokens": 391, + "output_tokens": 294 + }, + { + "input_tokens": 350, + "output_tokens": 210 + }, + { + "input_tokens": 439, + "output_tokens": 448 + }, + { + "input_tokens": 310, + "output_tokens": 81 + }, + { + "input_tokens": 374, + "output_tokens": 298 + }, + { + "input_tokens": 358, + "output_tokens": 199 + }, + { + "input_tokens": 339, + "output_tokens": 165 + }, + { + "input_tokens": 283, + "output_tokens": 10 + }, + { + "input_tokens": 327, + "output_tokens": 83 + }, + { + "input_tokens": 394, + "output_tokens": 346 + }, + { + "input_tokens": 539, + "output_tokens": 645 + }, + { + "input_tokens": 439, + "output_tokens": 411 + }, + { + "input_tokens": 402, + "output_tokens": 309 + }, + { + "input_tokens": 284, + "output_tokens": 15 + }, + { + "input_tokens": 466, + "output_tokens": 478 + }, + { + "input_tokens": 612, + "output_tokens": 770 + }, + { + "input_tokens": 439, + "output_tokens": 439 + }, + { + "input_tokens": 326, + "output_tokens": 116 + }, + { + "input_tokens": 292, + "output_tokens": 22 + }, + { + "input_tokens": 398, + "output_tokens": 227 + }, + { + "input_tokens": 424, + "output_tokens": 363 + }, + { + "input_tokens": 284, + "output_tokens": 13 + }, + { + "input_tokens": 391, + "output_tokens": 254 + }, + { + "input_tokens": 610, + "output_tokens": 820 + }, + { + "input_tokens": 624, + "output_tokens": 729 + }, + { + "input_tokens": 364, + "output_tokens": 234 + }, + { + "input_tokens": 359, + "output_tokens": 205 + }, + { + "input_tokens": 459, + "output_tokens": 415 + }, + { + "input_tokens": 285, + "output_tokens": 15 + }, + { + "input_tokens": 289, + "output_tokens": 27 + }, + { + "input_tokens": 396, + "output_tokens": 277 + }, + { + "input_tokens": 403, + "output_tokens": 389 + }, + { + "input_tokens": 370, + "output_tokens": 263 + }, + { + "input_tokens": 446, + "output_tokens": 426 + }, + { + "input_tokens": 404, + "output_tokens": 301 + }, + { + "input_tokens": 541, + "output_tokens": 636 + }, + { + "input_tokens": 291, + "output_tokens": 32 + }, + { + "input_tokens": 391, + "output_tokens": 215 + }, + { + "input_tokens": 436, + "output_tokens": 422 + }, + { + "input_tokens": 476, + "output_tokens": 501 + }, + { + "input_tokens": 371, + "output_tokens": 232 + }, + { + "input_tokens": 358, + "output_tokens": 229 + }, + { + "input_tokens": 408, + "output_tokens": 356 + }, + { + "input_tokens": 469, + "output_tokens": 527 + }, + { + "input_tokens": 461, + "output_tokens": 444 + }, + { + "input_tokens": 354, + "output_tokens": 220 + }, + { + "input_tokens": 446, + "output_tokens": 410 + }, + { + "input_tokens": 390, + "output_tokens": 298 + }, + { + "input_tokens": 488, + "output_tokens": 440 + }, + { + "input_tokens": 290, + "output_tokens": 28 + }, + { + "input_tokens": 391, + "output_tokens": 311 + }, + { + "input_tokens": 524, + "output_tokens": 618 + }, + { + "input_tokens": 295, + "output_tokens": 40 + }, + { + "input_tokens": 286, + "output_tokens": 20 + }, + { + "input_tokens": 284, + "output_tokens": 8 + }, + { + "input_tokens": 284, + "output_tokens": 15 + }, + { + "input_tokens": 283, + "output_tokens": 15 + }, + { + "input_tokens": 371, + "output_tokens": 194 + }, + { + "input_tokens": 418, + "output_tokens": 306 + }, + { + "input_tokens": 445, + "output_tokens": 435 + }, + { + "input_tokens": 475, + "output_tokens": 453 + }, + { + "input_tokens": 361, + "output_tokens": 208 + }, + { + "input_tokens": 307, + "output_tokens": 45 + }, + { + "input_tokens": 318, + "output_tokens": 79 + }, + { + "input_tokens": 355, + "output_tokens": 149 + }, + { + "input_tokens": 390, + "output_tokens": 243 + }, + { + "input_tokens": 371, + "output_tokens": 204 + }, + { + "input_tokens": 364, + "output_tokens": 190 + }, + { + "input_tokens": 328, + "output_tokens": 97 + }, + { + "input_tokens": 376, + "output_tokens": 226 + }, + { + "input_tokens": 301, + "output_tokens": 30 + }, + { + "input_tokens": 389, + "output_tokens": 262 + }, + { + "input_tokens": 297, + "output_tokens": 30 + }, + { + "input_tokens": 307, + "output_tokens": 43 + }, + { + "input_tokens": 298, + "output_tokens": 18 + }, + { + "input_tokens": 376, + "output_tokens": 226 + }, + { + "input_tokens": 395, + "output_tokens": 293 + }, + { + "input_tokens": 326, + "output_tokens": 95 + }, + { + "input_tokens": 329, + "output_tokens": 81 + }, + { + "input_tokens": 310, + "output_tokens": 61 + }, + { + "input_tokens": 317, + "output_tokens": 74 + }, + { + "input_tokens": 300, + "output_tokens": 35 + }, + { + "input_tokens": 298, + "output_tokens": 18 + }, + { + "input_tokens": 365, + "output_tokens": 186 + }, + { + "input_tokens": 516, + "output_tokens": 546 + }, + { + "input_tokens": 315, + "output_tokens": 53 + }, + { + "input_tokens": 422, + "output_tokens": 403 + }, + { + "input_tokens": 352, + "output_tokens": 152 + }, + { + "input_tokens": 510, + "output_tokens": 547 + }, + { + "input_tokens": 405, + "output_tokens": 310 + }, + { + "input_tokens": 438, + "output_tokens": 326 + }, + { + "input_tokens": 432, + "output_tokens": 348 + }, + { + "input_tokens": 582, + "output_tokens": 652 + }, + { + "input_tokens": 524, + "output_tokens": 497 + }, + { + "input_tokens": 364, + "output_tokens": 155 + }, + { + "input_tokens": 417, + "output_tokens": 296 + }, + { + "input_tokens": 653, + "output_tokens": 910 + }, + { + "input_tokens": 445, + "output_tokens": 336 + }, + { + "input_tokens": 466, + "output_tokens": 389 + }, + { + "input_tokens": 499, + "output_tokens": 585 + }, + { + "input_tokens": 593, + "output_tokens": 810 + }, + { + "input_tokens": 543, + "output_tokens": 556 + }, + { + "input_tokens": 400, + "output_tokens": 287 + }, + { + "input_tokens": 611, + "output_tokens": 747 + }, + { + "input_tokens": 561, + "output_tokens": 678 + }, + { + "input_tokens": 411, + "output_tokens": 336 + }, + { + "input_tokens": 300, + "output_tokens": 30 + }, + { + "input_tokens": 394, + "output_tokens": 272 + }, + { + "input_tokens": 460, + "output_tokens": 476 + }, + { + "input_tokens": 498, + "output_tokens": 529 + }, + { + "input_tokens": 399, + "output_tokens": 285 + }, + { + "input_tokens": 300, + "output_tokens": 29 + }, + { + "input_tokens": 326, + "output_tokens": 54 + }, + { + "input_tokens": 307, + "output_tokens": 27 + }, + { + "input_tokens": 305, + "output_tokens": 36 + }, + { + "input_tokens": 304, + "output_tokens": 44 + }, + { + "input_tokens": 305, + "output_tokens": 24 + }, + { + "input_tokens": 300, + "output_tokens": 28 + }, + { + "input_tokens": 310, + "output_tokens": 41 + }, + { + "input_tokens": 304, + "output_tokens": 38 + }, + { + "input_tokens": 419, + "output_tokens": 230 + }, + { + "input_tokens": 390, + "output_tokens": 255 + }, + { + "input_tokens": 502, + "output_tokens": 551 + }, + { + "input_tokens": 544, + "output_tokens": 671 + }, + { + "input_tokens": 540, + "output_tokens": 592 + }, + { + "input_tokens": 532, + "output_tokens": 487 + }, + { + "input_tokens": 464, + "output_tokens": 445 + }, + { + "input_tokens": 304, + "output_tokens": 29 + }, + { + "input_tokens": 369, + "output_tokens": 194 + }, + { + "input_tokens": 342, + "output_tokens": 90 + }, + { + "input_tokens": 343, + "output_tokens": 128 + }, + { + "input_tokens": 372, + "output_tokens": 215 + }, + { + "input_tokens": 608, + "output_tokens": 589 + }, + { + "input_tokens": 508, + "output_tokens": 374 + }, + { + "input_tokens": 486, + "output_tokens": 309 + }, + { + "input_tokens": 632, + "output_tokens": 607 + }, + { + "input_tokens": 548, + "output_tokens": 441 + }, + { + "input_tokens": 476, + "output_tokens": 319 + }, + { + "input_tokens": 570, + "output_tokens": 512 + }, + { + "input_tokens": 487, + "output_tokens": 304 + }, + { + "input_tokens": 467, + "output_tokens": 273 + }, + { + "input_tokens": 550, + "output_tokens": 447 + }, + { + "input_tokens": 551, + "output_tokens": 537 + }, + { + "input_tokens": 663, + "output_tokens": 732 + }, + { + "input_tokens": 379, + "output_tokens": 27 + }, + { + "input_tokens": 445, + "output_tokens": 181 + }, + { + "input_tokens": 682, + "output_tokens": 808 + }, + { + "input_tokens": 552, + "output_tokens": 503 + }, + { + "input_tokens": 513, + "output_tokens": 352 + }, + { + "input_tokens": 455, + "output_tokens": 248 + }, + { + "input_tokens": 575, + "output_tokens": 491 + }, + { + "input_tokens": 540, + "output_tokens": 407 + }, + { + "input_tokens": 563, + "output_tokens": 483 + }, + { + "input_tokens": 587, + "output_tokens": 530 + }, + { + "input_tokens": 471, + "output_tokens": 287 + }, + { + "input_tokens": 377, + "output_tokens": 29 + }, + { + "input_tokens": 506, + "output_tokens": 395 + }, + { + "input_tokens": 421, + "output_tokens": 140 + }, + { + "input_tokens": 374, + "output_tokens": 25 + }, + { + "input_tokens": 564, + "output_tokens": 533 + }, + { + "input_tokens": 374, + "output_tokens": 16 + }, + { + "input_tokens": 497, + "output_tokens": 388 + }, + { + "input_tokens": 561, + "output_tokens": 498 + }, + { + "input_tokens": 527, + "output_tokens": 419 + }, + { + "input_tokens": 425, + "output_tokens": 152 + }, + { + "input_tokens": 374, + "output_tokens": 20 + }, + { + "input_tokens": 536, + "output_tokens": 393 + }, + { + "input_tokens": 473, + "output_tokens": 238 + }, + { + "input_tokens": 435, + "output_tokens": 176 + }, + { + "input_tokens": 473, + "output_tokens": 270 + }, + { + "input_tokens": 640, + "output_tokens": 675 + }, + { + "input_tokens": 580, + "output_tokens": 590 + }, + { + "input_tokens": 477, + "output_tokens": 312 + }, + { + "input_tokens": 372, + "output_tokens": 13 + }, + { + "input_tokens": 536, + "output_tokens": 443 + }, + { + "input_tokens": 383, + "output_tokens": 43 + }, + { + "input_tokens": 482, + "output_tokens": 226 + }, + { + "input_tokens": 495, + "output_tokens": 252 + }, + { + "input_tokens": 464, + "output_tokens": 223 + }, + { + "input_tokens": 587, + "output_tokens": 522 + }, + { + "input_tokens": 518, + "output_tokens": 370 + }, + { + "input_tokens": 573, + "output_tokens": 484 + }, + { + "input_tokens": 534, + "output_tokens": 353 + }, + { + "input_tokens": 488, + "output_tokens": 287 + }, + { + "input_tokens": 492, + "output_tokens": 270 + }, + { + "input_tokens": 582, + "output_tokens": 546 + }, + { + "input_tokens": 539, + "output_tokens": 475 + }, + { + "input_tokens": 381, + "output_tokens": 34 + }, + { + "input_tokens": 487, + "output_tokens": 346 + }, + { + "input_tokens": 529, + "output_tokens": 423 + }, + { + "input_tokens": 575, + "output_tokens": 500 + }, + { + "input_tokens": 614, + "output_tokens": 597 + }, + { + "input_tokens": 375, + "output_tokens": 26 + }, + { + "input_tokens": 431, + "output_tokens": 168 + }, + { + "input_tokens": 375, + "output_tokens": 18 + }, + { + "input_tokens": 382, + "output_tokens": 32 + }, + { + "input_tokens": 377, + "output_tokens": 22 + }, + { + "input_tokens": 388, + "output_tokens": 48 + }, + { + "input_tokens": 379, + "output_tokens": 28 + }, + { + "input_tokens": 380, + "output_tokens": 35 + }, + { + "input_tokens": 593, + "output_tokens": 539 + }, + { + "input_tokens": 509, + "output_tokens": 336 + }, + { + "input_tokens": 388, + "output_tokens": 55 + }, + { + "input_tokens": 532, + "output_tokens": 384 + }, + { + "input_tokens": 462, + "output_tokens": 258 + }, + { + "input_tokens": 400, + "output_tokens": 105 + }, + { + "input_tokens": 376, + "output_tokens": 23 + }, + { + "input_tokens": 385, + "output_tokens": 51 + }, + { + "input_tokens": 381, + "output_tokens": 41 + }, + { + "input_tokens": 379, + "output_tokens": 40 + }, + { + "input_tokens": 377, + "output_tokens": 19 + }, + { + "input_tokens": 472, + "output_tokens": 281 + }, + { + "input_tokens": 424, + "output_tokens": 127 + }, + { + "input_tokens": 503, + "output_tokens": 362 + }, + { + "input_tokens": 374, + "output_tokens": 15 + }, + { + "input_tokens": 498, + "output_tokens": 334 + }, + { + "input_tokens": 385, + "output_tokens": 49 + }, + { + "input_tokens": 376, + "output_tokens": 17 + }, + { + "input_tokens": 493, + "output_tokens": 317 + }, + { + "input_tokens": 565, + "output_tokens": 538 + }, + { + "input_tokens": 615, + "output_tokens": 609 + }, + { + "input_tokens": 579, + "output_tokens": 579 + }, + { + "input_tokens": 423, + "output_tokens": 135 + }, + { + "input_tokens": 373, + "output_tokens": 8 + }, + { + "input_tokens": 373, + "output_tokens": 17 + }, + { + "input_tokens": 374, + "output_tokens": 20 + }, + { + "input_tokens": 496, + "output_tokens": 278 + }, + { + "input_tokens": 481, + "output_tokens": 238 + }, + { + "input_tokens": 457, + "output_tokens": 208 + }, + { + "input_tokens": 544, + "output_tokens": 403 + }, + { + "input_tokens": 438, + "output_tokens": 184 + }, + { + "input_tokens": 491, + "output_tokens": 359 + }, + { + "input_tokens": 517, + "output_tokens": 349 + }, + { + "input_tokens": 537, + "output_tokens": 435 + }, + { + "input_tokens": 535, + "output_tokens": 368 + }, + { + "input_tokens": 493, + "output_tokens": 267 + }, + { + "input_tokens": 528, + "output_tokens": 368 + }, + { + "input_tokens": 509, + "output_tokens": 391 + }, + { + "input_tokens": 420, + "output_tokens": 116 + }, + { + "input_tokens": 505, + "output_tokens": 287 + }, + { + "input_tokens": 458, + "output_tokens": 240 + }, + { + "input_tokens": 454, + "output_tokens": 207 + }, + { + "input_tokens": 436, + "output_tokens": 160 + }, + { + "input_tokens": 573, + "output_tokens": 501 + }, + { + "input_tokens": 468, + "output_tokens": 271 + }, + { + "input_tokens": 402, + "output_tokens": 99 + }, + { + "input_tokens": 386, + "output_tokens": 55 + }, + { + "input_tokens": 385, + "output_tokens": 36 + }, + { + "input_tokens": 510, + "output_tokens": 387 + }, + { + "input_tokens": 589, + "output_tokens": 587 + }, + { + "input_tokens": 457, + "output_tokens": 248 + }, + { + "input_tokens": 534, + "output_tokens": 418 + }, + { + "input_tokens": 627, + "output_tokens": 619 + }, + { + "input_tokens": 475, + "output_tokens": 299 + }, + { + "input_tokens": 480, + "output_tokens": 273 + }, + { + "input_tokens": 379, + "output_tokens": 34 + }, + { + "input_tokens": 436, + "output_tokens": 138 + }, + { + "input_tokens": 518, + "output_tokens": 293 + }, + { + "input_tokens": 484, + "output_tokens": 259 + }, + { + "input_tokens": 391, + "output_tokens": 57 + }, + { + "input_tokens": 540, + "output_tokens": 364 + }, + { + "input_tokens": 445, + "output_tokens": 172 + }, + { + "input_tokens": 419, + "output_tokens": 116 + }, + { + "input_tokens": 440, + "output_tokens": 187 + }, + { + "input_tokens": 467, + "output_tokens": 224 + }, + { + "input_tokens": 454, + "output_tokens": 252 + }, + { + "input_tokens": 398, + "output_tokens": 83 + }, + { + "input_tokens": 399, + "output_tokens": 91 + }, + { + "input_tokens": 403, + "output_tokens": 101 + }, + { + "input_tokens": 426, + "output_tokens": 124 + }, + { + "input_tokens": 462, + "output_tokens": 253 + }, + { + "input_tokens": 490, + "output_tokens": 320 + }, + { + "input_tokens": 637, + "output_tokens": 674 + }, + { + "input_tokens": 472, + "output_tokens": 291 + }, + { + "input_tokens": 601, + "output_tokens": 533 + }, + { + "input_tokens": 438, + "output_tokens": 192 + }, + { + "input_tokens": 375, + "output_tokens": 19 + }, + { + "input_tokens": 490, + "output_tokens": 321 + }, + { + "input_tokens": 493, + "output_tokens": 316 + }, + { + "input_tokens": 561, + "output_tokens": 505 + }, + { + "input_tokens": 495, + "output_tokens": 313 + }, + { + "input_tokens": 380, + "output_tokens": 39 + }, + { + "input_tokens": 529, + "output_tokens": 348 + }, + { + "input_tokens": 502, + "output_tokens": 314 + }, + { + "input_tokens": 489, + "output_tokens": 270 + }, + { + "input_tokens": 375, + "output_tokens": 16 + }, + { + "input_tokens": 470, + "output_tokens": 235 + }, + { + "input_tokens": 483, + "output_tokens": 254 + }, + { + "input_tokens": 572, + "output_tokens": 437 + }, + { + "input_tokens": 501, + "output_tokens": 264 + }, + { + "input_tokens": 495, + "output_tokens": 308 + }, + { + "input_tokens": 442, + "output_tokens": 158 + }, + { + "input_tokens": 432, + "output_tokens": 171 + }, + { + "input_tokens": 633, + "output_tokens": 634 + }, + { + "input_tokens": 376, + "output_tokens": 26 + }, + { + "input_tokens": 561, + "output_tokens": 509 + }, + { + "input_tokens": 529, + "output_tokens": 386 + }, + { + "input_tokens": 510, + "output_tokens": 333 + }, + { + "input_tokens": 577, + "output_tokens": 493 + }, + { + "input_tokens": 494, + "output_tokens": 327 + }, + { + "input_tokens": 495, + "output_tokens": 319 + }, + { + "input_tokens": 534, + "output_tokens": 363 + }, + { + "input_tokens": 483, + "output_tokens": 318 + }, + { + "input_tokens": 420, + "output_tokens": 128 + }, + { + "input_tokens": 373, + "output_tokens": 16 + }, + { + "input_tokens": 498, + "output_tokens": 294 + }, + { + "input_tokens": 508, + "output_tokens": 367 + }, + { + "input_tokens": 502, + "output_tokens": 299 + }, + { + "input_tokens": 644, + "output_tokens": 633 + }, + { + "input_tokens": 672, + "output_tokens": 766 + }, + { + "input_tokens": 515, + "output_tokens": 389 + }, + { + "input_tokens": 650, + "output_tokens": 840 + }, + { + "input_tokens": 465, + "output_tokens": 246 + }, + { + "input_tokens": 572, + "output_tokens": 464 + }, + { + "input_tokens": 453, + "output_tokens": 245 + }, + { + "input_tokens": 599, + "output_tokens": 552 + }, + { + "input_tokens": 380, + "output_tokens": 43 + }, + { + "input_tokens": 413, + "output_tokens": 96 + }, + { + "input_tokens": 525, + "output_tokens": 313 + }, + { + "input_tokens": 462, + "output_tokens": 226 + }, + { + "input_tokens": 488, + "output_tokens": 290 + }, + { + "input_tokens": 525, + "output_tokens": 422 + }, + { + "input_tokens": 497, + "output_tokens": 241 + }, + { + "input_tokens": 476, + "output_tokens": 255 + }, + { + "input_tokens": 546, + "output_tokens": 425 + }, + { + "input_tokens": 467, + "output_tokens": 272 + }, + { + "input_tokens": 518, + "output_tokens": 350 + }, + { + "input_tokens": 483, + "output_tokens": 230 + }, + { + "input_tokens": 544, + "output_tokens": 431 + }, + { + "input_tokens": 469, + "output_tokens": 259 + }, + { + "input_tokens": 638, + "output_tokens": 652 + }, + { + "input_tokens": 629, + "output_tokens": 671 + }, + { + "input_tokens": 534, + "output_tokens": 434 + }, + { + "input_tokens": 577, + "output_tokens": 504 + }, + { + "input_tokens": 536, + "output_tokens": 427 + }, + { + "input_tokens": 374, + "output_tokens": 19 + }, + { + "input_tokens": 462, + "output_tokens": 219 + }, + { + "input_tokens": 468, + "output_tokens": 282 + }, + { + "input_tokens": 519, + "output_tokens": 382 + }, + { + "input_tokens": 483, + "output_tokens": 318 + }, + { + "input_tokens": 640, + "output_tokens": 746 + }, + { + "input_tokens": 547, + "output_tokens": 468 + }, + { + "input_tokens": 565, + "output_tokens": 482 + }, + { + "input_tokens": 509, + "output_tokens": 379 + }, + { + "input_tokens": 447, + "output_tokens": 190 + }, + { + "input_tokens": 376, + "output_tokens": 25 + }, + { + "input_tokens": 463, + "output_tokens": 275 + }, + { + "input_tokens": 460, + "output_tokens": 232 + }, + { + "input_tokens": 596, + "output_tokens": 570 + }, + { + "input_tokens": 507, + "output_tokens": 259 + }, + { + "input_tokens": 419, + "output_tokens": 138 + }, + { + "input_tokens": 470, + "output_tokens": 253 + }, + { + "input_tokens": 474, + "output_tokens": 296 + }, + { + "input_tokens": 509, + "output_tokens": 340 + }, + { + "input_tokens": 514, + "output_tokens": 343 + }, + { + "input_tokens": 467, + "output_tokens": 243 + }, + { + "input_tokens": 573, + "output_tokens": 489 + }, + { + "input_tokens": 538, + "output_tokens": 450 + }, + { + "input_tokens": 450, + "output_tokens": 196 + }, + { + "input_tokens": 547, + "output_tokens": 465 + }, + { + "input_tokens": 529, + "output_tokens": 423 + }, + { + "input_tokens": 558, + "output_tokens": 460 + }, + { + "input_tokens": 505, + "output_tokens": 343 + }, + { + "input_tokens": 600, + "output_tokens": 595 + }, + { + "input_tokens": 661, + "output_tokens": 681 + }, + { + "input_tokens": 499, + "output_tokens": 361 + }, + { + "input_tokens": 497, + "output_tokens": 357 + }, + { + "input_tokens": 604, + "output_tokens": 570 + }, + { + "input_tokens": 382, + "output_tokens": 44 + }, + { + "input_tokens": 514, + "output_tokens": 306 + }, + { + "input_tokens": 482, + "output_tokens": 327 + }, + { + "input_tokens": 377, + "output_tokens": 27 + }, + { + "input_tokens": 376, + "output_tokens": 19 + }, + { + "input_tokens": 375, + "output_tokens": 28 + }, + { + "input_tokens": 375, + "output_tokens": 17 + }, + { + "input_tokens": 376, + "output_tokens": 24 + }, + { + "input_tokens": 389, + "output_tokens": 53 + }, + { + "input_tokens": 432, + "output_tokens": 169 + }, + { + "input_tokens": 573, + "output_tokens": 401 + }, + { + "input_tokens": 629, + "output_tokens": 573 + }, + { + "input_tokens": 535, + "output_tokens": 375 + }, + { + "input_tokens": 493, + "output_tokens": 327 + }, + { + "input_tokens": 413, + "output_tokens": 77 + }, + { + "input_tokens": 390, + "output_tokens": 19 + }, + { + "input_tokens": 390, + "output_tokens": 17 + }, + { + "input_tokens": 391, + "output_tokens": 24 + }, + { + "input_tokens": 404, + "output_tokens": 53 + }, + { + "input_tokens": 448, + "output_tokens": 178 + }, + { + "input_tokens": 601, + "output_tokens": 439 + }, + { + "input_tokens": 445, + "output_tokens": 143 + }, + { + "input_tokens": 615, + "output_tokens": 537 + }, + { + "input_tokens": 738, + "output_tokens": 909 + }, + { + "input_tokens": 392, + "output_tokens": 28 + }, + { + "input_tokens": 481, + "output_tokens": 204 + }, + { + "input_tokens": 470, + "output_tokens": 238 + }, + { + "input_tokens": 449, + "output_tokens": 162 + }, + { + "input_tokens": 585, + "output_tokens": 549 + }, + { + "input_tokens": 507, + "output_tokens": 293 + }, + { + "input_tokens": 388, + "output_tokens": 9 + }, + { + "input_tokens": 388, + "output_tokens": 9 + }, + { + "input_tokens": 388, + "output_tokens": 13 + }, + { + "input_tokens": 591, + "output_tokens": 424 + }, + { + "input_tokens": 438, + "output_tokens": 155 + }, + { + "input_tokens": 423, + "output_tokens": 94 + }, + { + "input_tokens": 395, + "output_tokens": 38 + }, + { + "input_tokens": 386, + "output_tokens": 6 + }, + { + "input_tokens": 525, + "output_tokens": 326 + }, + { + "input_tokens": 499, + "output_tokens": 317 + }, + { + "input_tokens": 475, + "output_tokens": 242 + }, + { + "input_tokens": 573, + "output_tokens": 551 + }, + { + "input_tokens": 490, + "output_tokens": 303 + }, + { + "input_tokens": 387, + "output_tokens": 18 + }, + { + "input_tokens": 387, + "output_tokens": 19 + }, + { + "input_tokens": 387, + "output_tokens": 18 + }, + { + "input_tokens": 388, + "output_tokens": 8 + }, + { + "input_tokens": 391, + "output_tokens": 25 + }, + { + "input_tokens": 393, + "output_tokens": 33 + }, + { + "input_tokens": 397, + "output_tokens": 33 + }, + { + "input_tokens": 572, + "output_tokens": 416 + }, + { + "input_tokens": 498, + "output_tokens": 271 + }, + { + "input_tokens": 522, + "output_tokens": 405 + }, + { + "input_tokens": 601, + "output_tokens": 546 + }, + { + "input_tokens": 459, + "output_tokens": 209 + }, + { + "input_tokens": 574, + "output_tokens": 519 + }, + { + "input_tokens": 388, + "output_tokens": 14 + }, + { + "input_tokens": 523, + "output_tokens": 284 + }, + { + "input_tokens": 502, + "output_tokens": 246 + }, + { + "input_tokens": 484, + "output_tokens": 258 + }, + { + "input_tokens": 511, + "output_tokens": 320 + }, + { + "input_tokens": 516, + "output_tokens": 341 + }, + { + "input_tokens": 587, + "output_tokens": 492 + }, + { + "input_tokens": 490, + "output_tokens": 284 + }, + { + "input_tokens": 392, + "output_tokens": 20 + }, + { + "input_tokens": 506, + "output_tokens": 296 + }, + { + "input_tokens": 562, + "output_tokens": 448 + }, + { + "input_tokens": 473, + "output_tokens": 225 + }, + { + "input_tokens": 397, + "output_tokens": 37 + }, + { + "input_tokens": 546, + "output_tokens": 411 + }, + { + "input_tokens": 502, + "output_tokens": 339 + }, + { + "input_tokens": 598, + "output_tokens": 492 + }, + { + "input_tokens": 526, + "output_tokens": 353 + }, + { + "input_tokens": 588, + "output_tokens": 522 + }, + { + "input_tokens": 395, + "output_tokens": 40 + }, + { + "input_tokens": 537, + "output_tokens": 318 + }, + { + "input_tokens": 562, + "output_tokens": 421 + }, + { + "input_tokens": 497, + "output_tokens": 243 + }, + { + "input_tokens": 639, + "output_tokens": 622 + }, + { + "input_tokens": 517, + "output_tokens": 366 + }, + { + "input_tokens": 540, + "output_tokens": 418 + }, + { + "input_tokens": 577, + "output_tokens": 397 + }, + { + "input_tokens": 527, + "output_tokens": 370 + }, + { + "input_tokens": 615, + "output_tokens": 567 + }, + { + "input_tokens": 488, + "output_tokens": 280 + }, + { + "input_tokens": 487, + "output_tokens": 164 + }, + { + "input_tokens": 389, + "output_tokens": 16 + }, + { + "input_tokens": 612, + "output_tokens": 642 + }, + { + "input_tokens": 392, + "output_tokens": 26 + }, + { + "input_tokens": 595, + "output_tokens": 540 + }, + { + "input_tokens": 540, + "output_tokens": 386 + }, + { + "input_tokens": 390, + "output_tokens": 15 + }, + { + "input_tokens": 583, + "output_tokens": 491 + }, + { + "input_tokens": 483, + "output_tokens": 283 + }, + { + "input_tokens": 684, + "output_tokens": 651 + }, + { + "input_tokens": 534, + "output_tokens": 432 + }, + { + "input_tokens": 557, + "output_tokens": 411 + }, + { + "input_tokens": 461, + "output_tokens": 205 + }, + { + "input_tokens": 392, + "output_tokens": 25 + }, + { + "input_tokens": 549, + "output_tokens": 455 + }, + { + "input_tokens": 567, + "output_tokens": 525 + }, + { + "input_tokens": 577, + "output_tokens": 491 + }, + { + "input_tokens": 390, + "output_tokens": 22 + }, + { + "input_tokens": 511, + "output_tokens": 306 + }, + { + "input_tokens": 603, + "output_tokens": 529 + }, + { + "input_tokens": 528, + "output_tokens": 381 + }, + { + "input_tokens": 559, + "output_tokens": 396 + }, + { + "input_tokens": 518, + "output_tokens": 369 + }, + { + "input_tokens": 628, + "output_tokens": 518 + }, + { + "input_tokens": 566, + "output_tokens": 468 + }, + { + "input_tokens": 492, + "output_tokens": 269 + }, + { + "input_tokens": 389, + "output_tokens": 32 + }, + { + "input_tokens": 457, + "output_tokens": 192 + }, + { + "input_tokens": 630, + "output_tokens": 582 + }, + { + "input_tokens": 523, + "output_tokens": 341 + }, + { + "input_tokens": 517, + "output_tokens": 337 + }, + { + "input_tokens": 407, + "output_tokens": 51 + }, + { + "input_tokens": 384, + "output_tokens": 24 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 400, + "output_tokens": 27 + }, + { + "input_tokens": 386, + "output_tokens": 17 + }, + { + "input_tokens": 395, + "output_tokens": 16 + }, + { + "input_tokens": 400, + "output_tokens": 21 + }, + { + "input_tokens": 400, + "output_tokens": 21 + }, + { + "input_tokens": 390, + "output_tokens": 19 + }, + { + "input_tokens": 610, + "output_tokens": 405 + }, + { + "input_tokens": 605, + "output_tokens": 624 + }, + { + "input_tokens": 447, + "output_tokens": 151 + }, + { + "input_tokens": 520, + "output_tokens": 246 + }, + { + "input_tokens": 398, + "output_tokens": 37 + }, + { + "input_tokens": 499, + "output_tokens": 224 + }, + { + "input_tokens": 384, + "output_tokens": 9 + }, + { + "input_tokens": 385, + "output_tokens": 11 + }, + { + "input_tokens": 511, + "output_tokens": 231 + }, + { + "input_tokens": 386, + "output_tokens": 20 + }, + { + "input_tokens": 464, + "output_tokens": 201 + }, + { + "input_tokens": 417, + "output_tokens": 115 + }, + { + "input_tokens": 449, + "output_tokens": 145 + }, + { + "input_tokens": 414, + "output_tokens": 52 + }, + { + "input_tokens": 385, + "output_tokens": 12 + }, + { + "input_tokens": 410, + "output_tokens": 62 + }, + { + "input_tokens": 483, + "output_tokens": 226 + }, + { + "input_tokens": 430, + "output_tokens": 98 + }, + { + "input_tokens": 431, + "output_tokens": 109 + }, + { + "input_tokens": 474, + "output_tokens": 120 + }, + { + "input_tokens": 435, + "output_tokens": 150 + }, + { + "input_tokens": 385, + "output_tokens": 6 + }, + { + "input_tokens": 383, + "output_tokens": 10 + }, + { + "input_tokens": 496, + "output_tokens": 270 + }, + { + "input_tokens": 385, + "output_tokens": 14 + }, + { + "input_tokens": 384, + "output_tokens": 16 + }, + { + "input_tokens": 389, + "output_tokens": 28 + }, + { + "input_tokens": 386, + "output_tokens": 24 + }, + { + "input_tokens": 387, + "output_tokens": 17 + }, + { + "input_tokens": 386, + "output_tokens": 20 + }, + { + "input_tokens": 521, + "output_tokens": 397 + }, + { + "input_tokens": 444, + "output_tokens": 179 + }, + { + "input_tokens": 611, + "output_tokens": 580 + }, + { + "input_tokens": 459, + "output_tokens": 207 + }, + { + "input_tokens": 446, + "output_tokens": 159 + }, + { + "input_tokens": 401, + "output_tokens": 30 + }, + { + "input_tokens": 453, + "output_tokens": 179 + }, + { + "input_tokens": 437, + "output_tokens": 165 + }, + { + "input_tokens": 499, + "output_tokens": 256 + }, + { + "input_tokens": 434, + "output_tokens": 104 + }, + { + "input_tokens": 388, + "output_tokens": 19 + }, + { + "input_tokens": 400, + "output_tokens": 50 + }, + { + "input_tokens": 400, + "output_tokens": 1050 + }, + { + "input_tokens": 512, + "output_tokens": 339 + }, + { + "input_tokens": 453, + "output_tokens": 199 + }, + { + "input_tokens": 449, + "output_tokens": 182 + }, + { + "input_tokens": 470, + "output_tokens": 207 + }, + { + "input_tokens": 500, + "output_tokens": 297 + }, + { + "input_tokens": 466, + "output_tokens": 175 + }, + { + "input_tokens": 456, + "output_tokens": 175 + }, + { + "input_tokens": 472, + "output_tokens": 197 + }, + { + "input_tokens": 440, + "output_tokens": 140 + }, + { + "input_tokens": 440, + "output_tokens": 133 + }, + { + "input_tokens": 456, + "output_tokens": 188 + }, + { + "input_tokens": 452, + "output_tokens": 171 + }, + { + "input_tokens": 389, + "output_tokens": 37 + }, + { + "input_tokens": 450, + "output_tokens": 179 + }, + { + "input_tokens": 441, + "output_tokens": 171 + }, + { + "input_tokens": 429, + "output_tokens": 133 + }, + { + "input_tokens": 412, + "output_tokens": 87 + }, + { + "input_tokens": 523, + "output_tokens": 239 + }, + { + "input_tokens": 384, + "output_tokens": 14 + }, + { + "input_tokens": 458, + "output_tokens": 194 + }, + { + "input_tokens": 393, + "output_tokens": 44 + }, + { + "input_tokens": 400, + "output_tokens": 2087 + }, + { + "input_tokens": 388, + "output_tokens": 25 + }, + { + "input_tokens": 396, + "output_tokens": 4111 + }, + { + "input_tokens": 388, + "output_tokens": 18 + }, + { + "input_tokens": 399, + "output_tokens": 573 + }, + { + "input_tokens": 387, + "output_tokens": 15 + }, + { + "input_tokens": 392, + "output_tokens": 351 + }, + { + "input_tokens": 403, + "output_tokens": 40 + }, + { + "input_tokens": 398, + "output_tokens": 1142 + }, + { + "input_tokens": 393, + "output_tokens": 24 + }, + { + "input_tokens": 397, + "output_tokens": 366 + }, + { + "input_tokens": 389, + "output_tokens": 21 + }, + { + "input_tokens": 398, + "output_tokens": 998 + }, + { + "input_tokens": 397, + "output_tokens": 22 + }, + { + "input_tokens": 401, + "output_tokens": 433 + }, + { + "input_tokens": 406, + "output_tokens": 57 + }, + { + "input_tokens": 478, + "output_tokens": 156 + }, + { + "input_tokens": 385, + "output_tokens": 15 + }, + { + "input_tokens": 445, + "output_tokens": 91 + }, + { + "input_tokens": 384, + "output_tokens": 16 + }, + { + "input_tokens": 487, + "output_tokens": 260 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 488, + "output_tokens": 240 + }, + { + "input_tokens": 389, + "output_tokens": 24 + }, + { + "input_tokens": 389, + "output_tokens": 24 + }, + { + "input_tokens": 404, + "output_tokens": 49 + }, + { + "input_tokens": 389, + "output_tokens": 25 + }, + { + "input_tokens": 411, + "output_tokens": 81 + }, + { + "input_tokens": 384, + "output_tokens": 15 + }, + { + "input_tokens": 390, + "output_tokens": 23 + }, + { + "input_tokens": 386, + "output_tokens": 16 + }, + { + "input_tokens": 385, + "output_tokens": 5 + }, + { + "input_tokens": 403, + "output_tokens": 54 + }, + { + "input_tokens": 389, + "output_tokens": 11 + }, + { + "input_tokens": 390, + "output_tokens": 12 + }, + { + "input_tokens": 389, + "output_tokens": 21 + }, + { + "input_tokens": 389, + "output_tokens": 11 + }, + { + "input_tokens": 465, + "output_tokens": 164 + }, + { + "input_tokens": 385, + "output_tokens": 7 + }, + { + "input_tokens": 388, + "output_tokens": 16 + }, + { + "input_tokens": 386, + "output_tokens": 9 + }, + { + "input_tokens": 392, + "output_tokens": 24 + }, + { + "input_tokens": 432, + "output_tokens": 125 + }, + { + "input_tokens": 385, + "output_tokens": 13 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 450, + "output_tokens": 183 + }, + { + "input_tokens": 388, + "output_tokens": 18 + }, + { + "input_tokens": 388, + "output_tokens": 15 + }, + { + "input_tokens": 388, + "output_tokens": 16 + }, + { + "input_tokens": 387, + "output_tokens": 21 + }, + { + "input_tokens": 427, + "output_tokens": 103 + }, + { + "input_tokens": 385, + "output_tokens": 5 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 385, + "output_tokens": 5 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 443, + "output_tokens": 172 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 385, + "output_tokens": 6 + }, + { + "input_tokens": 390, + "output_tokens": 23 + }, + { + "input_tokens": 397, + "output_tokens": 45 + }, + { + "input_tokens": 386, + "output_tokens": 12 + }, + { + "input_tokens": 398, + "output_tokens": 43 + }, + { + "input_tokens": 391, + "output_tokens": 34 + }, + { + "input_tokens": 412, + "output_tokens": 70 + }, + { + "input_tokens": 385, + "output_tokens": 11 + }, + { + "input_tokens": 383, + "output_tokens": 5 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 480, + "output_tokens": 219 + }, + { + "input_tokens": 389, + "output_tokens": 23 + }, + { + "input_tokens": 391, + "output_tokens": 36 + }, + { + "input_tokens": 390, + "output_tokens": 25 + }, + { + "input_tokens": 389, + "output_tokens": 28 + }, + { + "input_tokens": 454, + "output_tokens": 223 + }, + { + "input_tokens": 389, + "output_tokens": 28 + }, + { + "input_tokens": 390, + "output_tokens": 30 + }, + { + "input_tokens": 388, + "output_tokens": 15 + }, + { + "input_tokens": 390, + "output_tokens": 31 + }, + { + "input_tokens": 498, + "output_tokens": 338 + }, + { + "input_tokens": 385, + "output_tokens": 6 + }, + { + "input_tokens": 385, + "output_tokens": 7 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 384, + "output_tokens": 13 + }, + { + "input_tokens": 423, + "output_tokens": 116 + }, + { + "input_tokens": 389, + "output_tokens": 22 + }, + { + "input_tokens": 386, + "output_tokens": 18 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 388, + "output_tokens": 16 + }, + { + "input_tokens": 413, + "output_tokens": 75 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 431, + "output_tokens": 134 + }, + { + "input_tokens": 388, + "output_tokens": 10 + }, + { + "input_tokens": 385, + "output_tokens": 9 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 385, + "output_tokens": 17 + }, + { + "input_tokens": 466, + "output_tokens": 268 + }, + { + "input_tokens": 390, + "output_tokens": 23 + }, + { + "input_tokens": 389, + "output_tokens": 23 + }, + { + "input_tokens": 405, + "output_tokens": 80 + }, + { + "input_tokens": 395, + "output_tokens": 33 + }, + { + "input_tokens": 412, + "output_tokens": 94 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 384, + "output_tokens": 11 + }, + { + "input_tokens": 456, + "output_tokens": 183 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 385, + "output_tokens": 18 + }, + { + "input_tokens": 384, + "output_tokens": 10 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 478, + "output_tokens": 290 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 416, + "output_tokens": 116 + }, + { + "input_tokens": 386, + "output_tokens": 17 + }, + { + "input_tokens": 384, + "output_tokens": 15 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 430, + "output_tokens": 116 + }, + { + "input_tokens": 385, + "output_tokens": 18 + }, + { + "input_tokens": 385, + "output_tokens": 10 + }, + { + "input_tokens": 387, + "output_tokens": 19 + }, + { + "input_tokens": 385, + "output_tokens": 14 + }, + { + "input_tokens": 406, + "output_tokens": 62 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 386, + "output_tokens": 16 + }, + { + "input_tokens": 387, + "output_tokens": 19 + }, + { + "input_tokens": 386, + "output_tokens": 16 + }, + { + "input_tokens": 385, + "output_tokens": 9 + }, + { + "input_tokens": 392, + "output_tokens": 23 + }, + { + "input_tokens": 399, + "output_tokens": 70 + }, + { + "input_tokens": 394, + "output_tokens": 37 + }, + { + "input_tokens": 405, + "output_tokens": 71 + }, + { + "input_tokens": 388, + "output_tokens": 33 + }, + { + "input_tokens": 384, + "output_tokens": 16 + }, + { + "input_tokens": 386, + "output_tokens": 20 + }, + { + "input_tokens": 383, + "output_tokens": 9 + }, + { + "input_tokens": 385, + "output_tokens": 17 + }, + { + "input_tokens": 386, + "output_tokens": 10 + }, + { + "input_tokens": 387, + "output_tokens": 22 + }, + { + "input_tokens": 406, + "output_tokens": 71 + }, + { + "input_tokens": 387, + "output_tokens": 17 + }, + { + "input_tokens": 515, + "output_tokens": 379 + }, + { + "input_tokens": 481, + "output_tokens": 291 + }, + { + "input_tokens": 389, + "output_tokens": 22 + }, + { + "input_tokens": 385, + "output_tokens": 17 + }, + { + "input_tokens": 387, + "output_tokens": 16 + }, + { + "input_tokens": 396, + "output_tokens": 51 + }, + { + "input_tokens": 390, + "output_tokens": 44 + }, + { + "input_tokens": 386, + "output_tokens": 17 + }, + { + "input_tokens": 400, + "output_tokens": 54 + }, + { + "input_tokens": 386, + "output_tokens": 22 + }, + { + "input_tokens": 386, + "output_tokens": 18 + }, + { + "input_tokens": 387, + "output_tokens": 19 + }, + { + "input_tokens": 390, + "output_tokens": 23 + }, + { + "input_tokens": 388, + "output_tokens": 35 + }, + { + "input_tokens": 391, + "output_tokens": 42 + }, + { + "input_tokens": 407, + "output_tokens": 85 + }, + { + "input_tokens": 386, + "output_tokens": 21 + }, + { + "input_tokens": 475, + "output_tokens": 301 + }, + { + "input_tokens": 412, + "output_tokens": 80 + }, + { + "input_tokens": 399, + "output_tokens": 30 + }, + { + "input_tokens": 390, + "output_tokens": 25 + }, + { + "input_tokens": 398, + "output_tokens": 34 + }, + { + "input_tokens": 390, + "output_tokens": 22 + }, + { + "input_tokens": 388, + "output_tokens": 23 + }, + { + "input_tokens": 414, + "output_tokens": 116 + }, + { + "input_tokens": 388, + "output_tokens": 18 + }, + { + "input_tokens": 417, + "output_tokens": 93 + }, + { + "input_tokens": 466, + "output_tokens": 231 + }, + { + "input_tokens": 442, + "output_tokens": 121 + }, + { + "input_tokens": 462, + "output_tokens": 215 + }, + { + "input_tokens": 390, + "output_tokens": 20 + }, + { + "input_tokens": 410, + "output_tokens": 98 + }, + { + "input_tokens": 447, + "output_tokens": 191 + }, + { + "input_tokens": 479, + "output_tokens": 260 + }, + { + "input_tokens": 498, + "output_tokens": 245 + }, + { + "input_tokens": 491, + "output_tokens": 272 + }, + { + "input_tokens": 423, + "output_tokens": 122 + }, + { + "input_tokens": 390, + "output_tokens": 29 + }, + { + "input_tokens": 449, + "output_tokens": 221 + }, + { + "input_tokens": 509, + "output_tokens": 309 + }, + { + "input_tokens": 544, + "output_tokens": 372 + }, + { + "input_tokens": 506, + "output_tokens": 302 + }, + { + "input_tokens": 392, + "output_tokens": 28 + }, + { + "input_tokens": 458, + "output_tokens": 193 + }, + { + "input_tokens": 484, + "output_tokens": 284 + }, + { + "input_tokens": 435, + "output_tokens": 164 + }, + { + "input_tokens": 467, + "output_tokens": 239 + }, + { + "input_tokens": 390, + "output_tokens": 23 + }, + { + "input_tokens": 419, + "output_tokens": 134 + }, + { + "input_tokens": 404, + "output_tokens": 70 + }, + { + "input_tokens": 401, + "output_tokens": 60 + }, + { + "input_tokens": 405, + "output_tokens": 41 + }, + { + "input_tokens": 404, + "output_tokens": 81 + }, + { + "input_tokens": 403, + "output_tokens": 63 + }, + { + "input_tokens": 467, + "output_tokens": 281 + }, + { + "input_tokens": 424, + "output_tokens": 145 + }, + { + "input_tokens": 386, + "output_tokens": 18 + }, + { + "input_tokens": 407, + "output_tokens": 74 + }, + { + "input_tokens": 453, + "output_tokens": 125 + }, + { + "input_tokens": 402, + "output_tokens": 56 + }, + { + "input_tokens": 446, + "output_tokens": 192 + }, + { + "input_tokens": 420, + "output_tokens": 106 + }, + { + "input_tokens": 450, + "output_tokens": 182 + }, + { + "input_tokens": 459, + "output_tokens": 211 + }, + { + "input_tokens": 412, + "output_tokens": 52 + }, + { + "input_tokens": 440, + "output_tokens": 133 + }, + { + "input_tokens": 427, + "output_tokens": 121 + }, + { + "input_tokens": 424, + "output_tokens": 101 + }, + { + "input_tokens": 420, + "output_tokens": 88 + }, + { + "input_tokens": 386, + "output_tokens": 21 + }, + { + "input_tokens": 488, + "output_tokens": 258 + }, + { + "input_tokens": 385, + "output_tokens": 15 + }, + { + "input_tokens": 419, + "output_tokens": 100 + }, + { + "input_tokens": 386, + "output_tokens": 21 + }, + { + "input_tokens": 496, + "output_tokens": 242 + }, + { + "input_tokens": 512, + "output_tokens": 231 + }, + { + "input_tokens": 385, + "output_tokens": 28 + }, + { + "input_tokens": 431, + "output_tokens": 123 + }, + { + "input_tokens": 546, + "output_tokens": 426 + }, + { + "input_tokens": 501, + "output_tokens": 280 + }, + { + "input_tokens": 534, + "output_tokens": 354 + }, + { + "input_tokens": 385, + "output_tokens": 25 + }, + { + "input_tokens": 407, + "output_tokens": 71 + }, + { + "input_tokens": 469, + "output_tokens": 202 + }, + { + "input_tokens": 390, + "output_tokens": 21 + }, + { + "input_tokens": 388, + "output_tokens": 17 + }, + { + "input_tokens": 398, + "output_tokens": 23 + }, + { + "input_tokens": 391, + "output_tokens": 23 + }, + { + "input_tokens": 386, + "output_tokens": 16 + }, + { + "input_tokens": 507, + "output_tokens": 316 + }, + { + "input_tokens": 388, + "output_tokens": 15 + }, + { + "input_tokens": 399, + "output_tokens": 51 + }, + { + "input_tokens": 384, + "output_tokens": 12 + }, + { + "input_tokens": 384, + "output_tokens": 10 + }, + { + "input_tokens": 385, + "output_tokens": 14 + }, + { + "input_tokens": 385, + "output_tokens": 9 + }, + { + "input_tokens": 477, + "output_tokens": 227 + }, + { + "input_tokens": 447, + "output_tokens": 166 + }, + { + "input_tokens": 486, + "output_tokens": 247 + }, + { + "input_tokens": 474, + "output_tokens": 187 + }, + { + "input_tokens": 417, + "output_tokens": 107 + }, + { + "input_tokens": 386, + "output_tokens": 22 + }, + { + "input_tokens": 449, + "output_tokens": 127 + }, + { + "input_tokens": 426, + "output_tokens": 110 + }, + { + "input_tokens": 433, + "output_tokens": 182 + }, + { + "input_tokens": 454, + "output_tokens": 152 + }, + { + "input_tokens": 530, + "output_tokens": 436 + }, + { + "input_tokens": 391, + "output_tokens": 30 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 409, + "output_tokens": 1180 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 411, + "output_tokens": 1257 + }, + { + "input_tokens": 388, + "output_tokens": 10 + }, + { + "input_tokens": 410, + "output_tokens": 1645 + }, + { + "input_tokens": 386, + "output_tokens": 7 + }, + { + "input_tokens": 384, + "output_tokens": 10 + }, + { + "input_tokens": 529, + "output_tokens": 368 + }, + { + "input_tokens": 385, + "output_tokens": 14 + }, + { + "input_tokens": 450, + "output_tokens": 204 + }, + { + "input_tokens": 386, + "output_tokens": 18 + }, + { + "input_tokens": 409, + "output_tokens": 72 + }, + { + "input_tokens": 385, + "output_tokens": 17 + }, + { + "input_tokens": 398, + "output_tokens": 45 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 388, + "output_tokens": 11 + }, + { + "input_tokens": 388, + "output_tokens": 9 + }, + { + "input_tokens": 397, + "output_tokens": 24 + }, + { + "input_tokens": 404, + "output_tokens": 68 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 388, + "output_tokens": 11 + }, + { + "input_tokens": 387, + "output_tokens": 8 + }, + { + "input_tokens": 385, + "output_tokens": 9 + }, + { + "input_tokens": 398, + "output_tokens": 47 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 384, + "output_tokens": 6 + }, + { + "input_tokens": 388, + "output_tokens": 11 + }, + { + "input_tokens": 388, + "output_tokens": 9 + }, + { + "input_tokens": 393, + "output_tokens": 18 + }, + { + "input_tokens": 384, + "output_tokens": 15 + }, + { + "input_tokens": 387, + "output_tokens": 18 + }, + { + "input_tokens": 386, + "output_tokens": 11 + }, + { + "input_tokens": 388, + "output_tokens": 9 + }, + { + "input_tokens": 397, + "output_tokens": 24 + }, + { + "input_tokens": 400, + "output_tokens": 37 + }, + { + "input_tokens": 386, + "output_tokens": 10 + }, + { + "input_tokens": 387, + "output_tokens": 8 + }, + { + "input_tokens": 388, + "output_tokens": 13 + }, + { + "input_tokens": 394, + "output_tokens": 29 + }, + { + "input_tokens": 386, + "output_tokens": 10 + }, + { + "input_tokens": 387, + "output_tokens": 8 + }, + { + "input_tokens": 386, + "output_tokens": 15 + }, + { + "input_tokens": 389, + "output_tokens": 14 + }, + { + "input_tokens": 427, + "output_tokens": 91 + }, + { + "input_tokens": 428, + "output_tokens": 82 + }, + { + "input_tokens": 384, + "output_tokens": 16 + }, + { + "input_tokens": 396, + "output_tokens": 55 + }, + { + "input_tokens": 410, + "output_tokens": 81 + }, + { + "input_tokens": 399, + "output_tokens": 58 + }, + { + "input_tokens": 393, + "output_tokens": 48 + }, + { + "input_tokens": 384, + "output_tokens": 19 + }, + { + "input_tokens": 393, + "output_tokens": 32 + }, + { + "input_tokens": 414, + "output_tokens": 91 + }, + { + "input_tokens": 396, + "output_tokens": 51 + }, + { + "input_tokens": 392, + "output_tokens": 32 + }, + { + "input_tokens": 395, + "output_tokens": 49 + }, + { + "input_tokens": 384, + "output_tokens": 16 + }, + { + "input_tokens": 458, + "output_tokens": 247 + }, + { + "input_tokens": 460, + "output_tokens": 213 + }, + { + "input_tokens": 389, + "output_tokens": 20 + }, + { + "input_tokens": 384, + "output_tokens": 12 + }, + { + "input_tokens": 427, + "output_tokens": 130 + }, + { + "input_tokens": 412, + "output_tokens": 97 + }, + { + "input_tokens": 412, + "output_tokens": 86 + }, + { + "input_tokens": 385, + "output_tokens": 15 + }, + { + "input_tokens": 446, + "output_tokens": 147 + }, + { + "input_tokens": 384, + "output_tokens": 16 + }, + { + "input_tokens": 389, + "output_tokens": 24 + }, + { + "input_tokens": 394, + "output_tokens": 46 + }, + { + "input_tokens": 391, + "output_tokens": 28 + }, + { + "input_tokens": 389, + "output_tokens": 23 + }, + { + "input_tokens": 384, + "output_tokens": 20 + }, + { + "input_tokens": 387, + "output_tokens": 25 + }, + { + "input_tokens": 394, + "output_tokens": 40 + }, + { + "input_tokens": 398, + "output_tokens": 59 + }, + { + "input_tokens": 395, + "output_tokens": 58 + }, + { + "input_tokens": 400, + "output_tokens": 53 + }, + { + "input_tokens": 389, + "output_tokens": 27 + }, + { + "input_tokens": 387, + "output_tokens": 24 + }, + { + "input_tokens": 398, + "output_tokens": 35 + }, + { + "input_tokens": 398, + "output_tokens": 46 + }, + { + "input_tokens": 397, + "output_tokens": 33 + }, + { + "input_tokens": 400, + "output_tokens": 42 + }, + { + "input_tokens": 398, + "output_tokens": 40 + }, + { + "input_tokens": 384, + "output_tokens": 16 + }, + { + "input_tokens": 501, + "output_tokens": 360 + }, + { + "input_tokens": 390, + "output_tokens": 21 + }, + { + "input_tokens": 384, + "output_tokens": 12 + }, + { + "input_tokens": 449, + "output_tokens": 190 + }, + { + "input_tokens": 385, + "output_tokens": 15 + }, + { + "input_tokens": 464, + "output_tokens": 195 + }, + { + "input_tokens": 457, + "output_tokens": 145 + }, + { + "input_tokens": 386, + "output_tokens": 18 + }, + { + "input_tokens": 460, + "output_tokens": 181 + }, + { + "input_tokens": 446, + "output_tokens": 158 + }, + { + "input_tokens": 384, + "output_tokens": 16 + }, + { + "input_tokens": 395, + "output_tokens": 39 + }, + { + "input_tokens": 404, + "output_tokens": 82 + }, + { + "input_tokens": 415, + "output_tokens": 104 + }, + { + "input_tokens": 384, + "output_tokens": 15 + }, + { + "input_tokens": 401, + "output_tokens": 64 + }, + { + "input_tokens": 386, + "output_tokens": 17 + }, + { + "input_tokens": 394, + "output_tokens": 47 + }, + { + "input_tokens": 389, + "output_tokens": 24 + }, + { + "input_tokens": 392, + "output_tokens": 43 + }, + { + "input_tokens": 389, + "output_tokens": 25 + }, + { + "input_tokens": 401, + "output_tokens": 64 + }, + { + "input_tokens": 384, + "output_tokens": 8 + }, + { + "input_tokens": 390, + "output_tokens": 11 + }, + { + "input_tokens": 387, + "output_tokens": 8 + }, + { + "input_tokens": 388, + "output_tokens": 9 + }, + { + "input_tokens": 390, + "output_tokens": 21 + }, + { + "input_tokens": 388, + "output_tokens": 19 + }, + { + "input_tokens": 439, + "output_tokens": 152 + }, + { + "input_tokens": 384, + "output_tokens": 9 + }, + { + "input_tokens": 398, + "output_tokens": 44 + }, + { + "input_tokens": 390, + "output_tokens": 11 + }, + { + "input_tokens": 387, + "output_tokens": 8 + }, + { + "input_tokens": 388, + "output_tokens": 9 + }, + { + "input_tokens": 388, + "output_tokens": 16 + }, + { + "input_tokens": 384, + "output_tokens": 12 + }, + { + "input_tokens": 387, + "output_tokens": 19 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 387, + "output_tokens": 8 + }, + { + "input_tokens": 388, + "output_tokens": 9 + }, + { + "input_tokens": 395, + "output_tokens": 28 + }, + { + "input_tokens": 384, + "output_tokens": 19 + }, + { + "input_tokens": 457, + "output_tokens": 230 + }, + { + "input_tokens": 383, + "output_tokens": 11 + }, + { + "input_tokens": 398, + "output_tokens": 42 + }, + { + "input_tokens": 386, + "output_tokens": 22 + }, + { + "input_tokens": 386, + "output_tokens": 18 + }, + { + "input_tokens": 387, + "output_tokens": 19 + }, + { + "input_tokens": 390, + "output_tokens": 27 + }, + { + "input_tokens": 388, + "output_tokens": 35 + }, + { + "input_tokens": 391, + "output_tokens": 43 + }, + { + "input_tokens": 482, + "output_tokens": 304 + }, + { + "input_tokens": 410, + "output_tokens": 103 + }, + { + "input_tokens": 387, + "output_tokens": 22 + }, + { + "input_tokens": 473, + "output_tokens": 262 + }, + { + "input_tokens": 491, + "output_tokens": 249 + }, + { + "input_tokens": 485, + "output_tokens": 245 + }, + { + "input_tokens": 457, + "output_tokens": 190 + }, + { + "input_tokens": 451, + "output_tokens": 191 + }, + { + "input_tokens": 492, + "output_tokens": 318 + }, + { + "input_tokens": 384, + "output_tokens": 16 + }, + { + "input_tokens": 415, + "output_tokens": 98 + }, + { + "input_tokens": 391, + "output_tokens": 38 + }, + { + "input_tokens": 399, + "output_tokens": 44 + }, + { + "input_tokens": 388, + "output_tokens": 36 + }, + { + "input_tokens": 387, + "output_tokens": 27 + }, + { + "input_tokens": 412, + "output_tokens": 76 + }, + { + "input_tokens": 394, + "output_tokens": 39 + }, + { + "input_tokens": 395, + "output_tokens": 38 + }, + { + "input_tokens": 395, + "output_tokens": 41 + }, + { + "input_tokens": 394, + "output_tokens": 45 + }, + { + "input_tokens": 410, + "output_tokens": 96 + }, + { + "input_tokens": 399, + "output_tokens": 61 + }, + { + "input_tokens": 399, + "output_tokens": 46 + }, + { + "input_tokens": 398, + "output_tokens": 44 + }, + { + "input_tokens": 394, + "output_tokens": 22 + }, + { + "input_tokens": 465, + "output_tokens": 214 + }, + { + "input_tokens": 387, + "output_tokens": 18 + }, + { + "input_tokens": 388, + "output_tokens": 30 + }, + { + "input_tokens": 387, + "output_tokens": 14 + }, + { + "input_tokens": 388, + "output_tokens": 16 + }, + { + "input_tokens": 442, + "output_tokens": 150 + }, + { + "input_tokens": 399, + "output_tokens": 38 + }, + { + "input_tokens": 399, + "output_tokens": 51 + }, + { + "input_tokens": 396, + "output_tokens": 42 + }, + { + "input_tokens": 391, + "output_tokens": 21 + }, + { + "input_tokens": 435, + "output_tokens": 140 + }, + { + "input_tokens": 392, + "output_tokens": 29 + }, + { + "input_tokens": 394, + "output_tokens": 22 + }, + { + "input_tokens": 394, + "output_tokens": 46 + }, + { + "input_tokens": 401, + "output_tokens": 51 + }, + { + "input_tokens": 463, + "output_tokens": 257 + }, + { + "input_tokens": 384, + "output_tokens": 15 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 385, + "output_tokens": 6 + }, + { + "input_tokens": 438, + "output_tokens": 179 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 384, + "output_tokens": 15 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 443, + "output_tokens": 170 + }, + { + "input_tokens": 386, + "output_tokens": 10 + }, + { + "input_tokens": 384, + "output_tokens": 14 + }, + { + "input_tokens": 385, + "output_tokens": 20 + }, + { + "input_tokens": 387, + "output_tokens": 8 + }, + { + "input_tokens": 400, + "output_tokens": 51 + }, + { + "input_tokens": 393, + "output_tokens": 40 + }, + { + "input_tokens": 391, + "output_tokens": 44 + }, + { + "input_tokens": 395, + "output_tokens": 48 + }, + { + "input_tokens": 391, + "output_tokens": 35 + }, + { + "input_tokens": 411, + "output_tokens": 84 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 406, + "output_tokens": 81 + }, + { + "input_tokens": 393, + "output_tokens": 40 + }, + { + "input_tokens": 386, + "output_tokens": 21 + }, + { + "input_tokens": 393, + "output_tokens": 34 + }, + { + "input_tokens": 386, + "output_tokens": 15 + }, + { + "input_tokens": 413, + "output_tokens": 92 + }, + { + "input_tokens": 392, + "output_tokens": 36 + }, + { + "input_tokens": 429, + "output_tokens": 99 + }, + { + "input_tokens": 394, + "output_tokens": 31 + }, + { + "input_tokens": 403, + "output_tokens": 49 + }, + { + "input_tokens": 409, + "output_tokens": 89 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 384, + "output_tokens": 15 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 419, + "output_tokens": 98 + }, + { + "input_tokens": 386, + "output_tokens": 8 + }, + { + "input_tokens": 384, + "output_tokens": 5 + }, + { + "input_tokens": 384, + "output_tokens": 15 + }, + { + "input_tokens": 387, + "output_tokens": 9 + }, + { + "input_tokens": 385, + "output_tokens": 9 + }, + { + "input_tokens": 387, + "output_tokens": 25 + }, + { + "input_tokens": 400, + "output_tokens": 61 + }, + { + "input_tokens": 400, + "output_tokens": 51 + }, + { + "input_tokens": 400, + "output_tokens": 53 + }, + { + "input_tokens": 398, + "output_tokens": 32 + }, + { + "input_tokens": 488, + "output_tokens": 270 + }, + { + "input_tokens": 579, + "output_tokens": 535 + }, + { + "input_tokens": 400, + "output_tokens": 60 + }, + { + "input_tokens": 388, + "output_tokens": 32 + }, + { + "input_tokens": 387, + "output_tokens": 26 + }, + { + "input_tokens": 387, + "output_tokens": 25 + }, + { + "input_tokens": 386, + "output_tokens": 14 + }, + { + "input_tokens": 383, + "output_tokens": 9 + }, + { + "input_tokens": 384, + "output_tokens": 18 + }, + { + "input_tokens": 403, + "output_tokens": 59 + }, + { + "input_tokens": 388, + "output_tokens": 41 + }, + { + "input_tokens": 502, + "output_tokens": 285 + }, + { + "input_tokens": 406, + "output_tokens": 59 + }, + { + "input_tokens": 387, + "output_tokens": 11 + }, + { + "input_tokens": 466, + "output_tokens": 210 + }, + { + "input_tokens": 395, + "output_tokens": 45 + }, + { + "input_tokens": 422, + "output_tokens": 116 + }, + { + "input_tokens": 390, + "output_tokens": 26 } ] } \ No newline at end of file