MB-56946: Introduce AnalyzerV2 and Runtime registry

+ Introduce new analyzer interface (AnalyzerV2) capable of: + Generating values other than tokenStream, like embedding vectors + Error can be returned during the analysis + Update the analyzers registry to allow runtime registration/removal (*) Note on Backward compatibility - Old Analyzers are all wrapped under AnalyzerAdapter to ensure backward compatibility. - Test cases which register analyzer in the cache and call the Analyze method must be updated to deal with error returned by it (Refer the test cases updated in the PR)
blevesearch · Sep 5, 2024 · abeee2c · abeee2c
1 parent e9c45ff
commit abeee2c
Show file tree

Hide file tree

Showing 42 changed files with 292 additions and 66 deletions.
diff --git a/analysis/benchmark_test.go b/analysis/benchmark_test.go
@@ -32,7 +32,10 @@ func BenchmarkAnalysis(b *testing.B) {
 			b.Fatal(err)
 		}
 
-		ts := analyzer.Analyze(bleveWikiArticle)
+		ts, err := analysis.AnalyzeForTokens(analyzer, bleveWikiArticle)
+		if err != nil {
+			b.Fatalf("error analyzing text: %v", err)
+		}
 		freqs := analysis.TokenFrequency(ts, nil, index.IncludeTermVectors)
 		if len(freqs) != 511 {
 			b.Errorf("expected %d freqs, got %d", 511, len(freqs))

diff --git a/analysis/lang/ar/analyzer_ar_test.go b/analysis/lang/ar/analyzer_ar_test.go
@@ -175,7 +175,10 @@ func TestArabicAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 			t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term)

diff --git a/analysis/lang/cjk/analyzer_cjk_test.go b/analysis/lang/cjk/analyzer_cjk_test.go
@@ -617,7 +617,10 @@ func TestCJKAnalyzer(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

diff --git a/analysis/lang/ckb/analyzer_ckb_test.go b/analysis/lang/ckb/analyzer_ckb_test.go
@@ -69,7 +69,10 @@ func TestSoraniAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

diff --git a/analysis/lang/ckb/sorani_stemmer_filter_test.go b/analysis/lang/ckb/sorani_stemmer_filter_test.go
@@ -26,7 +26,7 @@ func TestSoraniStemmerFilter(t *testing.T) {
 
 	// in order to match the lucene tests
 	// we will test with an analyzer, not just the stemmer
-	analyzer := analysis.DefaultAnalyzer{
+	analyzer := &analysis.DefaultAnalyzer{
 		Tokenizer: single.NewSingleTokenTokenizer(),
 		TokenFilters: []analysis.TokenFilter{
 			NewSoraniNormalizeFilter(),
@@ -283,7 +283,11 @@ func TestSoraniStemmerFilter(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Errorf("error analyzing input: %v", err)
+		}
+
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("for input %s(% x)", test.input, test.input)
 			t.Errorf("\texpected:")

diff --git a/analysis/lang/da/analyzer_da_test.go b/analysis/lang/da/analyzer_da_test.go
@@ -63,7 +63,10 @@ func TestDanishAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

diff --git a/analysis/lang/de/analyzer_de_test.go b/analysis/lang/de/analyzer_de_test.go
@@ -147,7 +147,10 @@ func TestGermanAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

diff --git a/analysis/lang/en/analyzer_en_test.go b/analysis/lang/en/analyzer_en_test.go
@@ -97,7 +97,10 @@ func TestEnglishAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

diff --git a/analysis/lang/es/analyzer_es_test.go b/analysis/lang/es/analyzer_es_test.go
@@ -114,7 +114,10 @@ func TestSpanishAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

diff --git a/analysis/lang/fa/analyzer_fa_test.go b/analysis/lang/fa/analyzer_fa_test.go
@@ -305,7 +305,10 @@ func TestPersianAnalyzerVerbs(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}
@@ -600,7 +603,10 @@ func TestPersianAnalyzerVerbsDefective(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}
@@ -671,7 +677,10 @@ func TestPersianAnalyzerOthers(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/fi/analyzer_fi_test.go b/analysis/lang/fi/analyzer_fi_test.go
@@ -57,7 +57,10 @@ func TestFinishAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/fr/analyzer_fr_test.go b/analysis/lang/fr/analyzer_fr_test.go
@@ -196,7 +196,10 @@ func TestFrenchAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/hi/analyzer_hi_test.go b/analysis/lang/hi/analyzer_hi_test.go
@@ -58,7 +58,10 @@ func TestHindiAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

diff --git a/analysis/lang/hr/analyzer_hr_test.go b/analysis/lang/hr/analyzer_hr_test.go
@@ -84,7 +84,10 @@ func TestCroatianAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/hu/analyzer_hu_test.go b/analysis/lang/hu/analyzer_hu_test.go
@@ -57,7 +57,10 @@ func TestHungarianAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/it/analyzer_it_test.go b/analysis/lang/it/analyzer_it_test.go
@@ -83,7 +83,10 @@ func TestItalianAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/nl/analyzer_nl_test.go b/analysis/lang/nl/analyzer_nl_test.go
@@ -57,7 +57,10 @@ func TestDutchAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/no/analyzer_no_test.go b/analysis/lang/no/analyzer_no_test.go
@@ -57,7 +57,10 @@ func TestNorwegianAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/pl/analyzer_pl_test.go b/analysis/lang/pl/analyzer_pl_test.go
@@ -136,7 +136,10 @@ func TestPolishAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/pt/analyzer_pt_test.go b/analysis/lang/pt/analyzer_pt_test.go
@@ -57,7 +57,10 @@ func TestPortugueseAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/ro/analyzer_ro_test.go b/analysis/lang/ro/analyzer_ro_test.go
@@ -57,7 +57,10 @@ func TestRomanianAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/ru/analyzer_ru_test.go b/analysis/lang/ru/analyzer_ru_test.go
@@ -109,7 +109,10 @@ func TestRussianAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/sv/analyzer_sv_test.go b/analysis/lang/sv/analyzer_sv_test.go
@@ -57,7 +57,10 @@ func TestSwedishAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}

diff --git a/analysis/lang/tr/analyzer_tr_test.go b/analysis/lang/tr/analyzer_tr_test.go
@@ -77,7 +77,10 @@ func TestTurkishAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}