8.5 (#443)

* wip towards 8 * Various fixes * More fixes * More fixes * Final test fix? * Update workflows * Changes for ES 8.5 / Lucene 9.4.0 * WIP attempt at swapping out scripting for the new flavor * Need the init * Execute from compile to maintain similar legacy functionality * Branch es 8.5 wrapped (#437) * Convert to RawDoubleValuesScript * Rename new class, add test expression plugin. * Checkstyle fixes, bring mock expressions into Base integ test * Fix floating point assertion error, final style fix * Tidy code a bit. Co-authored-by: Daniel Worley <[email protected]> * Cleanup build config * Little more cleanup * Add missing override Co-authored-by: Nathan Day <[email protected]> Co-authored-by: Nikola Grcevski <[email protected]> With support by: mmcmahon-ag
o19s · Nov 2, 2022 · 6dbd7ff · 6dbd7ff
1 parent 9c1bb9a
commit 6dbd7ff
Show file tree

Hide file tree

Showing 86 changed files with 588 additions and 393 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -15,10 +15,10 @@ jobs:
     - uses: actions/checkout@v2
     - name: Set release version Name
       run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-    - name: Set up JDK 14.0
+    - name: Set up JDK 17.0
       uses: actions/setup-java@v1
       with:
-        java-version: 14.0
+        java-version: 17.0
     - name: Grant execute permission for gradlew
       run: chmod +x gradlew
     - name: Build with Gradle

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -14,10 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 14.0
+    - name: Set up JDK 17.0
       uses: actions/setup-java@v1
       with:
-        java-version: 14.0
+        java-version: 17.0
     - name: Grant execute permission for gradlew
       run: chmod +x gradlew
     - name: Build with Gradle

diff --git a/build.gradle b/build.gradle
@@ -32,7 +32,6 @@ allprojects {
 
     runtimeOnly 'org.locationtech.spatial4j:spatial4j:0.7'
     runtimeOnly 'org.locationtech.jts:jts-core:1.15.0'
-    runtimeOnly 'org.apache.logging.log4j:log4j-core:2.17.1'
   }
 }
 

diff --git a/gradle.properties b/gradle.properties
@@ -1,5 +1,5 @@
 ltrVersion = 1.5.8
-elasticsearchVersion = 7.17.7
-luceneVersion = 8.11.1
+elasticsearchVersion = 8.5.0
+luceneVersion = 9.4.1
 ow2Version = 8.0.1
 antlrVersion = 4.5.1-1
diff --git a/src/javaRestTest/java/com/o19s/es/ltr/action/BaseIntegrationTest.java b/src/javaRestTest/java/com/o19s/es/ltr/action/BaseIntegrationTest.java
@@ -16,6 +16,7 @@
 
 package com.o19s.es.ltr.action;
 
+import com.o19s.es.TestExpressionsPlugin;
 import com.o19s.es.ltr.LtrQueryParserPlugin;
 import com.o19s.es.ltr.action.FeatureStoreAction.FeatureStoreRequestBuilder;
 import com.o19s.es.ltr.action.FeatureStoreAction.FeatureStoreResponse;
@@ -37,7 +38,6 @@
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptEngine;
 import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -59,10 +59,13 @@ public abstract class BaseIntegrationTest extends ESSingleNodeTestCase {
     public static final ScriptContext<ScoreScript.Factory> AGGS_CONTEXT = new ScriptContext<>("aggs", ScoreScript.Factory.class);
 
     @Override
-    // TODO: Remove the TestGeoShapeFieldMapperPlugin once upstream has completed the migration.
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return Arrays.asList(LtrQueryParserPlugin.class, NativeScriptPlugin.class, InjectionScriptPlugin.class,
-                TestGeoShapeFieldMapperPlugin.class);
+        return Arrays.asList(
+            LtrQueryParserPlugin.class,
+            NativeScriptPlugin.class,
+            InjectionScriptPlugin.class,
+            TestExpressionsPlugin.class
+        );
     }
 
     public void createStore(String name) throws Exception {
@@ -125,7 +128,6 @@ public FeatureStoreResponse addElement(StorableElement element,
         builder.request().setValidation(validation);
         FeatureStoreResponse response = builder.execute().get();
         assertEquals(1, response.getResponse().getVersion());
-        assertEquals(IndexFeatureStore.ES_TYPE, response.getResponse().getType());
         assertEquals(DocWriteResponse.Result.CREATED, response.getResponse().getResult());
         assertEquals(element.id(), response.getResponse().getId());
         assertEquals(store, response.getResponse().getIndex());

diff --git a/src/javaRestTest/java/com/o19s/es/ltr/logging/LoggingIT.java b/src/javaRestTest/java/com/o19s/es/ltr/logging/LoggingIT.java
diff --git a/src/javaRestTest/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java b/src/javaRestTest/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java
@@ -296,7 +296,7 @@ public void testInvalidDerived() throws Exception {
 
     public void buildIndex() {
         client().admin().indices().prepareCreate("test_index").get();
-        client().prepareIndex("test_index", "test")
+        client().prepareIndex("test_index")
                 .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                 .setSource("field1", "hello world", "field2", "bonjour world")
                 .get();

diff --git a/src/main/java/com/o19s/es/explore/ExplorerQuery.java b/src/main/java/com/o19s/es/explore/ExplorerQuery.java
@@ -21,6 +21,7 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermStates;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Weight;
@@ -60,7 +61,6 @@ private boolean isCollectionScoped() {
 
     public String getType() { return this.type; }
 
-    @SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
     @Override
     public boolean equals(Object other) {
         return sameClassAs(other) &&
@@ -94,9 +94,8 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
         if (!scoreMode.needsScores()) {
             return searcher.createWeight(query, scoreMode, boost);
         }
-        final Weight subWeight = searcher.createWeight(query, scoreMode, boost);
-        Set<Term> terms = new HashSet<>();
-        subWeight.extractTerms(terms);
+        final Set<Term> terms = new HashSet<>();
+        this.visit(QueryVisitor.termCollector(terms));
         if (isCollectionScoped()) {
             ClassicSimilarity sim = new ClassicSimilarity();
             StatisticsHelper df_stats = new StatisticsHelper();
@@ -241,9 +240,8 @@ static class ExplorerWeight extends Weight {
             this.type = type;
         }
 
-        @Override
         public void extractTerms(Set<Term> terms) {
-            weight.extractTerms(terms);
+            QueryVisitor.termCollector(terms);
         }
 
         @Override
@@ -279,4 +277,9 @@ public Scorer scorer(LeafReaderContext context) throws IOException {
     public String toString(String field) {
         return query.toString();
     }
+
+    @Override
+    public void visit(QueryVisitor visitor) {
+        this.query.visit(visitor.getSubVisitor(BooleanClause.Occur.MUST, this));
+    }
 }
diff --git a/src/main/java/com/o19s/es/explore/ExplorerQueryBuilder.java b/src/main/java/com/o19s/es/explore/ExplorerQueryBuilder.java
@@ -17,6 +17,7 @@
 
 import org.apache.lucene.search.Query;
 import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -152,4 +153,10 @@ public ExplorerQueryBuilder statsType(String type) {
         this.type = type;
         return this;
     }
+
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return Version.V_7_0_0;
+    }
 }
diff --git a/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java b/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java
@@ -25,13 +25,14 @@
 import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.TermStates;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.DocIdSetIterator;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -62,7 +63,6 @@ public String toString(String field) {
         return buffer.toString();
     }
 
-    @SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
     @Override
     public boolean equals(Object obj) {
         return this.sameClassAs(obj)
@@ -117,7 +117,6 @@ static class PostingsExplorerWeight extends Weight {
             this.type = type;
         }
 
-        @Override
         public void extractTerms(Set<Term> terms) {
             terms.add(term);
         }
@@ -245,4 +244,11 @@ public float getMaxScore(int upTo) throws IOException {
             return Float.POSITIVE_INFINITY;
         }
     }
+
+    @Override
+    public void visit(QueryVisitor visitor) {
+        if (visitor.acceptField(this.term.field()) != false) {
+            visitor.consumeTerms(this, term);
+        }
+    }
 }
diff --git a/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java b/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java
@@ -63,16 +63,18 @@
 import com.o19s.es.ltr.stats.suppliers.PluginHealthStatusSupplier;
 import com.o19s.es.ltr.stats.suppliers.StoreStatsSupplier;
 import com.o19s.es.ltr.utils.FeatureStoreLoader;
+import com.o19s.es.ltr.utils.Scripting;
 import com.o19s.es.ltr.utils.Suppliers;
 import com.o19s.es.termstat.TermStatQueryBuilder;
 import org.apache.lucene.analysis.core.KeywordTokenizer;
 import org.apache.lucene.analysis.miscellaneous.LengthFilter;
 import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.xcontent.ParseField;
@@ -103,6 +105,7 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.tracing.Tracer;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.io.IOException;
@@ -248,14 +251,19 @@ public Collection<Object> createComponents(Client client,
                                                NodeEnvironment nodeEnvironment,
                                                NamedWriteableRegistry namedWriteableRegistry,
                                                IndexNameExpressionResolver indexNameExpressionResolver,
-                                               Supplier<RepositoriesService> repositoriesServiceSupplier) {
+                                               Supplier<RepositoriesService> repositoriesServiceSupplier,
+                                               Tracer tracer,
+                                               AllocationDeciders AllocationDeciders) {
         clusterService.addListener(event -> {
             for (Index i : event.indicesDeleted()) {
                 if (IndexFeatureStore.isIndexStore(i.getName())) {
                     caches.evict(i.getName());
                 }
             }
         });
+
+        Scripting.initScriptService(scriptService);
+
         return asList(caches, parserFactory, getStats(client, clusterService, indexNameExpressionResolver));
     }
 

diff --git a/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java b/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable.Reader;

diff --git a/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java b/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
-import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;

diff --git a/src/main/java/com/o19s/es/ltr/action/ClearCachesAction.java b/src/main/java/com/o19s/es/ltr/action/ClearCachesAction.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
-import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;

diff --git a/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java b/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.StatusToXContentObject;

diff --git a/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java b/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable.Reader;

diff --git a/src/main/java/com/o19s/es/ltr/action/LTRStatsAction.java b/src/main/java/com/o19s/es/ltr/action/LTRStatsAction.java
@@ -3,15 +3,15 @@
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.FailedNodeException;
-import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
-import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -39,7 +39,7 @@ public LTRStatsRequestBuilder(ElasticsearchClient client) {
         }
     }
 
-    public static class LTRStatsNodeRequest extends BaseNodeRequest {
+    public static class LTRStatsNodeRequest extends TransportRequest {
         private final LTRStatsNodesRequest nodesRequest;
 
         public LTRStatsNodeRequest(LTRStatsNodesRequest nodesRequest) {
@@ -158,7 +158,7 @@ protected void writeNodesTo(StreamOutput out, List<LTRStatsNodeResponse> nodeRes
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeMap(clusterStats);
+            out.writeGenericMap(clusterStats);
         }
 
         @Override

diff --git a/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java b/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
-import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-Original file line number
+Diff line change
@@ Expand Up / @@ -32,7 +32,6 @@ allprojects { @@
         runtimeOnly 'org.locationtech.spatial4j:spatial4j:0.7'
         runtimeOnly 'org.locationtech.jts:jts-core:1.15.0'
-        runtimeOnly 'org.apache.logging.log4j:log4j-core:2.17.1'
       }
     }
@@ Expand Down @@