From 61ced7380b990ed74b2ec8634bdd67eea76f5b64 Mon Sep 17 00:00:00 2001 From: Mark Cohen Date: Tue, 11 Jul 2023 14:24:51 -0400 Subject: [PATCH 1/7] any new issues should have an 'untriaged' label added by default to call out that we need to read it to ensure it's at least not a security issue Signed-off-by: Mark Cohen --- .github/workflows/add-untriaged-label.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/add-untriaged-label.yml diff --git a/.github/workflows/add-untriaged-label.yml b/.github/workflows/add-untriaged-label.yml new file mode 100644 index 00000000..bfff875b --- /dev/null +++ b/.github/workflows/add-untriaged-label.yml @@ -0,0 +1,19 @@ +name: Apply 'untriaged' label during issue lifecycle + +on: + issues: + types: [opened, transferred] + +jobs: + apply-label: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v6 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['untriaged'] + }) \ No newline at end of file From 657d8f539e3d01acdef2a916c10ea42ee770c5a9 Mon Sep 17 00:00:00 2001 From: Mark Cohen Date: Tue, 8 Aug 2023 22:28:53 -0400 Subject: [PATCH 2/7] updated to work with OpenSearch 2.6.0 Signed-off-by: Mark Cohen --- build.gradle | 4 ++-- gradle.properties | 8 ++++---- src/main/plugin-metadata/plugin-security.policy | 15 ++++++++------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/build.gradle b/build.gradle index 89b76bcd..db6a35da 100644 --- a/build.gradle +++ b/build.gradle @@ -61,15 +61,15 @@ repositories { } dependencies { - implementation "org.apache.lucene:lucene-expressions:${luceneVersion}" implementation "org.antlr:antlr4-runtime:${antlrVersion}" implementation "org.ow2.asm:asm:${ow2Version}" + implementation "org.apache.lucene:lucene-expressions:${luceneVersion}" implementation "org.ow2.asm:asm-commons:${ow2Version}" implementation "org.ow2.asm:asm-tree:${ow2Version}" implementation 'com.o19s:RankyMcRankFace:0.1.1' implementation "com.github.spullara.mustache.java:compiler:0.9.3" - //testImplementation 'org.apache.lucene:lucene-test-framework:${luceneVersion}' + testImplementation "org.apache.lucene:lucene-test-framework:${luceneVersion}" runtimeOnly 'org.locationtech.spatial4j:spatial4j:0.7' runtimeOnly 'org.locationtech.jts:jts-core:1.15.0' runtimeOnly 'org.apache.logging.log4j:log4j-core:2.17.1' diff --git a/gradle.properties b/gradle.properties index 694a5403..d450d873 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ -ltrVersion = 2.1.0 -opensearchVersion = 2.5.0 -luceneVersion = 9.4.2 +ltrVersion = 2.2.0 +opensearchVersion = 2.6.0 +luceneVersion = 9.5.0 ow2Version = 9.4 -antlrVersion=4.9.3 +antlrVersion=4.13.0 diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index 4835ab05..3ba9439e 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -22,11 +22,12 @@ grant { permission java.lang.RuntimePermission "createClassLoader"; // expression runtime - permission org.elasticsearch.script.ClassPermission "java.lang.String"; - permission org.elasticsearch.script.ClassPermission "org.apache.lucene.expressions.Expression"; - permission org.elasticsearch.script.ClassPermission "org.apache.lucene.search.DoubleValues"; + permission org.opensearch.script.ClassPermission "java.lang.String"; + permission org.opensearch.script.ClassPermission "org.apache.lucene.expressions.Expression"; + permission org.opensearch.script.ClassPermission "org.apache.lucene.expressions.js.ExpressionMath"; + permission org.opensearch.script.ClassPermission "org.apache.lucene.search.DoubleValues"; // available functions - permission org.elasticsearch.script.ClassPermission "java.lang.Math"; - permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.MathUtil"; - permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.SloppyMath"; -}; + permission org.opensearch.script.ClassPermission "java.lang.Math"; + permission org.opensearch.script.ClassPermission "org.apache.lucene.util.MathUtil"; + permission org.opensearch.script.ClassPermission "org.apache.lucene.util.SloppyMath"; +}; \ No newline at end of file From a756c689d398376b75a81cb285d09469a58b78d7 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Sun, 22 Oct 2023 23:58:12 +0530 Subject: [PATCH 3/7] update build.gradle Signed-off-by: Mohammad Hasnain Mohsin Rajan --- .github/workflows/ci.yml | 0 .github/workflows/release.yml | 4 +- NOTICE | 2 + NOTICE.txt | 4 - build.gradle | 238 +++++++++++++++++++--------------- 5 files changed, 137 insertions(+), 111 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 NOTICE delete mode 100644 NOTICE.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e69de29b diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e8905680..c5286d1e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,7 +25,7 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Build with Gradle - run: ./gradlew -Dtests.security.manager=false build + run: ./gradlew -Dskip.integtests=true -Djava.security.manager=allow -Dbuild.snapshot=false - name: Rename build assets run: mv ./build/distributions/ltr-*.zip ./ltr-plugin-${{ env.RELEASE_VERSION }}.zip - name: Create Release @@ -36,7 +36,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} tag: "release-${{ env.RELEASE_VERSION }}" - name: Upload Release Asset - id: upload-release-asset + id: upload-release-asset uses: softprops/action-gh-release@v1 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..731cb600 --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +OpenSearch (https://opensearch.org/) +Copyright OpenSearch Contributors diff --git a/NOTICE.txt b/NOTICE.txt deleted file mode 100644 index 7ae56dea..00000000 --- a/NOTICE.txt +++ /dev/null @@ -1,4 +0,0 @@ -Elasticsearch Learning to Rank -Copyright 2016-2017 OpenSource Connections and Wikimedia Foundation - - diff --git a/build.gradle b/build.gradle index db6a35da..d8837fe6 100644 --- a/build.gradle +++ b/build.gradle @@ -1,145 +1,173 @@ -import java.nio.file.Files +import org.opensearch.gradle.test.RestIntegTestTask -buildscript { - repositories { - mavenCentral() - mavenLocal() - } - - dependencies { - classpath "org.opensearch.gradle:build-tools:${opensearchVersion}" +apply plugin: 'java' +apply plugin: 'idea' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'opensearch.pluginzip' +apply plugin: 'jacoco' + +//group = 'org.opensearch' +group = 'com.o19s.es' +def pluginName = 'ltr' +def pluginDescription = 'Make Opensearch results more relevant.' +def projectPath = 'com.o19s.es' +def pathToPlugin = 'ltr' +def pluginClassName = 'LtrQueryParserPlugin' + +publishing { + publications { + pluginZip(MavenPublication) { publication -> + pom { + name = pluginName + description = pluginDescription + licenses { + license { + name = "The Apache License, Version 2.0" + url = "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + } + developers { + developer { + name = "OpenSearch" + url = "https://github.com/opensearch-project/search-processor" + } + } + } + } } } - -plugins { - id "co.riiid.gradle" version "0.4.2" +opensearchplugin { + name "opensearch-${pluginName}-${plugin_version}" + description pluginDescription + classname "${projectPath}.${pathToPlugin}.${pluginClassName}" + licenseFile rootProject.file('LICENSE') + noticeFile rootProject.file('NOTICE') } +// This requires an additional Jar not published as part of build-tools +loggerUsageCheck.enabled = false -group = 'com.o19s' -version = "${ltrVersion}-os${opensearchVersion}" - -apply plugin: 'java' -apply plugin: 'idea' -apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.java-rest-test' -apply plugin: 'opensearch.rest-resources' -//apply plugin: 'opensearch.yaml-rest-test' - +// No need to validate pom, as we do not upload to maven/sonatype +validateNebulaPom.enabled = false -idea { - module { - downloadJavadoc = true - downloadSources = true +buildscript { + ext { + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + opensearch_version = System.getProperty("opensearch.version", "2.6.0") + lucene_version = "9.5.0" + plugin_version = opensearch_version + ".0" + if (isSnapshot) { + opensearch_version += "-SNAPSHOT" + plugin_version += "-SNAPSHOT" } -} + println("opensearch_version ${opensearch_version}") + println("plugin_version ${plugin_version}") -// license of this project -licenseFile = rootProject.file('LICENSE.txt') -// copyright notices -noticeFile = rootProject.file('NOTICE.txt') + skipIntegTests = System.getProperty("skip.integtests", "false") + } -// disable uploadArchives task for now, no upload happening currently -//GSI TODO: disabled for now -//uploadArchives.enabled = false + repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } + } -opensearchplugin { - name 'ltr' - description 'Learning to Rank Query w/ RankLib Models' - classname 'com.o19s.es.ltr.LtrQueryParserPlugin' - // license of the plugin, may be different than the above license - licenseFile = rootProject.file('LICENSE.txt') - // copyright notices, may be different than the above notice - noticeFile = rootProject.file('NOTICE.txt') + dependencies { + classpath "org.opensearch.gradle:build-tools:${opensearch_version}" + } } -// In this section you declare the dependencies for your production and test code -// OpenSearch dependency is included due to the build-tools, test-framework as well repositories { - mavenCentral() mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } } dependencies { + implementation "com.github.spullara.mustache.java:compiler:0.9.3" + implementation "org.opensearch:opensearch:${opensearch_version}" + implementation "org.opensearch:opensearch-x-content:${opensearch_version}" + implementation "org.opensearch:opensearch-common:${opensearch_version}" + implementation "org.opensearch:opensearch-core:${opensearch_version}" + implementation "org.opensearch:common-utils:${plugin_version}" + implementation "org.apache.lucene:lucene-expressions:${luceneVersion}" + implementation 'com.o19s:RankyMcRankFace:0.1.1' implementation "org.antlr:antlr4-runtime:${antlrVersion}" implementation "org.ow2.asm:asm:${ow2Version}" - implementation "org.apache.lucene:lucene-expressions:${luceneVersion}" implementation "org.ow2.asm:asm-commons:${ow2Version}" implementation "org.ow2.asm:asm-tree:${ow2Version}" - implementation 'com.o19s:RankyMcRankFace:0.1.1' - implementation "com.github.spullara.mustache.java:compiler:0.9.3" - - testImplementation "org.apache.lucene:lucene-test-framework:${luceneVersion}" - runtimeOnly 'org.locationtech.spatial4j:spatial4j:0.7' - runtimeOnly 'org.locationtech.jts:jts-core:1.15.0' - runtimeOnly 'org.apache.logging.log4j:log4j-core:2.17.1' } - - -// see https://github.com/opensearch-project/OpenSearch/blob/0ba0e7cc26060f964fcbf6ee45bae53b3a9941d0/buildSrc/src/main/java/org/opensearch/gradle/precommit/DependencyLicensesTask.java -dependencyLicenses { - mapping from: /lucene-.*/, to: 'lucene' - mapping from: /asm-.*/, to: 'asm' - mapping from: /RankyMcRankFace.*/, to: 'lucene' - //mapping from: /Ranky.*/, to: 'lucene' - mapping from: /compiler.*/, to: 'lucene' -} - -sourceSets { - javaRestTest { - compileClasspath += sourceSets["main"].output + sourceSets["test"].output + configurations["testRuntimeClasspath"] - runtimeClasspath += output + compileClasspath +allprojects { + plugins.withId('jacoco') { + jacoco.toolVersion = '0.8.9' } } -java { - withJavadocJar() - withSourcesJar() -} +test { + include '**/*Tests.class' + finalizedBy jacocoTestReport +} -// Set to false to not use opensearch checkstyle rules -// checkstyleMain.enabled = true -// checkstyleTest.enabled = true +task integTest(type: RestIntegTestTask) { -// FIXME dependency license check needs to be enabled -// dependencyLicenses.enabled = false + description = "Run tests against a cluster" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +if (!skipIntegTests) { + tasks.named("check").configure { dependsOn(integTest) } +} else { + println("Skipping Integ Tests") +} -// FIXME thirdparty audit needs to be enabled -// thirdPartyAudit.enabled = false +integTest { + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } +} -// Uncomment this to skip license header checks -// licenseHeaders.enabled = false +testClusters.integTest { + testDistribution = "INTEG_TEST" -// No need to validate POM, as we do not upload to sonatype -validateNebulaPom.enabled = false + // This installs our plugin into the testClusters + plugin(project.tasks.bundlePlugin.archiveFile) +} -// Elastic tried to remove the logging requirement for plugins, but didn't get it quite right so this is a short term fix until 7.11 -// https://github.com/elastic/opensearch/issues/65247 -loggerUsageCheck.enabled = false +run { + if (skipIntegTests == "false") { + useCluster testClusters.integTest + } else { + println("skipping Integ Tests ${skipIntegTests}") + } +} -githubRelease.doFirst { - if (!System.getProperty('GITHUB_TOKEN', '')) { - throw new Exception('Missing property GITHUB_TOKEN') +sourceSets { + javaRestTest { + compileClasspath += sourceSets["main"].output + sourceSets["test"].output + configurations["testRuntimeClasspath"] + runtimeClasspath += output + compileClasspath + } + yamlRestTest { + compileClasspath += sourceSets["main"].output + sourceSets["test"].output + configurations["testRuntimeClasspath"] + runtimeClasspath += output + compileClasspath } +} - // check if zip file is there - assert file("build/distributions/ltr-${version}.zip").exists() - - // rename zip file - def currentVersion = version.replace('-SNAPSHOT', '') - def filename = "build/distributions/ltr-${currentVersion}.zip" - Files.copy(file("build/distributions/ltr-${version}.zip").toPath(), file(filename).toPath()) - - // configuration - github { - owner = 'gsingers' - repo = 'opensearch-learning-to-rank-base' - token = System.getProperty('GITHUB_TOKEN') - tagName = currentVersion - assets = [ filename ] - targetCommitish = 'main' +jacocoTestReport { + dependsOn test + reports { + xml.required = true + html.required = true } } + +// TODO: Enable these checks +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false From 377dda953d944bc642784c120da584ccf5ea60b3 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Mon, 23 Oct 2023 00:02:36 +0530 Subject: [PATCH 4/7] add ci and release-old Signed-off-by: Mohammad Hasnain Mohsin Rajan --- .github/workflows/ci.yml | 54 +++++++++++++++++++ .../{release.yml => release-old.yml} | 11 ++-- 2 files changed, 58 insertions(+), 7 deletions(-) rename .github/workflows/{release.yml => release-old.yml} (91%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e69de29b..42ece30a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: Build & Test Learning to Rank Plugin + +on: + schedule: + - cron: '0 0 * * *' # every night + push: + branches: + - "*" + - "feature/**" + pull_request: + branches: + - "*" + - "feature/**" + workflow_dispatch: + +jobs: + Build-ltr-plugin: + strategy: + matrix: + java: [11, 17] + os: [ubuntu-latest, macos-latest] + + name: Build and Test Learning to Rank Plugin + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout LTR Plugin + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Run non-Windows + if: ${{ matrix.os != 'windows-latest' }} + run: | + ./gradlew build -Dopensearch.version=2.7.0 -Dskip.integtests=true \ + -Djava.security.manager=allow -Dbuild.snapshot=false +# - name: Upload Coverage Report +# if: ${{matrix.os}} == 'ubuntu' +# uses: codecov/codecov-action@v1 +# with: +# token: ${{ secrets.CODECOV_TOKEN }} + + - name: Create Artifact Path + run: | + mkdir -p opensearch-ltr-builds + cp -r ./build/distributions/*.zip opensearch-ltr-builds + - name: Upload Artifacts + uses: actions/upload-artifact@v1 + with: + name: opensearch-ltr-${{ matrix.os }} + path: opensearch-ltr-builds diff --git a/.github/workflows/release.yml b/.github/workflows/release-old.yml similarity index 91% rename from .github/workflows/release.yml rename to .github/workflows/release-old.yml index c5286d1e..f1418e87 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release-old.yml @@ -1,13 +1,11 @@ # This workflow will build a Java project with Gradle # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle - name: Tag and publish a release - on: push: tags: - - 'v*.*.*' - branches: [ test-release ] + - "v*.*.*" + branches: [ "${{ github.ref }}", 'test-release' ] jobs: release: @@ -15,8 +13,8 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v2 - - name: Set release version Name + - uses: actions/checkout@v2 + - name: Set release version Name run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Set up JDK 17.0 uses: actions/setup-java@v1 @@ -44,4 +42,3 @@ jobs: #upload_url: ${{ steps.create_release.outputs.upload_url }} files: ./ltr-plugin-${{ env.RELEASE_VERSION }}.zip name: ${{ env.RELEASE_VERSION }} - From a61b42cab0e7ce34469cc9c59b7e2742835f7115 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Mon, 23 Oct 2023 00:08:12 +0530 Subject: [PATCH 5/7] add basic tests Signed-off-by: Mohammad Hasnain Mohsin Rajan --- .../o19s/es/ltr/LearningToRankPluginIT.java | 26 +++++++++++++++++++ .../LearningToRankClientYamlTestSuiteIT.java | 24 +++++++++++++++++ .../resources/rest-api-spec/test/10_basic.yml | 17 ++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 src/test/java/com/o19s/es/ltr/LearningToRankPluginIT.java create mode 100644 src/yamlRestTest/java/com/o19s/es/ltr/LearningToRankClientYamlTestSuiteIT.java create mode 100644 src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml diff --git a/src/test/java/com/o19s/es/ltr/LearningToRankPluginIT.java b/src/test/java/com/o19s/es/ltr/LearningToRankPluginIT.java new file mode 100644 index 00000000..417593de --- /dev/null +++ b/src/test/java/com/o19s/es/ltr/LearningToRankPluginIT.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.o19s.es.ltr; + +import org.apache.http.ParseException; +import org.apache.http.util.EntityUtils; +import org.opensearch.test.rest.OpenSearchRestTestCase; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import java.io.IOException; + +public class LearningToRankPluginIT extends OpenSearchRestTestCase { + public void testPluginInstalled() throws IOException, ParseException { + Response response = client().performRequest(new Request("GET", "/_cat/plugins")); + String body = EntityUtils.toString(response.getEntity()); + + logger.info("response body: {}", body); + assertNotNull(body); + assertTrue(body.contains("search-processor")); + } +} diff --git a/src/yamlRestTest/java/com/o19s/es/ltr/LearningToRankClientYamlTestSuiteIT.java b/src/yamlRestTest/java/com/o19s/es/ltr/LearningToRankClientYamlTestSuiteIT.java new file mode 100644 index 00000000..a02efaf8 --- /dev/null +++ b/src/yamlRestTest/java/com/o19s/es/ltr/LearningToRankClientYamlTestSuiteIT.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.o19s.es.ltr; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; +import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; + +public class LearningToRankClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { + public LearningToRankClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return OpenSearchClientYamlSuiteTestCase.createParameters(); + } + } diff --git a/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml new file mode 100644 index 00000000..ad242c84 --- /dev/null +++ b/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -0,0 +1,17 @@ +"Test that the plugin is loaded in OpenSearch": + - do: + cat.plugins: + local: true + h: component + + - match: + $body: /^opensearch-ltr-.*\n$/ + + - do: + indices.create: + index: test + + - do: + search: + index: test + body: { } From 0c494ae129a6e6a7e0449469a747b42db2b9375e Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Mon, 23 Oct 2023 00:16:05 +0530 Subject: [PATCH 6/7] set opensearch version to 2.6 in ci Signed-off-by: Mohammad Hasnain Mohsin Rajan --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42ece30a..e9a3b033 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: - name: Run non-Windows if: ${{ matrix.os != 'windows-latest' }} run: | - ./gradlew build -Dopensearch.version=2.7.0 -Dskip.integtests=true \ + ./gradlew build -Dopensearch.version=2.6.0 -Dskip.integtests=true \ -Djava.security.manager=allow -Dbuild.snapshot=false # - name: Upload Coverage Report # if: ${{matrix.os}} == 'ubuntu' From 2796168021349b36437c112d9e785532bd7bc435 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Mon, 23 Oct 2023 00:57:12 +0530 Subject: [PATCH 7/7] update known issues and readme Signed-off-by: Mohammad Hasnain Mohsin Rajan --- KNOWN_ISSUES.md | 15 +------------ README.md | 59 ++++++++++++++----------------------------------- 2 files changed, 17 insertions(+), 57 deletions(-) diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index cdcbed0e..2a0f2e33 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -1,15 +1,2 @@ # Known issues - -## Cache deadlock - -All elasticsearch versions between 5.5.3 (included) and 6.3.0 (excluded) are affected by a [bug](https://github.com/elastic/elasticsearch/pull/30461) in the internal cache that may lead to a serious deadlock that could in theory leak all the search threads. -A possible workaround is to configure the cache to avoid as much as possible its eviction mechanism by setting these options in your `elasticsearch.yml` file: - -``` -ltr.caches.expire_after_write: 0 -ltr.caches.expire_after_read: 0 -ltr.caches.max_mem: 100mb -``` - -elasticsearch versions 5.6.10, 6.3.0 and onwards should include the bugfix for this bug. - +2.6.0: yamlresttests fail; set -Dbuild.snapshot=false to build successfully diff --git a/README.md b/README.md index a15db81a..11c27ed4 100644 --- a/README.md +++ b/README.md @@ -9,15 +9,16 @@ The OpenSearch Learning to Rank plugin uses machine learning to improve search r To install, you'd run a command like this but replacing with the appropriate prebuilt version zip: -| OS | Command | -|-------|--------------------------------------------------------------------------------------------------------------------------------------------| -| 1.0.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.0.0/ltr-1.5.4-os1.0.0.zip` | -| 1.1.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.1.0/ltr-1.5.4-os1.1.0.zip` | -| 1.2.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.0/ltr-1.5.4-os1.2.0.zip` | -| 1.2.2 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.2/ltr-1.5.4-os1.2.2.zip` | -| 1.2.3 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.3/ltr-1.5.4-os1.2.3.zip` | -| 2.2.1 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/2.2.1/ltr-2.0.0-os2.2.1.zip` | -| 2.5.0 | `bin/opensearch-plugin install https://github.com/gsingers/opensearch-learning-to-rank-base/releases/download/release-v2.1.0/ltr-plugin-v2.1.0.zip` | +| OS | Command | +|--------|--------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.0.0/ltr-1.5.4-os1.0.0.zip` | +| 1.1.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.1.0/ltr-1.5.4-os1.1.0.zip` | +| 1.2.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.0/ltr-1.5.4-os1.2.0.zip` | +| 1.2.2 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.2/ltr-1.5.4-os1.2.2.zip` | +| 1.2.3 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.3/ltr-1.5.4-os1.2.3.zip` | +| 2.2.1 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/2.2.1/ltr-2.0.0-os2.2.1.zip` | +| 2.5.0 | `bin/opensearch-plugin install https://github.com/gsingers/opensearch-learning-to-rank-base/releases/download/release-v2.1.0/ltr-plugin-v2.1.0.zip` | +| 2.6.0 | `bin/opensearch-plugin install https://github.com/opensearch-project/opensearch-learning-to-rank-base/releases/download/release-v2.6.0/ltr-plugin-v2.6.0.zip` | (It's expected you'll confirm some security exceptions, you can pass `-b` to `opensearch-plugin` to automatically install) @@ -26,7 +27,7 @@ If you already are running OpenSearch, don't forget to restart! # Releases -Releases can be found at https://github.com/gsingers/opensearch-learning-to-rank-base/releases. +Releases can be found at https://github.com/opensearch-project/opensearch-learning-to-rank-base/releases. ## Releasing/Packaging @@ -34,50 +35,22 @@ Releases can be found at https://github.com/gsingers/opensearch-learning-to-rank Releases are done through Github Workflows (see `.github/workflows` in the root directory) on an as needed basis. If you do `./gradlew build` as per above under building, it will build all the artifacts that are in the release. -## About alpha releases - -These releases are alpha because some issues with the tests due to securemock that depends on ElasticSearch security stuff. -And there are 14 failing tests. - -``` -Tests with failures: -- com.o19s.es.ltr.feature.store.StoredFeatureSetParserTests.testExpressionDoubleQueryParameter -- com.o19s.es.ltr.feature.store.StoredFeatureSetParserTests.testExpressionMissingQueryParameter -- com.o19s.es.ltr.feature.store.StoredFeatureSetParserTests.testExpressionIntegerQueryParameter -- com.o19s.es.ltr.feature.store.StoredFeatureSetParserTests.testExpressionShortQueryParameter -- com.o19s.es.ltr.feature.store.StoredFeatureSetParserTests.testExpressionInvalidQueryParameter -- com.o19s.es.termstat.TermStatQueryBuilderTests.testMustRewrite -- com.o19s.es.termstat.TermStatQueryBuilderTests.testToQuery -- com.o19s.es.termstat.TermStatQueryBuilderTests.testCacheability -- com.o19s.es.ltr.feature.store.StoredFeatureParserTests.testExpressionOptimization -- com.o19s.es.termstat.TermStatQueryTests.testEmptyTerms -- com.o19s.es.termstat.TermStatQueryTests.testUniqueCount -- com.o19s.es.termstat.TermStatQueryTests.testBasicFormula -- com.o19s.es.termstat.TermStatQueryTests.testQuery -- com.o19s.es.termstat.TermStatQueryTests.testMatchCount - -228 tests completed, 14 failed -``` - - # Development -To build, you need to disable the Java security manager +To build, you need to explicitly enable Java security and disable snapshot builds (until the YamlRestTests are fixed): -./gradlew -Dtests.security.manager=false clean build +./gradlew -Dopensearch.version={opensearch-version-to-build-on} -Djava.security.manager=allow -Dbuild.snapshot=false # Upgrading the OpenSearch Versions -1. Edit `gradle.properties` to have the appropriate versions (it's often easiest to go download the latest tarball from OpenSearch and simply check the versions that ship) and to increment the version of this plugin -2. Build and test as above -3. Update this README with the version info in the table above -4. Upgrade the Docker file versions in the `docker` directory +1. Build and test as above +2. Update this README with the version info in the table above +3. Upgrade the Docker file versions in the `docker` directory 4. Test the docker image, per below. ## Development Notes - # Docker A custom image of [OpenSearch](https://hub.docker.com/r/opensearchproject/opensearch) with the [OpenSearch Learning to Rank plugin](https://github.com/gsingers/opensearch-learning-to-rank-base) installed.