From efdd61cf4d8ca7e5e11cb771c8b57e01288a23e6 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 18 Dec 2023 20:56:33 +0000 Subject: [PATCH 1/3] Run ML plugin on local cluster Signed-off-by: Tyler Ohlsen --- build.gradle | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d8f23cfa..0788c49a 100644 --- a/build.gradle +++ b/build.gradle @@ -3,12 +3,23 @@ * SPDX-License-Identifier: Apache-2.0 */ +import org.opensearch.gradle.test.RestIntegTestTask +import java.util.concurrent.Callable + buildscript { ext { opensearch_group = "org.opensearch" - opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.11.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") + version_tokens = opensearch_version.tokenize('-') + opensearch_build = version_tokens[0] + '.0' + if (buildVersionQualifier) { + opensearch_build += "-${buildVersionQualifier}" + } + if (isSnapshot) { + opensearch_build += "-SNAPSHOT" + } } repositories { @@ -104,6 +115,9 @@ dependencies { testImplementation "com.cronutils:cron-utils:9.2.1" testImplementation "commons-validator:commons-validator:1.8.0" testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.10.1' + + // ZipArchive dependencies used for integration tests + zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}" } task extractSqlJar(type: Copy) { @@ -205,6 +219,92 @@ publishing { gradle.startParameter.setLogLevel(LogLevel.DEBUG) } +def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile +opensearch_tmp_dir.mkdirs() +def _numNodes = findProperty('numNodes') as Integer ?: 1 + +// Set up integration tests +task integTest(type: RestIntegTestTask) { + description = "Run tests against a cluster" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +tasks.named("check").configure { dependsOn(integTest) } + +integTest { + + dependsOn "bundlePlugin" + systemProperty 'tests.security.manager', 'false' + systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath + systemProperty('project.root', project.rootDir.absolutePath) + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + + + // doFirst delays this block until execution time + doFirst { + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can + // use longer timeouts for requests. + def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null + systemProperty 'cluster.debug', isDebuggingCluster + // Set number of nodes system property to be used in tests + systemProperty 'cluster.number_of_nodes', "${_numNodes}" + // There seems to be an issue when running multi node run or integ tasks with unicast_hosts + // not being written, the waitForAllConditions ensures it's written + getClusters().forEach { cluster -> + cluster.waitForAllConditions() + } + } + + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } +} + +// Set up integration test clusters, installs all zipArchive dependencies and Flow Framework +testClusters.integTest { + testDistribution = "ARCHIVE" + + // Installs all registered zipArchive dependencies on integTest cluster nodes + configurations.zipArchive.asFileTree.each { + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return it + } + } + } + })) + } + + // Install Flow Framwork Plugin on integTest cluster nodes + plugin(project.tasks.bundlePlugin.archiveFile) + + // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 + if (_numNodes > 1) numberOfNodes = _numNodes + + // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore + // i.e. we have to use a custom property to flag when we want to debug OpenSearch JVM + // since we also support multi node integration tests we increase debugPort per node + if (System.getProperty("opensearch.debug") != null) { + def debugPort = 5005 + nodes.forEach { node -> + node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}") + debugPort += 1 + } + } +} + +// Automatically sets up the integration test cluster locally +run { + useCluster testClusters.integTest +} + // updateVersion: Task to auto increment to the next development iteration task updateVersion { onlyIf { System.getProperty('newVersion') } From bbe784801721ad5eff0bd62c33b4cd651b8795e3 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 18 Dec 2023 21:00:58 +0000 Subject: [PATCH 2/3] Revert to 3.0 Signed-off-by: Tyler Ohlsen --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 0788c49a..601d5abf 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ import java.util.concurrent.Callable buildscript { ext { opensearch_group = "org.opensearch" - opensearch_version = System.getProperty("opensearch.version", "2.11.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') From c58cdc8d6cd03aefc7c5fad5a25e4298c3ea1185 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 18 Dec 2023 21:05:04 +0000 Subject: [PATCH 3/3] Update comments Signed-off-by: Tyler Ohlsen --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 601d5abf..e40ac6b6 100644 --- a/build.gradle +++ b/build.gradle @@ -263,7 +263,7 @@ integTest { } } -// Set up integration test clusters, installs all zipArchive dependencies and Flow Framework +// Set up integration test clusters, installs all zipArchive dependencies and this plugin testClusters.integTest { testDistribution = "ARCHIVE" @@ -282,7 +282,7 @@ testClusters.integTest { })) } - // Install Flow Framwork Plugin on integTest cluster nodes + // Install skills plugin on integTest cluster nodes plugin(project.tasks.bundlePlugin.archiveFile) // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1